diff --git a/services/web/app/src/Features/Project/ProjectController.mjs b/services/web/app/src/Features/Project/ProjectController.mjs index 11ee8bd13c..ec1d820678 100644 --- a/services/web/app/src/Features/Project/ProjectController.mjs +++ b/services/web/app/src/Features/Project/ProjectController.mjs @@ -520,6 +520,7 @@ const _ProjectController = { 'compile-with-checkpoint', 'themed-modals', 'intermittent-connection-improvements', + 'symbol-recognition', ].filter(Boolean) const getUserValues = async userId => diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 4c1ef76182..01f9bb36a9 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -307,6 +307,7 @@ "cite_a_reference_from_your_reference_manager_to_automatically_add_it_to_your_bib_file": "", "cite_directly_or_import_references": "", "cite_faster": "", + "clear": "", "clear_cached_files": "", "clear_filters": "", "clear_search": "", @@ -2050,6 +2051,11 @@ "switch_to_pdf": "", "symbol": "", "symbol_palette": "", + "symbol_palette_draw_a_symbol": "", + "symbol_palette_draw_instructions": "", + "symbol_palette_loading_model": "", + "symbol_palette_recognised_symbols": "", + "symbol_palette_recognition_unavailable": "", "sync": "", "sync_all_projects_with_dropbox": "", "sync_dropbox_github": "", diff --git a/services/web/frontend/js/features/ide-react/components/layout/editor.tsx b/services/web/frontend/js/features/ide-react/components/layout/editor.tsx index 5e757304e6..a86f6728b9 100644 --- a/services/web/frontend/js/features/ide-react/components/layout/editor.tsx +++ b/services/web/frontend/js/features/ide-react/components/layout/editor.tsx @@ -11,6 +11,7 @@ import SymbolPalettePane from '@/features/ide-react/components/editor/symbol-pal import { useEditorPropertiesContext } from '@/features/ide-react/context/editor-properties-context' import { isSplitTestEnabled } from '@/utils/splitTestUtils' import importOverleafModules from '../../../../../macros/import-overleaf-module.macro' +import { useFeatureFlag } from '@/shared/context/split-test-context.tsx' const [pythonRunnerModule] = importOverleafModules('pythonRunner') as { import: { PythonEditorSplit: FC } @@ -22,6 +23,8 @@ export const Editor = () => { const { selectedEntityCount, openEntity } = useFileTreeOpenContext() const { currentDocumentId, currentDocument } = useEditorOpenDocContext() + const drawEnabled = useFeatureFlag('symbol-recognition') + if (!currentDocumentId) { return null } @@ -64,7 +67,7 @@ export const Editor = () => { diff --git a/services/web/frontend/stylesheets/modules/symbol-palette.scss b/services/web/frontend/stylesheets/modules/symbol-palette.scss index 66c9f7169e..cb7d64184c 100644 --- a/services/web/frontend/stylesheets/modules/symbol-palette.scss +++ b/services/web/frontend/stylesheets/modules/symbol-palette.scss @@ -45,7 +45,7 @@ justify-content: space-between; align-items: stretch; font-family: $font-family-sans-serif; - font-size: var(--font-size-03); + font-size: var(--font-size-02); background: var(--symbol-palette-header-background); box-shadow: inset 0 1px 0 rgb(255 255 255 / 10%); } @@ -105,6 +105,7 @@ line-height: 42px; height: 42px; width: 42px; + padding: 0; margin: var(--spacing-03); color: var(--symbol-palette-item-color); background: var(--symbol-palette-item-bg); @@ -115,6 +116,18 @@ justify-content: center; } +// Recognised commands with no Unicode glyph show their command name instead of +// an empty square. Shrink and wrap the name to fit the fixed glyph box; the full +// \command is in the tooltip. +.symbol-palette-item-label { + font-family: monospace; + font-size: 9px; + line-height: 1.1; + padding: 2px; + overflow: hidden; + overflow-wrap: anywhere; +} + .symbol-palette-item-command { font-family: monospace; font-weight: bold; @@ -146,6 +159,24 @@ margin-left: var(--spacing-03); } +.symbol-palette-draw-toggle { + display: inline-flex; + align-items: center; + align-self: center; // don't stretch to the full header height + margin: 0 var(--spacing-02); + padding: var(--spacing-02); + border: none; + border-radius: var(--border-radius-base); + background: transparent; + color: inherit; + cursor: pointer; + + &[aria-pressed='true'] { + background: var(--bg-accent-01); + color: var(--content-primary-dark); // white icon on the accent background + } +} + .symbol-palette-info-link, .symbol-palette-info-link:focus, .symbol-palette-info-link:hover { @@ -195,3 +226,69 @@ column-count: 2; } } + +.symbol-palette-draw { + display: flex; + flex-direction: row; + gap: var(--spacing-04); + height: 100%; + padding: var(--spacing-04); +} + +// left column: recognised-results grid, scrolls independently +.symbol-palette-draw-results { + flex: 1 1 auto; + min-width: 0; + overflow-y: auto; +} + +// shown in the results column while the canvas is empty +.symbol-palette-draw-instructions { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + margin: 0; + padding: var(--spacing-06); + text-align: center; + opacity: 0.7; +} + +// right column: square canvas pinned to the panel height +.symbol-palette-draw-canvas-panel { + flex: 0 0 auto; + display: flex; + flex-direction: column; + gap: var(--spacing-02); + height: 100%; + aspect-ratio: 1; +} + +.symbol-palette-draw-canvas { + flex: 1 1 auto; + width: 100%; + min-height: 0; + border: 1px dashed var(--border-primary); + border-radius: var(--border-radius-base); + background: var(--bg-light-primary); + touch-action: none; +} + +.symbol-palette-draw-toolbar { + display: flex; + flex-shrink: 0; + align-items: center; + gap: var(--spacing-04); + min-height: 20px; + font-size: var(--font-size-01); +} + +.symbol-palette-draw-clear { + appearance: none; + border: 1px solid var(--border-primary); + border-radius: var(--border-radius-base); + background: transparent; + color: inherit; + cursor: pointer; + padding: var(--spacing-01) var(--spacing-05); +} diff --git a/services/web/locales/en.json b/services/web/locales/en.json index 2f95a31e40..ad4cbad9b0 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -395,6 +395,7 @@ "cite_directly_or_import_references": "Cite directly or import references", "cite_faster": "Cite faster", "city": "City", + "clear": "Clear", "clear_cached_files": "Clear cached files", "clear_filters": "Clear filters", "clear_search": "clear search", @@ -2633,8 +2634,13 @@ "switch_to_pdf": "Switch to PDF", "symbol": "Symbol", "symbol_palette": "Symbol palette", + "symbol_palette_draw_a_symbol": "Draw a symbol", + "symbol_palette_draw_instructions": "Draw a symbol in the box on the right, then choose the matching command from the results shown here.", "symbol_palette_explanation": "Insert math symbols with the click of a button.", "symbol_palette_info_new": "Insert math symbols into your document with the click of a button.", + "symbol_palette_loading_model": "Loading model…", + "symbol_palette_recognised_symbols": "Recognised symbols", + "symbol_palette_recognition_unavailable": "Recognition unavailable", "sync": "Sync", "sync_all_projects_with_dropbox": "Sync all projects with Dropbox.", "sync_dropbox_github": "Sync with Dropbox and GitHub", diff --git a/services/web/package.json b/services/web/package.json index 672a3e0c35..52e971dc62 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -394,6 +394,7 @@ "nock": "^13.5.6", "nvd3": "^1.8.6", "nyc": "^17.1.0", + "onnxruntime-web": "1.26.0", "p-reflect": "^3.1.0", "path-browserify": "^1.0.1", "pdfjs-dist": "5.1.91", diff --git a/services/web/test/frontend/bootstrap.js b/services/web/test/frontend/bootstrap.js index 72c471c266..b2abdb8767 100644 --- a/services/web/test/frontend/bootstrap.js +++ b/services/web/test/frontend/bootstrap.js @@ -126,7 +126,7 @@ globalThis.fetch = // ignore style/image files const { addHook } = require('pirates') addHook(() => '', { - exts: ['.css', '.scss', '.svg', '.png', '.gif', '.mp4'], + exts: ['.css', '.scss', '.svg', '.png', '.gif', '.mp4', '.ort'], ignoreNodeModules: false, }) diff --git a/services/web/types/assets.d.ts b/services/web/types/assets.d.ts index fca1c29b40..4c8e111976 100644 --- a/services/web/types/assets.d.ts +++ b/services/web/types/assets.d.ts @@ -18,6 +18,11 @@ declare module '*.mp4' { export default src } +declare module '*.ort' { + const src: string + export default src +} + declare module '*.wasm' { const src: string export default src diff --git a/services/web/webpack.config.js b/services/web/webpack.config.js index 2867cb541a..4c573e5098 100644 --- a/services/web/webpack.config.js +++ b/services/web/webpack.config.js @@ -1,4 +1,4 @@ -const path = require('path') +const path = require('node:path') const { globSync } = require('glob') const webpack = require('webpack') const CopyPlugin = require('copy-webpack-plugin') @@ -149,6 +149,7 @@ module.exports = { /node_modules\/(?!(react-dnd|chart\.js|@uppy|@writefull|pdfjs-dist|react-resizable-panels)\/)/, vendorDir, path.resolve(__dirname, 'modules/writefull/frontend/js/integration'), + /ort-wasm-simd-threaded\.mjs$/, ], use: [ { @@ -174,6 +175,22 @@ module.exports = { filename: 'js/[name]-[contenthash][ext]', }, }, + { + // ONNX Runtime model files (symbol-recognition) + test: /\.ort$/, + type: 'asset/resource', + generator: { + filename: 'js/[name]-[contenthash][ext]', + }, + }, + { + // The reduced onnxruntime-web wasm glue (symbol-recognition) + test: /ort-wasm-simd-threaded\.mjs$/, + type: 'asset/resource', + generator: { + filename: 'js/[name]-[contenthash][ext]', + }, + }, { test: /\.txt$/, type: 'asset/source', @@ -474,7 +491,7 @@ module.exports = { to: 'images/pdfjs-dist', context: pdfjsDir, }, - ], + ].filter(item => !!item), }), ], } diff --git a/yarn.lock b/yarn.lock index 9abf373aa2..1649b1c7c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8191,6 +8191,7 @@ __metadata: nvd3: "npm:^1.8.6" nyc: "npm:^17.1.0" on-headers: "npm:^1.0.2" + onnxruntime-web: "npm:1.26.0" otplib: "npm:^12.0.1" overleaf-editor-core: "workspace:*" p-limit: "npm:^2.3.0" @@ -19119,6 +19120,13 @@ __metadata: languageName: node linkType: hard +"flatbuffers@npm:^25.1.24": + version: 25.9.23 + resolution: "flatbuffers@npm:25.9.23" + checksum: 10c0/957c4ae2a02be1703c98b36b4dc8ceb81613cf8e2333026afc95a7b68b088bed5458056dc29d0ab7ce8bc403b8c003732b0968d24aba46f5e7c8f71789a6bd9e + languageName: node + linkType: hard + "flatted@npm:^3.2.9, flatted@npm:^3.4.2": version: 3.4.2 resolution: "flatted@npm:3.4.2" @@ -20064,6 +20072,13 @@ __metadata: languageName: node linkType: hard +"guid-typescript@npm:^1.0.9": + version: 1.0.9 + resolution: "guid-typescript@npm:1.0.9" + checksum: 10c0/fa0a2b2b4e06e0976a81c947b74e114b92f6647e84b52e24ab0981d2fdbaa1a640641d8fd269004dd7c581baebeb4f9d9782b74391e717e47c9b822bea4b3be6 + languageName: node + linkType: hard + "gulp-sort@npm:^2.0.0": version: 2.0.0 resolution: "gulp-sort@npm:2.0.0" @@ -23260,7 +23275,7 @@ __metadata: languageName: node linkType: hard -"long@npm:^5.0.0, long@npm:^5.3.2": +"long@npm:^5.0.0, long@npm:^5.2.3, long@npm:^5.3.2": version: 5.3.2 resolution: "long@npm:5.3.2" checksum: 10c0/7130fe1cbce2dca06734b35b70d380ca3f70271c7f8852c922a7c62c86c4e35f0c39290565eca7133c625908d40e126ac57c02b1b1a4636b9457d77e1e60b981 @@ -25837,6 +25852,27 @@ __metadata: languageName: node linkType: hard +"onnxruntime-common@npm:1.26.0": + version: 1.26.0 + resolution: "onnxruntime-common@npm:1.26.0" + checksum: 10c0/220679e2b0129fcc8efd710e1502f792ee5583173d261840369c2dd07bbfdcb9dbbeb027002131094f05e0d287354f7cf66605084b6eb0d90d9543fe485c8c6f + languageName: node + linkType: hard + +"onnxruntime-web@npm:1.26.0": + version: 1.26.0 + resolution: "onnxruntime-web@npm:1.26.0" + dependencies: + flatbuffers: "npm:^25.1.24" + guid-typescript: "npm:^1.0.9" + long: "npm:^5.2.3" + onnxruntime-common: "npm:1.26.0" + platform: "npm:^1.3.6" + protobufjs: "npm:^7.2.4" + checksum: 10c0/6e63265dd2810c22897843c140b5c6b5912f285274f5f743af83ebdfeded863e5dea9f92e5f246cf72a0eb8ae4f38e64990869f6cd7472a0f20940b19b362c83 + languageName: node + linkType: hard + "open@npm:^10.0.3, open@npm:^10.2.0": version: 10.2.0 resolution: "open@npm:10.2.0" @@ -27048,7 +27084,7 @@ __metadata: languageName: node linkType: hard -"platform@npm:^1.3.3": +"platform@npm:^1.3.3, platform@npm:^1.3.6": version: 1.3.6 resolution: "platform@npm:1.3.6" checksum: 10c0/69f2eb692e15f1a343dd0d9347babd9ca933824c8673096be746ff66f99f2bdc909fadd8609076132e6ec768349080babb7362299f2a7f885b98f1254ae6224b