Add Draw mode to the symbol palette (#34868)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
GitOrigin-RevId: 4ea4f97023e65fdb8b018f610f77d7dd107c8e76

services/web/app/src/Features/Project/ProjectController.mjs

@@ -520,6 +520,7 @@ const _ProjectController = {
       'compile-with-checkpoint',
       'themed-modals',
       'intermittent-connection-improvements',
+      'symbol-recognition',
     ].filter(Boolean)
 
     const getUserValues = async userId =>

services/web/frontend/extracted-translations.json

@@ -307,6 +307,7 @@
   "cite_a_reference_from_your_reference_manager_to_automatically_add_it_to_your_bib_file": "",
   "cite_directly_or_import_references": "",
   "cite_faster": "",
+  "clear": "",
   "clear_cached_files": "",
   "clear_filters": "",
   "clear_search": "",
@@ -2050,6 +2051,11 @@
   "switch_to_pdf": "",
   "symbol": "",
   "symbol_palette": "",
+  "symbol_palette_draw_a_symbol": "",
+  "symbol_palette_draw_instructions": "",
+  "symbol_palette_loading_model": "",
+  "symbol_palette_recognised_symbols": "",
+  "symbol_palette_recognition_unavailable": "",
   "sync": "",
   "sync_all_projects_with_dropbox": "",
   "sync_dropbox_github": "",

services/web/frontend/js/features/ide-react/components/layout/editor.tsx

@@ -11,6 +11,7 @@ import SymbolPalettePane from '@/features/ide-react/components/editor/symbol-pal
 import { useEditorPropertiesContext } from '@/features/ide-react/context/editor-properties-context'
 import { isSplitTestEnabled } from '@/utils/splitTestUtils'
 import importOverleafModules from '../../../../../macros/import-overleaf-module.macro'
+import { useFeatureFlag } from '@/shared/context/split-test-context.tsx'
 
 const [pythonRunnerModule] = importOverleafModules('pythonRunner') as {
   import: { PythonEditorSplit: FC }
@@ -22,6 +23,8 @@ export const Editor = () => {
   const { selectedEntityCount, openEntity } = useFileTreeOpenContext()
   const { currentDocumentId, currentDocument } = useEditorOpenDocContext()
 
+  const drawEnabled = useFeatureFlag('symbol-recognition')
+
   if (!currentDocumentId) {
     return null
   }
@@ -64,7 +67,7 @@ export const Editor = () => {
             <Panel
               id="ide-redesign-panel-symbol-palette"
               order={2}
-              defaultSize={25}
+              defaultSize={drawEnabled ? 50 : 25}
               minSize={10}
               maxSize={50}
             >

services/web/frontend/stylesheets/modules/symbol-palette.scss

@@ -45,7 +45,7 @@
   justify-content: space-between;
   align-items: stretch;
   font-family: $font-family-sans-serif;
-  font-size: var(--font-size-03);
+  font-size: var(--font-size-02);
   background: var(--symbol-palette-header-background);
   box-shadow: inset 0 1px 0 rgb(255 255 255 / 10%);
 }
@@ -105,6 +105,7 @@
   line-height: 42px;
   height: 42px;
   width: 42px;
+  padding: 0;
   margin: var(--spacing-03);
   color: var(--symbol-palette-item-color);
   background: var(--symbol-palette-item-bg);
@@ -115,6 +116,18 @@
   justify-content: center;
 }
 
+// Recognised commands with no Unicode glyph show their command name instead of
+// an empty square. Shrink and wrap the name to fit the fixed glyph box; the full
+// \command is in the tooltip.
+.symbol-palette-item-label {
+  font-family: monospace;
+  font-size: 9px;
+  line-height: 1.1;
+  padding: 2px;
+  overflow: hidden;
+  overflow-wrap: anywhere;
+}
+
 .symbol-palette-item-command {
   font-family: monospace;
   font-weight: bold;
@@ -146,6 +159,24 @@
   margin-left: var(--spacing-03);
 }
 
+.symbol-palette-draw-toggle {
+  display: inline-flex;
+  align-items: center;
+  align-self: center; // don't stretch to the full header height
+  margin: 0 var(--spacing-02);
+  padding: var(--spacing-02);
+  border: none;
+  border-radius: var(--border-radius-base);
+  background: transparent;
+  color: inherit;
+  cursor: pointer;
+
+  &[aria-pressed='true'] {
+    background: var(--bg-accent-01);
+    color: var(--content-primary-dark); // white icon on the accent background
+  }
+}
+
 .symbol-palette-info-link,
 .symbol-palette-info-link:focus,
 .symbol-palette-info-link:hover {
@@ -195,3 +226,69 @@
     column-count: 2;
   }
 }
+
+.symbol-palette-draw {
+  display: flex;
+  flex-direction: row;
+  gap: var(--spacing-04);
+  height: 100%;
+  padding: var(--spacing-04);
+}
+
+// left column: recognised-results grid, scrolls independently
+.symbol-palette-draw-results {
+  flex: 1 1 auto;
+  min-width: 0;
+  overflow-y: auto;
+}
+
+// shown in the results column while the canvas is empty
+.symbol-palette-draw-instructions {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  height: 100%;
+  margin: 0;
+  padding: var(--spacing-06);
+  text-align: center;
+  opacity: 0.7;
+}
+
+// right column: square canvas pinned to the panel height
+.symbol-palette-draw-canvas-panel {
+  flex: 0 0 auto;
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-02);
+  height: 100%;
+  aspect-ratio: 1;
+}
+
+.symbol-palette-draw-canvas {
+  flex: 1 1 auto;
+  width: 100%;
+  min-height: 0;
+  border: 1px dashed var(--border-primary);
+  border-radius: var(--border-radius-base);
+  background: var(--bg-light-primary);
+  touch-action: none;
+}
+
+.symbol-palette-draw-toolbar {
+  display: flex;
+  flex-shrink: 0;
+  align-items: center;
+  gap: var(--spacing-04);
+  min-height: 20px;
+  font-size: var(--font-size-01);
+}
+
+.symbol-palette-draw-clear {
+  appearance: none;
+  border: 1px solid var(--border-primary);
+  border-radius: var(--border-radius-base);
+  background: transparent;
+  color: inherit;
+  cursor: pointer;
+  padding: var(--spacing-01) var(--spacing-05);
+}

services/web/locales/en.json

@@ -395,6 +395,7 @@
   "cite_directly_or_import_references": "Cite directly or import references",
   "cite_faster": "Cite faster",
   "city": "City",
+  "clear": "Clear",
   "clear_cached_files": "Clear cached files",
   "clear_filters": "Clear filters",
   "clear_search": "clear search",
@@ -2633,8 +2634,13 @@
   "switch_to_pdf": "Switch to PDF",
   "symbol": "Symbol",
   "symbol_palette": "Symbol palette",
+  "symbol_palette_draw_a_symbol": "Draw a symbol",
+  "symbol_palette_draw_instructions": "Draw a symbol in the box on the right, then choose the matching command from the results shown here.",
   "symbol_palette_explanation": "Insert math symbols with the click of a button.",
   "symbol_palette_info_new": "Insert math symbols into your document with the click of a button.",
+  "symbol_palette_loading_model": "Loading model…",
+  "symbol_palette_recognised_symbols": "Recognised symbols",
+  "symbol_palette_recognition_unavailable": "Recognition unavailable",
   "sync": "Sync",
   "sync_all_projects_with_dropbox": "Sync all projects with Dropbox.",
   "sync_dropbox_github": "Sync with Dropbox and GitHub",

services/web/package.json

@@ -394,6 +394,7 @@
     "nock": "^13.5.6",
     "nvd3": "^1.8.6",
     "nyc": "^17.1.0",
+    "onnxruntime-web": "1.26.0",
     "p-reflect": "^3.1.0",
     "path-browserify": "^1.0.1",
     "pdfjs-dist": "5.1.91",

services/web/test/frontend/bootstrap.js

@@ -126,7 +126,7 @@ globalThis.fetch =
 // ignore style/image files
 const { addHook } = require('pirates')
 addHook(() => '', {
-  exts: ['.css', '.scss', '.svg', '.png', '.gif', '.mp4'],
+  exts: ['.css', '.scss', '.svg', '.png', '.gif', '.mp4', '.ort'],
   ignoreNodeModules: false,
 })
 

services/web/types/assets.d.ts

@@ -18,6 +18,11 @@ declare module '*.mp4' {
   export default src
 }
 
+declare module '*.ort' {
+  const src: string
+  export default src
+}
+
 declare module '*.wasm' {
   const src: string
   export default src

services/web/webpack.config.js

@@ -1,4 +1,4 @@
-const path = require('path')
+const path = require('node:path')
 const { globSync } = require('glob')
 const webpack = require('webpack')
 const CopyPlugin = require('copy-webpack-plugin')
@@ -149,6 +149,7 @@ module.exports = {
           /node_modules\/(?!(react-dnd|chart\.js|@uppy|@writefull|pdfjs-dist|react-resizable-panels)\/)/,
           vendorDir,
           path.resolve(__dirname, 'modules/writefull/frontend/js/integration'),
+          /ort-wasm-simd-threaded\.mjs$/,
         ],
         use: [
           {
@@ -174,6 +175,22 @@ module.exports = {
           filename: 'js/[name]-[contenthash][ext]',
         },
       },
+      {
+        // ONNX Runtime model files (symbol-recognition)
+        test: /\.ort$/,
+        type: 'asset/resource',
+        generator: {
+          filename: 'js/[name]-[contenthash][ext]',
+        },
+      },
+      {
+        // The reduced onnxruntime-web wasm glue (symbol-recognition)
+        test: /ort-wasm-simd-threaded\.mjs$/,
+        type: 'asset/resource',
+        generator: {
+          filename: 'js/[name]-[contenthash][ext]',
+        },
+      },
       {
         test: /\.txt$/,
         type: 'asset/source',
@@ -474,7 +491,7 @@ module.exports = {
           to: 'images/pdfjs-dist',
           context: pdfjsDir,
         },
-      ],
+      ].filter(item => !!item),
     }),
   ],
 }

yarn.lock

@@ -8191,6 +8191,7 @@ __metadata:
     nvd3: "npm:^1.8.6"
     nyc: "npm:^17.1.0"
     on-headers: "npm:^1.0.2"
+    onnxruntime-web: "npm:1.26.0"
     otplib: "npm:^12.0.1"
     overleaf-editor-core: "workspace:*"
     p-limit: "npm:^2.3.0"
@@ -19119,6 +19120,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"flatbuffers@npm:^25.1.24":
+  version: 25.9.23
+  resolution: "flatbuffers@npm:25.9.23"
+  checksum: 10c0/957c4ae2a02be1703c98b36b4dc8ceb81613cf8e2333026afc95a7b68b088bed5458056dc29d0ab7ce8bc403b8c003732b0968d24aba46f5e7c8f71789a6bd9e
+  languageName: node
+  linkType: hard
+
 "flatted@npm:^3.2.9, flatted@npm:^3.4.2":
   version: 3.4.2
   resolution: "flatted@npm:3.4.2"
@@ -20064,6 +20072,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"guid-typescript@npm:^1.0.9":
+  version: 1.0.9
+  resolution: "guid-typescript@npm:1.0.9"
+  checksum: 10c0/fa0a2b2b4e06e0976a81c947b74e114b92f6647e84b52e24ab0981d2fdbaa1a640641d8fd269004dd7c581baebeb4f9d9782b74391e717e47c9b822bea4b3be6
+  languageName: node
+  linkType: hard
+
 "gulp-sort@npm:^2.0.0":
   version: 2.0.0
   resolution: "gulp-sort@npm:2.0.0"
@@ -23260,7 +23275,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"long@npm:^5.0.0, long@npm:^5.3.2":
+"long@npm:^5.0.0, long@npm:^5.2.3, long@npm:^5.3.2":
   version: 5.3.2
   resolution: "long@npm:5.3.2"
   checksum: 10c0/7130fe1cbce2dca06734b35b70d380ca3f70271c7f8852c922a7c62c86c4e35f0c39290565eca7133c625908d40e126ac57c02b1b1a4636b9457d77e1e60b981
@@ -25837,6 +25852,27 @@ __metadata:
   languageName: node
   linkType: hard
 
+"onnxruntime-common@npm:1.26.0":
+  version: 1.26.0
+  resolution: "onnxruntime-common@npm:1.26.0"
+  checksum: 10c0/220679e2b0129fcc8efd710e1502f792ee5583173d261840369c2dd07bbfdcb9dbbeb027002131094f05e0d287354f7cf66605084b6eb0d90d9543fe485c8c6f
+  languageName: node
+  linkType: hard
+
+"onnxruntime-web@npm:1.26.0":
+  version: 1.26.0
+  resolution: "onnxruntime-web@npm:1.26.0"
+  dependencies:
+    flatbuffers: "npm:^25.1.24"
+    guid-typescript: "npm:^1.0.9"
+    long: "npm:^5.2.3"
+    onnxruntime-common: "npm:1.26.0"
+    platform: "npm:^1.3.6"
+    protobufjs: "npm:^7.2.4"
+  checksum: 10c0/6e63265dd2810c22897843c140b5c6b5912f285274f5f743af83ebdfeded863e5dea9f92e5f246cf72a0eb8ae4f38e64990869f6cd7472a0f20940b19b362c83
+  languageName: node
+  linkType: hard
+
 "open@npm:^10.0.3, open@npm:^10.2.0":
   version: 10.2.0
   resolution: "open@npm:10.2.0"
@@ -27048,7 +27084,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"platform@npm:^1.3.3":
+"platform@npm:^1.3.3, platform@npm:^1.3.6":
   version: 1.3.6
   resolution: "platform@npm:1.3.6"
   checksum: 10c0/69f2eb692e15f1a343dd0d9347babd9ca933824c8673096be746ff66f99f2bdc909fadd8609076132e6ec768349080babb7362299f2a7f885b98f1254ae6224b