Merge pull request #34816 from overleaf/revert-34643-em-library-backend-search

Revert "Library search: make backend search tokens consistent with frontend (follow-up to #34386)"

GitOrigin-RevId: 83328f2ee4a9c3f668325fa4de1ca652e7c718bf

services/web/.storybook/main.ts

@@ -121,7 +121,6 @@ export default defineMain({
           ...storybookConfig.resolve?.alias,
           // custom prefixes for import paths
           '@': path.join(rootDir, 'frontend/js/'),
-          '@modules': path.join(rootDir, 'modules/'),
           '@ol-types': path.join(rootDir, 'types/'),
           '@ol-storybook': path.join(rootDir, '.storybook/'),
           '@wf': path.join(

services/web/scripts/backfill_library_references_search.mjs

@@ -2,17 +2,13 @@
 import minimist from 'minimist'
 import logger from '@overleaf/logger'
 import { db } from '../app/src/infrastructure/mongodb.mjs'
-import {
+import { buildSearchFields } from '../modules/library/app/src/LibraryReferenceRepository.mts'
-  buildSearchTokens,
-  docSchema,
-} from '../modules/library/app/src/LibraryReferenceRepository.mts'
-import { tokenize } from '../modules/library/app/src/bibtex-search-tokens.mts'
 import { scriptRunner } from './lib/ScriptRunner.mjs'
 
 /** @typedef {import('mongodb').AnyBulkWriteOperation} AnyBulkWriteOperation */
 
 const argv = minimist(process.argv.slice(2), {
-  boolean: ['commit', 'all', 'help'],
+  boolean: ['commit', 'rollback', 'all', 'help'],
   default: { 'batch-size': 1000 },
 })
 
@@ -21,15 +17,16 @@ function usage() {
     {},
     `Usage: node backfill_library_references_search.mjs [options]
 
-Populates searchKey and searchTokens on libraryReferences so the
+Populates searchKey and fields.searchValue on libraryReferences so the
-account-level library search can index them. Also unsets the obsolete
+account-level library search can index them. Safe to rerun; picks up only
-fields.$[].searchValue. Safe to rerun; picks up only un-indexed rows
+un-indexed rows by default.
-by default.
 
 Options:
   --commit          Apply changes. Without this, runs as a dry run.
-  --all             Re-index every row, not just rows where searchTokens
+  --rollback        Unset searchKey and fields.searchValue on all rows that
-                    is null. Use when the tokenization format has changed.
+                    have them. Mirrors the original migration's rollback.
+  --all             Re-index every row, not just rows where searchKey is null.
+                    Use when the tokenization format has changed.
   --batch-size <n>  bulkWrite batch size (default 1000).
 `
   )
@@ -44,10 +41,11 @@ const BATCH_SIZE = Number(argv['batch-size'])
 
 /** @param {(message: string) => Promise<void>} trackProgress */
 async function backfill(trackProgress) {
-  const filter = argv.all ? {} : { searchTokens: null }
+  const filter = argv.all ? {} : { searchKey: null }
   const cursor = db.libraryReferences
     .find(filter)
-    .project({ key: 1, type: 1, fields: 1, updatedAt: 1 })
+    .hint({ userId: 1, searchKey: 1 })
+    .project({ key: 1, fields: 1 })
 
   let processed = 0
   /** @type {AnyBulkWriteOperation[]} */
@@ -66,20 +64,19 @@ async function backfill(trackProgress) {
   }
 
   for await (const doc of cursor) {
-    const entry = docSchema.parse({
+    const { searchKey, fields } = buildSearchFields({
-      ...doc,
+      key: doc.key,
-      type: doc.type ?? 'misc',
+      fields: (doc.fields ?? []).map(
-      updatedAt: doc.updatedAt ?? new Date(0),
+        (/** @type {{ name: string; editableValue?: string }} */ f) => ({
+          name: f.name,
+          editableValue: f.editableValue ?? '',
+        })
+      ),
     })
-    const searchKey = tokenize(doc.key)
-    const searchTokens = buildSearchTokens(entry)
     ops.push({
       updateOne: {
         filter: { _id: doc._id },
-        update: {
+        update: { $set: { searchKey, fields } },
-          $set: { searchKey, searchTokens },
-          $unset: { 'fields.$[].searchValue': 1 },
-        },
       },
     })
     if (ops.length >= BATCH_SIZE) {
@@ -90,17 +87,39 @@ async function backfill(trackProgress) {
   await trackProgress(`done; processed ${processed} docs`)
 }
 
+/** @param {(message: string) => Promise<void>} trackProgress */
+async function rollback(trackProgress) {
+  if (!argv.commit) {
+    const count = await db.libraryReferences.countDocuments({
+      searchKey: { $ne: null },
+    })
+    await trackProgress(`[dry-run] would unset search fields on ${count} docs`)
+    return
+  }
+  const result = await db.libraryReferences.updateMany(
+    { searchKey: { $ne: null } },
+    { $unset: { searchKey: 1, 'fields.$[].searchValue': 1 } },
+    { hint: { userId: 1, searchKey: 1 } }
+  )
+  await trackProgress(`unset search fields on ${result.modifiedCount} docs`)
+}
+
 /** @param {(message: string) => Promise<void>} trackProgress */
 async function main(trackProgress) {
   if (!argv.commit) {
     await trackProgress('DRY RUN. Pass --commit to apply changes.')
   }
-  await backfill(trackProgress)
+  if (argv.rollback) {
+    await rollback(trackProgress)
+  } else {
+    await backfill(trackProgress)
+  }
 }
 
 try {
   await scriptRunner(main, {
     commit: Boolean(argv.commit),
+    rollback: Boolean(argv.rollback),
     all: Boolean(argv.all),
     batchSize: BATCH_SIZE,
   })

services/web/tsconfig.backend.json

@@ -6,7 +6,6 @@
   "include": [
     "app/src/**/*",
     "modules/*/app/src/**/*",
-    "modules/*/shared/**/*",
     "modules/*/test/acceptance/**/*",
     "modules/*/test/unit/**/*",
     "scripts/**/*",

services/web/tsconfig.json

@@ -14,7 +14,6 @@
     "forceConsistentCasingInFileNames": true,
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
-    "rewriteRelativeImportExtensions": true,
     "baseUrl": ".",
     "paths": {
       "@/*": ["./frontend/js/*"],
@@ -39,7 +38,6 @@
   "include": [
     "frontend/js/**/*.*",
     "modules/**/frontend/js/**/*.*",
-    "modules/*/shared/**/*.*",
     "test/frontend/**/*.*",
     "modules/**/test/frontend/**/*.*",
     "frontend/stories/**/*.*",

tools/migrations/20260612120000_replace_libraryReferences_searchTokens_index.mjs

@@ -1,26 +0,0 @@
-import Helpers from './lib/helpers.mjs'
-
-const tags = ['saas']
-
-const newIndexes = [
-  {
-    key: { userId: 1, searchTokens: 1 },
-    name: 'userId_1_searchTokens_1',
-  },
-]
-
-const migrate = async client => {
-  const { db } = client
-  await Helpers.addIndexesToCollection(db.libraryReferences, newIndexes)
-}
-
-const rollback = async client => {
-  const { db } = client
-  await Helpers.dropIndexesFromCollection(db.libraryReferences, newIndexes)
-}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}