diff --git a/services/web/.storybook/main.ts b/services/web/.storybook/main.ts index fc0672e6e0..462e56cfb8 100644 --- a/services/web/.storybook/main.ts +++ b/services/web/.storybook/main.ts @@ -121,6 +121,7 @@ export default defineMain({ ...storybookConfig.resolve?.alias, // custom prefixes for import paths '@': path.join(rootDir, 'frontend/js/'), + '@modules': path.join(rootDir, 'modules/'), '@ol-types': path.join(rootDir, 'types/'), '@ol-storybook': path.join(rootDir, '.storybook/'), '@wf': path.join( diff --git a/services/web/scripts/backfill_library_references_search.mjs b/services/web/scripts/backfill_library_references_search.mjs index 8733c2163d..b381dae27e 100644 --- a/services/web/scripts/backfill_library_references_search.mjs +++ b/services/web/scripts/backfill_library_references_search.mjs @@ -2,13 +2,17 @@ import minimist from 'minimist' import logger from '@overleaf/logger' import { db } from '../app/src/infrastructure/mongodb.mjs' -import { buildSearchFields } from '../modules/library/app/src/LibraryReferenceRepository.mts' +import { + buildSearchTokens, + docSchema, +} from '../modules/library/app/src/LibraryReferenceRepository.mts' +import { tokenize } from '../modules/library/app/src/bibtex-search-tokens.mts' import { scriptRunner } from './lib/ScriptRunner.mjs' /** @typedef {import('mongodb').AnyBulkWriteOperation} AnyBulkWriteOperation */ const argv = minimist(process.argv.slice(2), { - boolean: ['commit', 'rollback', 'all', 'help'], + boolean: ['commit', 'all', 'help'], default: { 'batch-size': 1000 }, }) @@ -17,16 +21,15 @@ function usage() { {}, `Usage: node backfill_library_references_search.mjs [options] -Populates searchKey and fields.searchValue on libraryReferences so the -account-level library search can index them. Safe to rerun; picks up only -un-indexed rows by default. +Populates searchKey and searchTokens on libraryReferences so the +account-level library search can index them. Also unsets the obsolete +fields.$[].searchValue. Safe to rerun; picks up only un-indexed rows +by default. Options: --commit Apply changes. Without this, runs as a dry run. - --rollback Unset searchKey and fields.searchValue on all rows that - have them. Mirrors the original migration's rollback. - --all Re-index every row, not just rows where searchKey is null. - Use when the tokenization format has changed. + --all Re-index every row, not just rows where searchTokens + is null. Use when the tokenization format has changed. --batch-size bulkWrite batch size (default 1000). ` ) @@ -41,11 +44,10 @@ const BATCH_SIZE = Number(argv['batch-size']) /** @param {(message: string) => Promise} trackProgress */ async function backfill(trackProgress) { - const filter = argv.all ? {} : { searchKey: null } + const filter = argv.all ? {} : { searchTokens: null } const cursor = db.libraryReferences .find(filter) - .hint({ userId: 1, searchKey: 1 }) - .project({ key: 1, fields: 1 }) + .project({ key: 1, type: 1, fields: 1, updatedAt: 1 }) let processed = 0 /** @type {AnyBulkWriteOperation[]} */ @@ -64,19 +66,20 @@ async function backfill(trackProgress) { } for await (const doc of cursor) { - const { searchKey, fields } = buildSearchFields({ - key: doc.key, - fields: (doc.fields ?? []).map( - (/** @type {{ name: string; editableValue?: string }} */ f) => ({ - name: f.name, - editableValue: f.editableValue ?? '', - }) - ), + const entry = docSchema.parse({ + ...doc, + type: doc.type ?? 'misc', + updatedAt: doc.updatedAt ?? new Date(0), }) + const searchKey = tokenize(doc.key) + const searchTokens = buildSearchTokens(entry) ops.push({ updateOne: { filter: { _id: doc._id }, - update: { $set: { searchKey, fields } }, + update: { + $set: { searchKey, searchTokens }, + $unset: { 'fields.$[].searchValue': 1 }, + }, }, }) if (ops.length >= BATCH_SIZE) { @@ -87,39 +90,17 @@ async function backfill(trackProgress) { await trackProgress(`done; processed ${processed} docs`) } -/** @param {(message: string) => Promise} trackProgress */ -async function rollback(trackProgress) { - if (!argv.commit) { - const count = await db.libraryReferences.countDocuments({ - searchKey: { $ne: null }, - }) - await trackProgress(`[dry-run] would unset search fields on ${count} docs`) - return - } - const result = await db.libraryReferences.updateMany( - { searchKey: { $ne: null } }, - { $unset: { searchKey: 1, 'fields.$[].searchValue': 1 } }, - { hint: { userId: 1, searchKey: 1 } } - ) - await trackProgress(`unset search fields on ${result.modifiedCount} docs`) -} - /** @param {(message: string) => Promise} trackProgress */ async function main(trackProgress) { if (!argv.commit) { await trackProgress('DRY RUN. Pass --commit to apply changes.') } - if (argv.rollback) { - await rollback(trackProgress) - } else { - await backfill(trackProgress) - } + await backfill(trackProgress) } try { await scriptRunner(main, { commit: Boolean(argv.commit), - rollback: Boolean(argv.rollback), all: Boolean(argv.all), batchSize: BATCH_SIZE, }) diff --git a/services/web/tsconfig.backend.json b/services/web/tsconfig.backend.json index 6549536bc8..2207ee3d38 100644 --- a/services/web/tsconfig.backend.json +++ b/services/web/tsconfig.backend.json @@ -6,6 +6,7 @@ "include": [ "app/src/**/*", "modules/*/app/src/**/*", + "modules/*/shared/**/*", "modules/*/test/acceptance/**/*", "modules/*/test/unit/**/*", "scripts/**/*", diff --git a/services/web/tsconfig.json b/services/web/tsconfig.json index 2e5fc9ad56..4fc80342c8 100644 --- a/services/web/tsconfig.json +++ b/services/web/tsconfig.json @@ -14,6 +14,7 @@ "forceConsistentCasingInFileNames": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, + "allowImportingTsExtensions": true, "baseUrl": ".", "paths": { "@/*": ["./frontend/js/*"], @@ -38,6 +39,7 @@ "include": [ "frontend/js/**/*.*", "modules/**/frontend/js/**/*.*", + "modules/*/shared/**/*.*", "test/frontend/**/*.*", "modules/**/test/frontend/**/*.*", "frontend/stories/**/*.*", diff --git a/services/web/webpack.config.js b/services/web/webpack.config.js index 35b4e96cd3..d7eb9a9470 100644 --- a/services/web/webpack.config.js +++ b/services/web/webpack.config.js @@ -142,7 +142,7 @@ module.exports = { { // Pass application JS/TS files through babel-loader, // transpiling to targets defined in browserslist - test: /\.([jt]sx?|[cm]js)$/, + test: /\.([jt]sx?|[cm][jt]s)$/, // Only compile application files and specific dependencies // (other npm and vendored dependencies must be in ES5 already) exclude: [ diff --git a/tools/migrations/20260612120000_replace_libraryReferences_searchTokens_index.mjs b/tools/migrations/20260612120000_replace_libraryReferences_searchTokens_index.mjs new file mode 100644 index 0000000000..c7e99643c9 --- /dev/null +++ b/tools/migrations/20260612120000_replace_libraryReferences_searchTokens_index.mjs @@ -0,0 +1,26 @@ +import Helpers from './lib/helpers.mjs' + +const tags = ['saas'] + +const newIndexes = [ + { + key: { userId: 1, searchTokens: 1 }, + name: 'userId_1_searchTokens_1', + }, +] + +const migrate = async client => { + const { db } = client + await Helpers.addIndexesToCollection(db.libraryReferences, newIndexes) +} + +const rollback = async client => { + const { db } = client + await Helpers.dropIndexesFromCollection(db.libraryReferences, newIndexes) +} + +export default { + tags, + migrate, + rollback, +}