Merge pull request #34884 from overleaf/em-already-in-your-library

Library import: flag entries that are already in the user's library

GitOrigin-RevId: f3c92fb59283631e1a9e78d2e08a3a365981df45

services/web/frontend/extracted-translations.json

@@ -146,6 +146,7 @@
   "all_these_experiments_are_available_exclusively": "",
   "allocate_license": "",
   "allows_to_search_by_author_title_etc_possible_to_pull_results_directly_from_your_reference_manager_if_connected": "",
+  "already_in_your_library": "",
   "an_email_has_already_been_sent_to": "",
   "an_error_occured_while_restoring_project": "",
   "and_much_more": "",

services/web/locales/en.json

@@ -183,6 +183,7 @@
   "allows_to_search_by_author_title_etc_possible_to_pull_results_directly_from_your_reference_manager_if_connected": "Allows to search by author, title, etc. Possible to pull results directly from your reference manager (if connected).",
   "already_have_an_account": "Already have an account?",
   "already_have_sl_account": "Already have an __appName__ account?",
+  "already_in_your_library": "Already in your library",
   "also": "Also",
   "alternatively_create_new_institution_account": "Alternatively, you can create a <b>new account</b> with your institution email (<b>__email__</b>) by clicking <b>__clickText__</b>.",
   "an_email_has_already_been_sent_to": "An email has already been sent to <0>__email__</0>. Please wait and try again later.",

services/web/scripts/backfill_library_references_search.mjs

@@ -4,6 +4,7 @@ import logger from '@overleaf/logger'
 import { db } from '../app/src/infrastructure/mongodb.mjs'
 import {
   buildSearchTokens,
+  buildMatchTokens,
   docSchema,
 } from '../modules/library/app/src/LibraryReferenceRepository.mts'
 import { tokenize } from '../modules/library/app/src/bibtex-search-tokens.mts'
@@ -21,10 +22,10 @@ function usage() {
     {},
     `Usage: node backfill_library_references_search.mjs [options]
 
-Populates searchKey and searchTokens on libraryReferences so the
+Populates searchKey, searchTokens and matchTokens on libraryReferences so
-account-level library search can index them. Also unsets the obsolete
+the account-level library search and import duplicate-detection can index
-fields.$[].searchValue. Safe to rerun; picks up only un-indexed rows
+them. Also unsets the obsolete fields.$[].searchValue. Safe to rerun; picks
-by default.
+up only un-indexed rows by default.
 
 Options:
   --commit          Apply changes. Without this, runs as a dry run.
@@ -73,11 +74,12 @@ async function backfill(trackProgress) {
     })
     const searchKey = tokenize(doc.key)
     const searchTokens = buildSearchTokens(entry)
+    const matchTokens = buildMatchTokens(entry)
     ops.push({
       updateOne: {
         filter: { _id: doc._id },
         update: {
-          $set: { searchKey, searchTokens },
+          $set: { searchKey, searchTokens, matchTokens },
           $unset: { 'fields.$[].searchValue': 1 },
         },
       },

tools/migrations/20260619120000_create_libraryReferences_match_indexes.mjs

@@ -0,0 +1,36 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+// Partial indexes so docs without a given match token are not indexed. They
+// back the batch duplicate-detection query in findMatchingReferences.
+const newIndexes = [
+  {
+    key: { userId: 1, 'matchTokens.doi': 1 },
+    name: 'userId_1_matchTokens.doi_1',
+    partialFilterExpression: { 'matchTokens.doi': { $exists: true } },
+  },
+  {
+    key: { userId: 1, 'matchTokens.authorTitleYear': 1 },
+    name: 'userId_1_matchTokens.authorTitleYear_1',
+    partialFilterExpression: {
+      'matchTokens.authorTitleYear': { $exists: true },
+    },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.libraryReferences, newIndexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.libraryReferences, newIndexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}