Lo zotero initial library sync (#34456)

* Migration and index setup script

* Helper script to clear linked library

* Pull users collections

* Implementing get items from users zotero library

* Initial zotero sync client

* Pulling users overleaf collection

* Pulling users overleaf collection

* Initial sync pull from Zotero to Library

* Sync and import workflow from Zotero into Library

* Using user getter and setter for updating user library info

* Using new collection for library sync states

* Adding tests for new sync functionality

* Harden Zotero sync route

* Simplify Zotero sync state read and skip accounting

Fold the ensure+get double round-trip in sync-state discovery into one
findOneAndUpdate, and derive itemsSkipped from items.length - itemsInserted
instead of a running counter.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Redact URL-encoded token secret from OAuth1 request logs

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fixing type failure

* Rename upsertZoteroReferences to importZoteroReferences

The method is insert-only; the bulkWrite upsert just makes re-runs
idempotent rather than updating, so the name now reflects that.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Validate librarySyncStates reads with zod

Replace the `as` / `as unknown as` casts in getLibrarySyncState and
ensureLibrarySyncState with a zod schema parse, matching
LibraryReferenceRepository. Types are now derived from the schema.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Defer Zotero sync watermark until pagination exists

Don't record lastSyncedVersion after a single-page sync: committing the
page-1 version before pagination (#34286) lands would let incremental
sync (#34282) skip items beyond the first page. Drops the commitSyncState
call and toVersion from SyncResult; the version envelope is left for the
follow-ups. commitSyncState stays in the repository for #34282.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Extract OAuth1Transport from OAuth1Handler

Lift the OAuth1 client/config registry, getAccessToken, redactSecret and
classifyOAuth1Error into oauth1/OAuth1Transport.js, plus a signedGet
helper for provider modules to reuse. OAuth1Handler now imports them; its
method bodies are unchanged. Test mocks move to esmock's global scope so
they reach the transport, with Errors pinned for instanceof checks.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Move Zotero collections/items into a per-provider ZoteroProvider

Carve the Zotero-only operations out of the generic OAuth1 handler into
providers/ZoteroProvider.js, dispatched via providers/ProviderRegistry.js.
ReferenceProviderHandler now resolves the provider from the registry and
throws UnsupportedOperationError (mapped to 501) for providers without the
operation, instead of bolting Zotero-only methods onto OAuth1Handler.

Zotero's collectionsUri/itemsTopUri are read from settings by ZoteroProvider
rather than copied into the shared OAuth1 registry. Removes getCollections/
getUsersItemsChanges (and their tests) from OAuth1Handler.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Add ZoteroProvider unit tests and unsupported-provider coverage

Cover ZoteroProvider collections/items parsing, header envelope and
malformed-payload rejection, and assert ReferenceProviderHandler throws
UnsupportedOperationError for providers without a sync implementation.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Make addSyncedCollection idempotent

Only push a synced collection when its key isn't already recorded, so a
repeat sync can't create duplicate syncedCollections entries regardless of
the caller-side guard. Adds a sequential test asserting no duplicate.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Hardcode Zotero library endpoint paths, derive host from site

Move the collections/items-top endpoints out of settings and into
ZoteroProvider as path constants, taking only the host from the
provider's `site`. Aligns Zotero `site` with the other providers
(API host) and avoids adding per-endpoint config points.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Make Zotero sync routes explicit, 404 unknown providers

Hardcode the collections/items routes to /zotero rather than a generic
/:ref_provider, so a non-Zotero request 404s on the missing route
instead of returning a misleading 501. Drop the UnsupportedOperationError
501 branch, reverting OAuth2 unsupported ops to their prior 500.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
GitOrigin-RevId: 50e16a9c46f995c2e9db6eed50ef81c3a71ea71d

services/web/app/src/infrastructure/mongodb.mjs

@@ -54,6 +54,7 @@ export const db = {
   groupAuditLogEntries: internalDb.collection('groupAuditLogEntries'),
   institutions: internalDb.collection('institutions'),
   libraryReferences: internalDb.collection('libraryReferences'),
+  librarySyncStates: internalDb.collection('librarySyncStates'),
   messages: internalDb.collection('messages'),
   migrations: internalDb.collection('migrations'),
   notifications: internalDb.collection('notifications'),

services/web/scripts/zotero/clear_synced_references.mjs

@@ -0,0 +1,101 @@
+// @ts-check
+import minimist from 'minimist'
+import logger from '@overleaf/logger'
+import { db, ObjectId } from '../../app/src/infrastructure/mongodb.mjs'
+import { scriptRunner } from '../lib/ScriptRunner.mjs'
+
+// Only zotero is wired up today; the flag exists so the script generalises to
+// other reference providers without a rewrite.
+const SUPPORTED_SOURCES = ['zotero']
+
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['commit', 'keep-sync-state', 'help'],
+  string: ['user-id', 'source'],
+  default: { source: 'zotero' },
+})
+
+function usage() {
+  logger.info(
+    {},
+    `Usage: node scripts/zotero/clear_synced_references.mjs --user-id <id> [options]
+
+Resets a user's reference-sync state for local debugging. Deletes every
+libraryReferences entry linked to the given source (sources.<source> present)
+and removes the user's librarySyncStates rows for that provider.
+
+Options:
+  --user-id <id>     Required. The Overleaf user _id to reset.
+  --source <name>    Reference source to clear (default: zotero).
+                     Supported: ${SUPPORTED_SOURCES.join(', ')}.
+  --commit           Apply changes. Without this, runs as a dry run.
+  --keep-sync-state  Only delete the references; leave the
+                     librarySyncStates rows untouched.
+`
+  )
+}
+
+if (argv.help || !argv['user-id']) {
+  usage()
+  process.exit(argv.help ? 0 : 1)
+}
+
+const source = argv.source
+if (!SUPPORTED_SOURCES.includes(source)) {
+  logger.error(
+    { source },
+    `unsupported source; expected one of ${SUPPORTED_SOURCES.join(', ')}`
+  )
+  process.exit(1)
+}
+
+const userId = new ObjectId(argv['user-id'])
+
+/** @param {(message: string) => Promise<void>} trackProgress */
+async function main(trackProgress) {
+  if (!argv.commit) {
+    await trackProgress('DRY RUN. Pass --commit to apply changes.')
+  }
+
+  const refFilter = { userId, [`sources.${source}`]: { $exists: true } }
+  const refCount = await db.libraryReferences.countDocuments(refFilter)
+  if (argv.commit) {
+    const result = await db.libraryReferences.deleteMany(refFilter)
+    await trackProgress(
+      `deleted ${result.deletedCount} ${source}-linked references`
+    )
+  } else {
+    await trackProgress(
+      `[dry-run] would delete ${refCount} ${source}-linked references`
+    )
+  }
+
+  if (argv['keep-sync-state']) {
+    return
+  }
+
+  const stateFilter = { userId, provider: source }
+  if (argv.commit) {
+    const result = await db.librarySyncStates.deleteMany(stateFilter)
+    await trackProgress(
+      `deleted ${result.deletedCount} librarySyncStates row(s)`
+    )
+  } else {
+    const stateCount = await db.librarySyncStates.countDocuments(stateFilter)
+    await trackProgress(
+      `[dry-run] would delete ${stateCount} librarySyncStates row(s)`
+    )
+  }
+}
+
+try {
+  await scriptRunner(main, {
+    userId: argv['user-id'],
+    source,
+    commit: Boolean(argv.commit),
+    keepSyncState: Boolean(argv['keep-sync-state']),
+  })
+  process.exit(0)
+} catch (err) {
+  logger.error({ err }, 'clear_synced_references failed')
+  process.exit(1)
+}

tools/migrations/20260604174200_zotero_libraryReferences_sources.mjs

@@ -0,0 +1,42 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      userId: 1,
+      'sources.zotero.libraryType': 1,
+      'sources.zotero.libraryId': 1,
+      'sources.zotero.itemKey': 1,
+    },
+    name: 'userId_1_sources.zotero.libraryType_1_sources.zotero.libraryId_1_sources.zotero.itemKey_1',
+    unique: true,
+    // Not `sparse`: userId is always present, so a sparse compound index would
+    // include every doc and collide native entries on (userId, null, null,
+    // null). Scope uniqueness to Zotero-linked docs only.
+    partialFilterExpression: { 'sources.zotero.itemKey': { $exists: true } },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.libraryReferences,
+    { sources: { $exists: false } },
+    { $set: { sources: {} } }
+  )
+  await Helpers.addIndexesToCollection(db.libraryReferences, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.libraryReferences, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20260609120000_create_librarySyncStates_collection.mjs

@@ -0,0 +1,28 @@
+import Helpers from './lib/helpers.mjs'
+import { getCollectionInternal } from './lib/mongodb.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: { userId: 1, provider: 1, libraryType: 1, libraryId: 1 },
+    name: 'userId_1_provider_1_libraryType_1_libraryId_1',
+    unique: true,
+  },
+]
+
+const migrate = async () => {
+  const librarySyncStates = await getCollectionInternal('librarySyncStates')
+  await Helpers.addIndexesToCollection(librarySyncStates, indexes)
+}
+
+const rollback = async () => {
+  const librarySyncStates = await getCollectionInternal('librarySyncStates')
+  await Helpers.dropIndexesFromCollection(librarySyncStates, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}