Lo zotero initial library sync (#34456)

* Migration and index setup script * Helper script to clear linked library * Pull users collections * Implementing get items from users zotero library * Initial zotero sync client * Pulling users overleaf collection * Pulling users overleaf collection * Initial sync pull from Zotero to Library * Sync and import workflow from Zotero into Library * Using user getter and setter for updating user library info * Using new collection for library sync states * Adding tests for new sync functionality * Harden Zotero sync route * Simplify Zotero sync state read and skip accounting Fold the ensure+get double round-trip in sync-state discovery into one findOneAndUpdate, and derive itemsSkipped from items.length - itemsInserted instead of a running counter. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Redact URL-encoded token secret from OAuth1 request logs Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fixing type failure * Rename upsertZoteroReferences to importZoteroReferences The method is insert-only; the bulkWrite upsert just makes re-runs idempotent rather than updating, so the name now reflects that. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Validate librarySyncStates reads with zod Replace the `as` / `as unknown as` casts in getLibrarySyncState and ensureLibrarySyncState with a zod schema parse, matching LibraryReferenceRepository. Types are now derived from the schema. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Defer Zotero sync watermark until pagination exists Don't record lastSyncedVersion after a single-page sync: committing the page-1 version before pagination (#34286) lands would let incremental sync (#34282) skip items beyond the first page. Drops the commitSyncState call and toVersion from SyncResult; the version envelope is left for the follow-ups. commitSyncState stays in the repository for #34282. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Extract OAuth1Transport from OAuth1Handler Lift the OAuth1 client/config registry, getAccessToken, redactSecret and classifyOAuth1Error into oauth1/OAuth1Transport.js, plus a signedGet helper for provider modules to reuse. OAuth1Handler now imports them; its method bodies are unchanged. Test mocks move to esmock's global scope so they reach the transport, with Errors pinned for instanceof checks. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Move Zotero collections/items into a per-provider ZoteroProvider Carve the Zotero-only operations out of the generic OAuth1 handler into providers/ZoteroProvider.js, dispatched via providers/ProviderRegistry.js. ReferenceProviderHandler now resolves the provider from the registry and throws UnsupportedOperationError (mapped to 501) for providers without the operation, instead of bolting Zotero-only methods onto OAuth1Handler. Zotero's collectionsUri/itemsTopUri are read from settings by ZoteroProvider rather than copied into the shared OAuth1 registry. Removes getCollections/ getUsersItemsChanges (and their tests) from OAuth1Handler. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Add ZoteroProvider unit tests and unsupported-provider coverage Cover ZoteroProvider collections/items parsing, header envelope and malformed-payload rejection, and assert ReferenceProviderHandler throws UnsupportedOperationError for providers without a sync implementation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Make addSyncedCollection idempotent Only push a synced collection when its key isn't already recorded, so a repeat sync can't create duplicate syncedCollections entries regardless of the caller-side guard. Adds a sequential test asserting no duplicate. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Hardcode Zotero library endpoint paths, derive host from site Move the collections/items-top endpoints out of settings and into ZoteroProvider as path constants, taking only the host from the provider's `site`. Aligns Zotero `site` with the other providers (API host) and avoids adding per-endpoint config points. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Make Zotero sync routes explicit, 404 unknown providers Hardcode the collections/items routes to /zotero rather than a generic /:ref_provider, so a non-Zotero request 404s on the missing route instead of returning a misleading 501. Drop the UnsupportedOperationError 501 branch, reverting OAuth2 unsupported ops to their prior 500. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> GitOrigin-RevId: 50e16a9c46f995c2e9db6eed50ef81c3a71ea71d
2026-11-12 13:13:45 +01:00 · 2026-06-19 08:06:07 +00:00
parent e5f712a857
commit bc9c85019f
4 changed files with 172 additions and 0 deletions
@@ -54,6 +54,7 @@ export const db = {
  groupAuditLogEntries: internalDb.collection('groupAuditLogEntries'),
  institutions: internalDb.collection('institutions'),
  libraryReferences: internalDb.collection('libraryReferences'),
+  librarySyncStates: internalDb.collection('librarySyncStates'),
  messages: internalDb.collection('messages'),
  migrations: internalDb.collection('migrations'),
  notifications: internalDb.collection('notifications'),
@@ -0,0 +1,101 @@
+// @ts-check
+import minimist from 'minimist'
+import logger from '@overleaf/logger'
+import { db, ObjectId } from '../../app/src/infrastructure/mongodb.mjs'
+import { scriptRunner } from '../lib/ScriptRunner.mjs'
+
+// Only zotero is wired up today; the flag exists so the script generalises to
+// other reference providers without a rewrite.
+const SUPPORTED_SOURCES = ['zotero']
+
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['commit', 'keep-sync-state', 'help'],
+  string: ['user-id', 'source'],
+  default: { source: 'zotero' },
+})
+
+function usage() {
+  logger.info(
+    {},
+    `Usage: node scripts/zotero/clear_synced_references.mjs --user-id <id> [options]
+
+Resets a user's reference-sync state for local debugging. Deletes every
+libraryReferences entry linked to the given source (sources.<source> present)
+and removes the user's librarySyncStates rows for that provider.
+
+Options:
+  --user-id <id>     Required. The Overleaf user _id to reset.
+  --source <name>    Reference source to clear (default: zotero).
+                     Supported: ${SUPPORTED_SOURCES.join(', ')}.
+  --commit           Apply changes. Without this, runs as a dry run.
+  --keep-sync-state  Only delete the references; leave the
+                     librarySyncStates rows untouched.
+`
+  )
+}
+
+if (argv.help || !argv['user-id']) {
+  usage()
+  process.exit(argv.help ? 0 : 1)
+}
+
+const source = argv.source
+if (!SUPPORTED_SOURCES.includes(source)) {
+  logger.error(
+    { source },
+    `unsupported source; expected one of ${SUPPORTED_SOURCES.join(', ')}`
+  )
+  process.exit(1)
+}
+
+const userId = new ObjectId(argv['user-id'])
+
+/** @param {(message: string) => Promise<void>} trackProgress */
+async function main(trackProgress) {
+  if (!argv.commit) {
+    await trackProgress('DRY RUN. Pass --commit to apply changes.')
+  }
+
+  const refFilter = { userId, [`sources.${source}`]: { $exists: true } }
+  const refCount = await db.libraryReferences.countDocuments(refFilter)
+  if (argv.commit) {
+    const result = await db.libraryReferences.deleteMany(refFilter)
+    await trackProgress(
+      `deleted ${result.deletedCount} ${source}-linked references`
+    )
+  } else {
+    await trackProgress(
+      `[dry-run] would delete ${refCount} ${source}-linked references`
+    )
+  }
+
+  if (argv['keep-sync-state']) {
+    return
+  }
+
+  const stateFilter = { userId, provider: source }
+  if (argv.commit) {
+    const result = await db.librarySyncStates.deleteMany(stateFilter)
+    await trackProgress(
+      `deleted ${result.deletedCount} librarySyncStates row(s)`
+    )
+  } else {
+    const stateCount = await db.librarySyncStates.countDocuments(stateFilter)
+    await trackProgress(
+      `[dry-run] would delete ${stateCount} librarySyncStates row(s)`
+    )
+  }
+}
+
+try {
+  await scriptRunner(main, {
+    userId: argv['user-id'],
+    source,
+    commit: Boolean(argv.commit),
+    keepSyncState: Boolean(argv['keep-sync-state']),
+  })
+  process.exit(0)
+} catch (err) {
+  logger.error({ err }, 'clear_synced_references failed')
+  process.exit(1)
+}
@@ -0,0 +1,42 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      userId: 1,
+      'sources.zotero.libraryType': 1,
+      'sources.zotero.libraryId': 1,
+      'sources.zotero.itemKey': 1,
+    },
+    name: 'userId_1_sources.zotero.libraryType_1_sources.zotero.libraryId_1_sources.zotero.itemKey_1',
+    unique: true,
+    // Not `sparse`: userId is always present, so a sparse compound index would
+    // include every doc and collide native entries on (userId, null, null,
+    // null). Scope uniqueness to Zotero-linked docs only.
+    partialFilterExpression: { 'sources.zotero.itemKey': { $exists: true } },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.libraryReferences,
+    { sources: { $exists: false } },
+    { $set: { sources: {} } }
+  )
+  await Helpers.addIndexesToCollection(db.libraryReferences, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.libraryReferences, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}
@@ -0,0 +1,28 @@
+import Helpers from './lib/helpers.mjs'
+import { getCollectionInternal } from './lib/mongodb.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: { userId: 1, provider: 1, libraryType: 1, libraryId: 1 },
+    name: 'userId_1_provider_1_libraryType_1_libraryId_1',
+    unique: true,
+  },
+]
+
+const migrate = async () => {
+  const librarySyncStates = await getCollectionInternal('librarySyncStates')
+  await Helpers.addIndexesToCollection(librarySyncStates, indexes)
+}
+
+const rollback = async () => {
+  const librarySyncStates = await getCollectionInternal('librarySyncStates')
+  await Helpers.dropIndexesFromCollection(librarySyncStates, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}