From bc9c85019f52bc2ba438b9e995e823f290533bce Mon Sep 17 00:00:00 2001 From: Liam O'Brien Date: Thu, 18 Jun 2026 14:17:10 +0100 Subject: [PATCH] Lo zotero initial library sync (#34456) * Migration and index setup script * Helper script to clear linked library * Pull users collections * Implementing get items from users zotero library * Initial zotero sync client * Pulling users overleaf collection * Pulling users overleaf collection * Initial sync pull from Zotero to Library * Sync and import workflow from Zotero into Library * Using user getter and setter for updating user library info * Using new collection for library sync states * Adding tests for new sync functionality * Harden Zotero sync route * Simplify Zotero sync state read and skip accounting Fold the ensure+get double round-trip in sync-state discovery into one findOneAndUpdate, and derive itemsSkipped from items.length - itemsInserted instead of a running counter. Co-Authored-By: Claude Opus 4.8 * Redact URL-encoded token secret from OAuth1 request logs Co-Authored-By: Claude Opus 4.8 * fixing type failure * Rename upsertZoteroReferences to importZoteroReferences The method is insert-only; the bulkWrite upsert just makes re-runs idempotent rather than updating, so the name now reflects that. Co-Authored-By: Claude Opus 4.8 * Validate librarySyncStates reads with zod Replace the `as` / `as unknown as` casts in getLibrarySyncState and ensureLibrarySyncState with a zod schema parse, matching LibraryReferenceRepository. Types are now derived from the schema. Co-Authored-By: Claude Opus 4.8 * Defer Zotero sync watermark until pagination exists Don't record lastSyncedVersion after a single-page sync: committing the page-1 version before pagination (#34286) lands would let incremental sync (#34282) skip items beyond the first page. Drops the commitSyncState call and toVersion from SyncResult; the version envelope is left for the follow-ups. commitSyncState stays in the repository for #34282. Co-Authored-By: Claude Opus 4.8 * Extract OAuth1Transport from OAuth1Handler Lift the OAuth1 client/config registry, getAccessToken, redactSecret and classifyOAuth1Error into oauth1/OAuth1Transport.js, plus a signedGet helper for provider modules to reuse. OAuth1Handler now imports them; its method bodies are unchanged. Test mocks move to esmock's global scope so they reach the transport, with Errors pinned for instanceof checks. Co-Authored-By: Claude Opus 4.8 * Move Zotero collections/items into a per-provider ZoteroProvider Carve the Zotero-only operations out of the generic OAuth1 handler into providers/ZoteroProvider.js, dispatched via providers/ProviderRegistry.js. ReferenceProviderHandler now resolves the provider from the registry and throws UnsupportedOperationError (mapped to 501) for providers without the operation, instead of bolting Zotero-only methods onto OAuth1Handler. Zotero's collectionsUri/itemsTopUri are read from settings by ZoteroProvider rather than copied into the shared OAuth1 registry. Removes getCollections/ getUsersItemsChanges (and their tests) from OAuth1Handler. Co-Authored-By: Claude Opus 4.8 * Add ZoteroProvider unit tests and unsupported-provider coverage Cover ZoteroProvider collections/items parsing, header envelope and malformed-payload rejection, and assert ReferenceProviderHandler throws UnsupportedOperationError for providers without a sync implementation. Co-Authored-By: Claude Opus 4.8 * Make addSyncedCollection idempotent Only push a synced collection when its key isn't already recorded, so a repeat sync can't create duplicate syncedCollections entries regardless of the caller-side guard. Adds a sequential test asserting no duplicate. Co-Authored-By: Claude Opus 4.8 * Hardcode Zotero library endpoint paths, derive host from site Move the collections/items-top endpoints out of settings and into ZoteroProvider as path constants, taking only the host from the provider's `site`. Aligns Zotero `site` with the other providers (API host) and avoids adding per-endpoint config points. Co-Authored-By: Claude Opus 4.8 (1M context) * Make Zotero sync routes explicit, 404 unknown providers Hardcode the collections/items routes to /zotero rather than a generic /:ref_provider, so a non-Zotero request 404s on the missing route instead of returning a misleading 501. Drop the UnsupportedOperationError 501 branch, reverting OAuth2 unsupported ops to their prior 500. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 GitOrigin-RevId: 50e16a9c46f995c2e9db6eed50ef81c3a71ea71d --- .../web/app/src/infrastructure/mongodb.mjs | 1 + .../zotero/clear_synced_references.mjs | 101 ++++++++++++++++++ ...74200_zotero_libraryReferences_sources.mjs | 42 ++++++++ ...00_create_librarySyncStates_collection.mjs | 28 +++++ 4 files changed, 172 insertions(+) create mode 100644 services/web/scripts/zotero/clear_synced_references.mjs create mode 100644 tools/migrations/20260604174200_zotero_libraryReferences_sources.mjs create mode 100644 tools/migrations/20260609120000_create_librarySyncStates_collection.mjs diff --git a/services/web/app/src/infrastructure/mongodb.mjs b/services/web/app/src/infrastructure/mongodb.mjs index d61f45f5f0..1032f0da83 100644 --- a/services/web/app/src/infrastructure/mongodb.mjs +++ b/services/web/app/src/infrastructure/mongodb.mjs @@ -54,6 +54,7 @@ export const db = { groupAuditLogEntries: internalDb.collection('groupAuditLogEntries'), institutions: internalDb.collection('institutions'), libraryReferences: internalDb.collection('libraryReferences'), + librarySyncStates: internalDb.collection('librarySyncStates'), messages: internalDb.collection('messages'), migrations: internalDb.collection('migrations'), notifications: internalDb.collection('notifications'), diff --git a/services/web/scripts/zotero/clear_synced_references.mjs b/services/web/scripts/zotero/clear_synced_references.mjs new file mode 100644 index 0000000000..250f55a124 --- /dev/null +++ b/services/web/scripts/zotero/clear_synced_references.mjs @@ -0,0 +1,101 @@ +// @ts-check +import minimist from 'minimist' +import logger from '@overleaf/logger' +import { db, ObjectId } from '../../app/src/infrastructure/mongodb.mjs' +import { scriptRunner } from '../lib/ScriptRunner.mjs' + +// Only zotero is wired up today; the flag exists so the script generalises to +// other reference providers without a rewrite. +const SUPPORTED_SOURCES = ['zotero'] + +const argv = minimist(process.argv.slice(2), { + boolean: ['commit', 'keep-sync-state', 'help'], + string: ['user-id', 'source'], + default: { source: 'zotero' }, +}) + +function usage() { + logger.info( + {}, + `Usage: node scripts/zotero/clear_synced_references.mjs --user-id [options] + +Resets a user's reference-sync state for local debugging. Deletes every +libraryReferences entry linked to the given source (sources. present) +and removes the user's librarySyncStates rows for that provider. + +Options: + --user-id Required. The Overleaf user _id to reset. + --source Reference source to clear (default: zotero). + Supported: ${SUPPORTED_SOURCES.join(', ')}. + --commit Apply changes. Without this, runs as a dry run. + --keep-sync-state Only delete the references; leave the + librarySyncStates rows untouched. +` + ) +} + +if (argv.help || !argv['user-id']) { + usage() + process.exit(argv.help ? 0 : 1) +} + +const source = argv.source +if (!SUPPORTED_SOURCES.includes(source)) { + logger.error( + { source }, + `unsupported source; expected one of ${SUPPORTED_SOURCES.join(', ')}` + ) + process.exit(1) +} + +const userId = new ObjectId(argv['user-id']) + +/** @param {(message: string) => Promise} trackProgress */ +async function main(trackProgress) { + if (!argv.commit) { + await trackProgress('DRY RUN. Pass --commit to apply changes.') + } + + const refFilter = { userId, [`sources.${source}`]: { $exists: true } } + const refCount = await db.libraryReferences.countDocuments(refFilter) + if (argv.commit) { + const result = await db.libraryReferences.deleteMany(refFilter) + await trackProgress( + `deleted ${result.deletedCount} ${source}-linked references` + ) + } else { + await trackProgress( + `[dry-run] would delete ${refCount} ${source}-linked references` + ) + } + + if (argv['keep-sync-state']) { + return + } + + const stateFilter = { userId, provider: source } + if (argv.commit) { + const result = await db.librarySyncStates.deleteMany(stateFilter) + await trackProgress( + `deleted ${result.deletedCount} librarySyncStates row(s)` + ) + } else { + const stateCount = await db.librarySyncStates.countDocuments(stateFilter) + await trackProgress( + `[dry-run] would delete ${stateCount} librarySyncStates row(s)` + ) + } +} + +try { + await scriptRunner(main, { + userId: argv['user-id'], + source, + commit: Boolean(argv.commit), + keepSyncState: Boolean(argv['keep-sync-state']), + }) + process.exit(0) +} catch (err) { + logger.error({ err }, 'clear_synced_references failed') + process.exit(1) +} diff --git a/tools/migrations/20260604174200_zotero_libraryReferences_sources.mjs b/tools/migrations/20260604174200_zotero_libraryReferences_sources.mjs new file mode 100644 index 0000000000..27577ac228 --- /dev/null +++ b/tools/migrations/20260604174200_zotero_libraryReferences_sources.mjs @@ -0,0 +1,42 @@ +import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import Helpers from './lib/helpers.mjs' + +const tags = ['saas'] + +const indexes = [ + { + key: { + userId: 1, + 'sources.zotero.libraryType': 1, + 'sources.zotero.libraryId': 1, + 'sources.zotero.itemKey': 1, + }, + name: 'userId_1_sources.zotero.libraryType_1_sources.zotero.libraryId_1_sources.zotero.itemKey_1', + unique: true, + // Not `sparse`: userId is always present, so a sparse compound index would + // include every doc and collide native entries on (userId, null, null, + // null). Scope uniqueness to Zotero-linked docs only. + partialFilterExpression: { 'sources.zotero.itemKey': { $exists: true } }, + }, +] + +const migrate = async client => { + const { db } = client + await batchedUpdate( + db.libraryReferences, + { sources: { $exists: false } }, + { $set: { sources: {} } } + ) + await Helpers.addIndexesToCollection(db.libraryReferences, indexes) +} + +const rollback = async client => { + const { db } = client + await Helpers.dropIndexesFromCollection(db.libraryReferences, indexes) +} + +export default { + tags, + migrate, + rollback, +} diff --git a/tools/migrations/20260609120000_create_librarySyncStates_collection.mjs b/tools/migrations/20260609120000_create_librarySyncStates_collection.mjs new file mode 100644 index 0000000000..8a99cf0ea9 --- /dev/null +++ b/tools/migrations/20260609120000_create_librarySyncStates_collection.mjs @@ -0,0 +1,28 @@ +import Helpers from './lib/helpers.mjs' +import { getCollectionInternal } from './lib/mongodb.mjs' + +const tags = ['saas'] + +const indexes = [ + { + key: { userId: 1, provider: 1, libraryType: 1, libraryId: 1 }, + name: 'userId_1_provider_1_libraryType_1_libraryId_1', + unique: true, + }, +] + +const migrate = async () => { + const librarySyncStates = await getCollectionInternal('librarySyncStates') + await Helpers.addIndexesToCollection(librarySyncStates, indexes) +} + +const rollback = async () => { + const librarySyncStates = await getCollectionInternal('librarySyncStates') + await Helpers.dropIndexesFromCollection(librarySyncStates, indexes) +} + +export default { + tags, + migrate, + rollback, +}