Merge pull request #35377 from overleaf/em-change-library-mongo-cluster

Add auxiliary Mongo cluster support for library collections GitOrigin-RevId: 1cfdbe4bbeb6c95f1158834ae60a4e34bef7bb87
2026-11-12 13:13:45 +01:00 · 2026-09-17 11:26:00 +00:00
parent 1acb00407e
commit 528e587038
15 changed files with 405 additions and 28 deletions
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const indexes = [
  {
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const oldIndex = {
  key: { userId: 1, updatedAt: 1 },
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const indexes = [
  {
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const oldIndexes = [
  {
@@ -1,7 +1,7 @@
 import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const indexes = [
  {
@@ -1,7 +1,6 @@
 import Helpers from './lib/helpers.mjs'
-import { getCollectionInternal } from './lib/mongodb.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const indexes = [
  {
@@ -11,14 +10,14 @@ const indexes = [
  },
 ]

-const migrate = async () => {
-  const librarySyncStates = await getCollectionInternal('librarySyncStates')
-  await Helpers.addIndexesToCollection(librarySyncStates, indexes)
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.librarySyncStates, indexes)
 }

-const rollback = async () => {
-  const librarySyncStates = await getCollectionInternal('librarySyncStates')
-  await Helpers.dropIndexesFromCollection(librarySyncStates, indexes)
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.librarySyncStates, indexes)
 }

 export default {
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const newIndexes = [
  {
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 // Partial indexes so docs without a given match token are not indexed. They
 // back the batch duplicate-detection query in findMatchingReferences.
@@ -1,6 +1,6 @@
 import Helpers from './lib/helpers.mjs'

-const tags = ['saas']
+const tags = ['saas', 'auxiliary']

 const oldIndexes = [
  {
@@ -16,6 +16,7 @@ module.exports = {
      process.env.MONGO_CONNECTION_STRING ||
      process.env.MONGO_URL ||
      `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`,
+    auxUrl: process.env.MONGO_AUX_CONNECTION_STRING,
    hasSecondaries: process.env.MONGO_HAS_SECONDARIES === 'true',
  },
 }
@@ -1,10 +1,19 @@
 import Path from 'node:path'
-import { db } from './mongodb.mjs'
+import { db, auxInternalDb } from './mongodb.mjs'
 import { fileURLToPath } from 'node:url'

 const __filename = fileURLToPath(import.meta.url)
 const __dirname = Path.dirname(__filename)

+// Migrations tagged 'auxiliary' track their execution history on the
+// auxiliary Mongo cluster (when one is configured) instead of the primary
+// one, so a migration already marked executed against the primary cluster
+// still runs once the auxiliary cluster is provisioned. When no auxiliary
+// cluster is configured, everything falls back to the primary cluster.
+const auxMigrations = auxInternalDb
+  ? auxInternalDb.collection('migrations')
+  : null
+
 class Adapter {
  constructor(params) {
    if (
@@ -16,6 +25,7 @@ class Adapter {
      process.exit(1)
    }
    this.params = params || {}
+    this.migrationTagsByName = new Map()
  }

  getTemplatePath() {
@@ -30,22 +40,75 @@ class Adapter {
    return Promise.resolve()
  }

+  async _getMigrationTags(name) {
+    if (!this.migrationTagsByName.has(name)) {
+      const { dir, migrationExtension } = this.params
+      const migrationPath = Path.resolve(dir, `${name}.${migrationExtension}`)
+      let tags = []
+      try {
+        const { default: migration } = await import(migrationPath)
+        tags = migration.tags || []
+      } catch (err) {
+        if (err.code !== 'ERR_MODULE_NOT_FOUND') {
+          throw err
+        }
+        // The migration file no longer exists on disk (e.g. it was removed
+        // long after it ran everywhere). We can't know if it was tagged
+        // 'auxiliary', so it stays tracked as a primary-only migration.
+      }
+      this.migrationTagsByName.set(name, tags)
+    }
+    return this.migrationTagsByName.get(name)
+  }
+
+  async _migrationsCollectionFor(name) {
+    if (!auxMigrations) {
+      return db.migrations
+    }
+    const tags = await this._getMigrationTags(name)
+    return tags.includes('auxiliary') ? auxMigrations : db.migrations
+  }
+
  async getExecutedMigrationNames() {
-    const migrations = await db.migrations
-      .find({}, { sort: { migratedAt: 1 }, projection: { name: 1 } })
-      .toArray()
-    return migrations.map(migration => migration.name)
+    const primaryNames = (
+      await db.migrations.find({}, { projection: { name: 1 } }).toArray()
+    ).map(migration => migration.name)
+
+    if (!auxMigrations) {
+      return primaryNames.sort()
+    }
+
+    // Once an auxiliary cluster is configured, an 'auxiliary'-tagged
+    // migration counts as executed only if it ran against the auxiliary
+    // cluster — a stale primary-side record from before the auxiliary
+    // cluster existed is ignored, so the migration runs again on the
+    // auxiliary cluster.
+    const nonAuxPrimaryNames = []
+    for (const name of primaryNames) {
+      const tags = await this._getMigrationTags(name)
+      if (!tags.includes('auxiliary')) {
+        nonAuxPrimaryNames.push(name)
+      }
+    }
+
+    const auxNames = (
+      await auxMigrations.find({}, { projection: { name: 1 } }).toArray()
+    ).map(migration => migration.name)
+
+    return [...nonAuxPrimaryNames, ...auxNames].sort()
  }

  async markExecuted(name) {
-    return await db.migrations.insertOne({
+    const collection = await this._migrationsCollectionFor(name)
+    return await collection.insertOne({
      name,
      migratedAt: new Date(),
    })
  }

  async unmarkExecuted(name) {
-    return await db.migrations.deleteOne({
+    const collection = await this._migrationsCollectionFor(name)
+    return await collection.deleteOne({
      name,
    })
  }
@@ -11,6 +11,18 @@ export const READ_PREFERENCE_SECONDARY = Settings.mongo.hasSecondaries
 const mongoClient = new MongoClient(Settings.mongo.url, Settings.mongo.options)

 const internalDb = mongoClient.db()
+
+const auxMongoClient = Settings.mongo.auxUrl
+  ? new MongoClient(Settings.mongo.auxUrl, Settings.mongo.options)
+  : null
+
+export const auxInternalDb = auxMongoClient ? auxMongoClient.db() : null
+const auxConnectionPromise = auxMongoClient
+  ? auxMongoClient.connect()
+  : Promise.resolve()
+
+const libraryDb = auxInternalDb || internalDb
+
 export const db = {
  contacts: internalDb.collection('contacts'),
  deletedProjects: internalDb.collection('deletedProjects'),
@@ -29,7 +41,9 @@ export const db = {
  grouppolicies: internalDb.collection('grouppolicies'),
  groupAuditLogEntries: internalDb.collection('groupAuditLogEntries'),
  institutions: internalDb.collection('institutions'),
-  libraryReferences: internalDb.collection('libraryReferences'),
+  libraryReferences: libraryDb.collection('libraryReferences'),
+  librarySizes: libraryDb.collection('librarySizes'),
+  librarySyncStates: libraryDb.collection('librarySyncStates'),
  messages: internalDb.collection('messages'),
  migrations: internalDb.collection('migrations'),
  notifications: internalDb.collection('notifications'),
@@ -83,6 +97,7 @@ export async function getCollectionInternal(name) {

 export async function waitForDb() {
  await connectionPromise
+  await auxConnectionPromise
 }

 const mongodb = {