move migrations to shared location (#28306)

* fix: correct typedef for Document in helpers.mjs * add move-migrations codemod * update migration paths to use shared migrations directory * move migrations to shared location * fix: update Dockerfile and docker-compose.ci.yml to include migrations directory * feat: add migrations tool to workspaces in package.json * [monorepo] Fix order of docker ignore rules * [web] remove unused docker ignore file * [monorepo] replace old references to migrations folder * [server-ce] copy migrations from new place * [migrations] Inline web scripts Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] move three web scripts over Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] add missing collection Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] remove lodash dependency Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] avoid mongodb-legacy dependency Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [monorepo] run migrations from tools/migrations Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] simplify migration for adding gitBridge feature to users * [monorepo] run migrations from tests in all the services * [migrations] add Jenkins pipeline for linting/formatting * [monorepo] fixup running web migrations everywhere * [monorepo] trigger Jenkins builds on changes to mongo migrations * [migrations] add Jenkins pipeline for linting/formatting * [monorepo] build scripts: update devDependencies before deps scanning * [monorepo] build scripts: formerly depend on tools/migrations * [monorepo] run eslint on .mjs files * [migrations] enable more eslint rules and fix all the errors * [rake] fix migrations:list task --------- Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com> GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614
2026-11-14 14:13:43 +01:00 · 2025-10-16 08:07:37 +00:00
parent fbea855690
commit 729e0f5ac9
249 changed files with 802 additions and 984 deletions
@@ -0,0 +1,44 @@
+import { db } from '../lib/mongodb.mjs'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S, 10) || 5
+
+async function main() {
+  let processed = 0
+  let deleted = 0
+  let lastLog = 0
+
+  function logProgress() {
+    console.log(`rev missing ${processed} | deleted=true ${deleted}`)
+  }
+
+  await batchedUpdate(
+    db.docs,
+    { rev: { $exists: false } },
+    async docs => {
+      await db.docs.updateMany(
+        {
+          _id: { $in: docs.map(doc => doc._id) },
+          rev: { $exists: false },
+        },
+        { $set: { rev: 1 } }
+      )
+
+      processed += docs.length
+      deleted += docs.filter(doc => doc.deleted).length
+
+      if (Date.now() - lastLog >= LOG_EVERY_IN_S * 1000) {
+        logProgress()
+        lastLog = Date.now()
+      }
+    },
+    {
+      _id: 1,
+      deleted: true,
+    }
+  )
+
+  logProgress()
+}
+
+export default main
@@ -0,0 +1,115 @@
+import mongodb from 'mongodb'
+import { db, READ_PREFERENCE_SECONDARY } from '../lib/mongodb.mjs'
+
+const { ObjectId } = mongodb
+
+const NOW_IN_S = Date.now() / 1000
+const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
+
+const DUMMY_NAME = 'unknown.tex'
+const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
+
+let deletedProjectsCache = null
+
+function getSecondsFromObjectId(id) {
+  return id.getTimestamp().getTime() / 1000
+}
+
+async function main(firstProjectId) {
+  const options = {
+    firstProjectId,
+    incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S,
+    batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000,
+    stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S,
+  }
+
+  deletedProjectsCache = new Map()
+
+  let startId = options.firstProjectId
+
+  let nProcessed = 0
+  while (getSecondsFromObjectId(startId) <= options.stopAtS) {
+    const end = getSecondsFromObjectId(startId) + options.incrementByS
+    let endId = ObjectId.createFromTime(end)
+    const query = {
+      project_id: {
+        // include edge
+        $gte: startId,
+        // exclude edge
+        $lt: endId,
+      },
+      deleted: true,
+      name: {
+        $exists: false,
+      },
+    }
+    const docs = await db.docs
+      .find(query, { readPreference: READ_PREFERENCE_SECONDARY })
+      .project({ _id: 1, project_id: 1 })
+      .limit(options.batchSize)
+      .toArray()
+
+    if (docs.length) {
+      const docIds = docs.map(doc => doc._id)
+      console.log('Back filling dummy meta data for', JSON.stringify(docIds))
+      await processBatch(docs, options)
+      nProcessed += docIds.length
+
+      if (docs.length === options.batchSize) {
+        endId = docs[docs.length - 1].project_id
+      }
+    }
+    console.error('Processed %d until %s', nProcessed, endId)
+
+    startId = endId
+  }
+}
+
+async function getDeletedProject(projectId) {
+  const cacheKey = projectId.toString()
+  if (deletedProjectsCache.has(cacheKey)) {
+    return deletedProjectsCache.get(cacheKey)
+  }
+  const deletedProject = await db.deletedProjects.findOne(
+    { 'deleterData.deletedProjectId': projectId },
+    {
+      projection: {
+        _id: 1,
+        'project.deletedDocs': 1,
+      },
+    }
+  )
+  deletedProjectsCache.set(cacheKey, deletedProject)
+  return deletedProject
+}
+
+async function processBatch(docs, options) {
+  for (const doc of docs) {
+    const { _id: docId, project_id: projectId } = doc
+    const deletedProject = await getDeletedProject(projectId)
+    let name = DUMMY_NAME
+    let deletedAt = DUMMY_TIME
+    if (deletedProject) {
+      const project = deletedProject.project
+      if (project) {
+        const deletedDoc =
+          project.deletedDocs &&
+          project.deletedDocs.find(deletedDoc => docId.equals(deletedDoc._id))
+        if (deletedDoc) {
+          console.log('Found deletedDoc for %s', docId)
+          name = deletedDoc.name
+          deletedAt = deletedDoc.deletedAt
+        } else {
+          console.log('Missing deletedDoc for %s', docId)
+        }
+      } else {
+        console.log('Orphaned deleted doc %s (failed hard deletion)', docId)
+      }
+    } else {
+      console.log('Orphaned deleted doc %s (no deletedProjects entry)', docId)
+    }
+    await db.docs.updateOne({ _id: docId }, { $set: { name, deletedAt } })
+  }
+}
+
+export default main
@@ -0,0 +1,214 @@
+import { db, READ_PREFERENCE_SECONDARY } from '../lib/mongodb.mjs'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import mongodb from 'mongodb'
+import logger from '@overleaf/logger'
+import OError from '@overleaf/o-error'
+
+const { ObjectId } = mongodb
+
+async function findUserIds() {
+  const userIds = new Set()
+  const cursor = db.users.find(
+    {},
+    {
+      projection: { _id: 1 },
+      readPreference: READ_PREFERENCE_SECONDARY,
+    }
+  )
+  for await (const user of cursor) {
+    userIds.add(user._id.toString())
+    if (userIds.size % 1_000_000 === 0) {
+      console.log(`=> ${userIds.size} users added`, new Date().toISOString())
+    }
+  }
+  console.log(`=> User ids count: ${userIds.size}`)
+  return userIds
+}
+
+export default async function fixProjectsWithInvalidTokenAccessRefsIds() {
+  const DELETED_USER_COLLABORATOR_IDS = new Set()
+  const PROJECTS_WITH_DELETED_USER = new Set()
+
+  // get a set of all users ids as an in-memory cache
+  const userIds = await findUserIds()
+
+  // default query for finding all projects with non-existing/null or non-empty token access fields
+  const query = {
+    $or: [
+      { tokenAccessReadOnly_refs: { $not: { $type: 'array' } } },
+      { tokenAccessReadAndWrite_refs: { $not: { $type: 'array' } } },
+      { 'tokenAccessReadOnly_refs.0': { $exists: true } },
+      { 'tokenAccessReadAndWrite_refs.0': { $exists: true } },
+    ],
+  }
+
+  await batchedUpdate(
+    db.projects,
+    query,
+    async projects => {
+      for (const project of projects) {
+        const isTokenAccessFieldMissing =
+          !project.tokenAccessReadOnly_refs ||
+          !project.tokenAccessReadAndWrite_refs
+        project.tokenAccessReadOnly_refs ??= []
+        project.tokenAccessReadAndWrite_refs ??= []
+
+        // update the token access fields if necessary
+        if (isTokenAccessFieldMissing) {
+          const fields = [
+            'tokenAccessReadOnly_refs',
+            'tokenAccessReadAndWrite_refs',
+          ]
+          for (const field of fields) {
+            await db.projects.updateOne(
+              {
+                _id: project._id,
+                [field]: { $not: { $type: 'array' } },
+              },
+              { $set: { [field]: [] } }
+            )
+          }
+          console.log(
+            `=> Fixed non-existing token access fields in project ${project._id.toString()}`
+          )
+        }
+
+        // find the set of user ids that are in the token access fields
+        // i.e. the set of collaborators
+        const collaboratorIds = new Set()
+        for (const roUserId of project.tokenAccessReadOnly_refs) {
+          collaboratorIds.add(roUserId.toString())
+        }
+        for (const rwUserId of project.tokenAccessReadAndWrite_refs) {
+          collaboratorIds.add(rwUserId.toString())
+        }
+        // determine which collaborator ids are not in the `users` collection
+        // i.e. the user has been deleted
+        const deletedUserIds = new Set()
+        for (const collaboratorId of collaboratorIds) {
+          if (!userIds.has(collaboratorId)) {
+            deletedUserIds.add(collaboratorId)
+          }
+        }
+
+        // double-check that users doesn't exist in the users collection
+        // we don't want to remove users that were added after the initial query
+        const existingUsersCursor = db.users.find(
+          { _id: { $in: [...deletedUserIds].map(id => new ObjectId(id)) } },
+          { _id: 1 }
+        )
+        for await (const user of existingUsersCursor) {
+          const id = user._id.toString()
+          deletedUserIds.delete(id)
+          // add the user id to the cache
+          userIds.add(id)
+        }
+
+        // remove the actual deleted users
+        for (const deletedUserId of deletedUserIds) {
+          DELETED_USER_COLLABORATOR_IDS.add(deletedUserId)
+          PROJECTS_WITH_DELETED_USER.add(project._id.toString())
+          console.log(
+            '=> Found deleted user id:',
+            deletedUserId,
+            'in project:',
+            project._id.toString()
+          )
+          console.log(
+            `=> Removing deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})`
+          )
+          await removeUserFromAllProjects(new ObjectId(deletedUserId))
+        }
+      }
+    },
+    { tokenAccessReadOnly_refs: 1, tokenAccessReadAndWrite_refs: 1 }
+  )
+
+  console.log(`Deleted user ids (${DELETED_USER_COLLABORATOR_IDS.size})`)
+  if (DELETED_USER_COLLABORATOR_IDS.size) {
+    console.log(Array.from(DELETED_USER_COLLABORATOR_IDS).join('\n'))
+  }
+  console.log(
+    `=> Projects with deleted user ids (${PROJECTS_WITH_DELETED_USER.size})`
+  )
+  if (PROJECTS_WITH_DELETED_USER.size) {
+    console.log(Array.from(PROJECTS_WITH_DELETED_USER).join('\n'))
+  }
+}
+
+// Copied from services/web/app/src/Features/Collaborators/CollaboratorsHandler.js
+async function removeUserFromAllProjects(userId) {
+  const { readAndWrite, readOnly, tokenReadAndWrite, tokenReadOnly } =
+    await dangerouslyGetAllProjectsUserIsMemberOf(userId, { _id: 1 })
+  const allProjects = readAndWrite
+    .concat(readOnly)
+    .concat(tokenReadAndWrite)
+    .concat(tokenReadOnly)
+  logger.info(
+    {
+      userId,
+      readAndWriteCount: readAndWrite.length,
+      readOnlyCount: readOnly.length,
+      tokenReadAndWriteCount: tokenReadAndWrite.length,
+      tokenReadOnlyCount: tokenReadOnly.length,
+    },
+    'removing user from projects'
+  )
+  for (const project of allProjects) {
+    await removeUserFromProject(project._id, userId)
+  }
+  logger.info(
+    {
+      userId,
+      allProjectsCount: allProjects.length,
+    },
+    'removed user from all projects'
+  )
+}
+
+// Copied from services/web/app/src/Features/Collaborators/CollaboratorsHandler.js
+async function removeUserFromProject(projectId, userId) {
+  try {
+    await db.projects.updateOne(
+      { _id: projectId },
+      {
+        $pull: {
+          collaberator_refs: userId,
+          readOnly_refs: userId,
+          reviewer_refs: userId,
+          pendingEditor_refs: userId,
+          pendingReviewer_refs: userId,
+          tokenAccessReadOnly_refs: userId,
+          tokenAccessReadAndWrite_refs: userId,
+          archived: userId,
+          trashed: userId,
+        },
+      }
+    )
+  } catch (err) {
+    throw OError.tag(err, 'problem removing user from project collaborators', {
+      projectId,
+      userId,
+    })
+  }
+}
+
+// Copied from services/web/app/src/Features/Collaborators/CollaboratorsGetter.js
+// This function returns all the projects that a user is a member of, regardless of
+// the current state of the project, so it includes those projects where token access
+// has been disabled.
+async function dangerouslyGetAllProjectsUserIsMemberOf(userId, fields) {
+  const readAndWrite = await db.projects
+    .find({ collaberator_refs: userId }, fields)
+    .toArray()
+  const readOnly = await db.projects
+    .find({ readOnly_refs: userId }, fields)
+    .toArray()
+  const tokenReadAndWrite = await db.projects
+    .find({ tokenAccessReadAndWrite_refs: userId }, fields)
+    .toArray()
+  const tokenReadOnly = await db.projects
+    .find({ tokenAccessReadOnly_refs: userId }, fields)
+    .toArray()
+  return { readAndWrite, readOnly, tokenReadAndWrite, tokenReadOnly }
+}