move migrations to shared location (#28306)

* fix: correct typedef for Document in helpers.mjs

* add move-migrations codemod

* update migration paths to use shared migrations directory

* move migrations to shared location

* fix: update Dockerfile and docker-compose.ci.yml to include migrations directory

* feat: add migrations tool to workspaces in package.json

* [monorepo] Fix order of docker ignore rules

* [web] remove unused docker ignore file

* [monorepo] replace old references to migrations folder

* [server-ce] copy migrations from new place

* [migrations] Inline web scripts

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] move three web scripts over

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] add missing collection

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] remove lodash dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] avoid mongodb-legacy dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [monorepo] run migrations from tools/migrations

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] simplify migration for adding gitBridge feature to users

* [monorepo] run migrations from tests in all the services

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] fixup running web migrations everywhere

* [monorepo] trigger Jenkins builds on changes to mongo migrations

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] build scripts: update devDependencies before deps scanning

* [monorepo] build scripts: formerly depend on tools/migrations

* [monorepo] run eslint on .mjs files

* [migrations] enable more eslint rules and fix all the errors

* [rake] fix migrations:list task

---------

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614

tools/migrations/.eastrc

@@ -0,0 +1,6 @@
+{
+  "adapter": "./lib/adapter.mjs",
+  "dir": ".",
+  "migrationNumberFormat": "dateTime",
+  "migrationExtension": "mjs"
+}

tools/migrations/.jenkinsIncludeFile

@@ -0,0 +1,6 @@
+tools/migrations/**
+
+package.json
+package-lock.json
+
+Makefile

tools/migrations/20190720165251_create_migrations.mjs

@@ -0,0 +1,31 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.migrations, [
+    {
+      key: { name: 1 },
+      unique: true,
+    },
+  ])
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.migrations, [{ name: 1 }])
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145001_create_contacts_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.contacts, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.contacts, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145002_create_deletedProjects_indexes.mjs

@@ -0,0 +1,42 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      'deleterData.deletedAt': 1,
+    },
+    name: 'deleterData.deletedAt_1',
+  },
+  {
+    key: {
+      'deleterData.deletedProjectId': 1,
+    },
+    name: 'deleterData.deletedProjectId_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.deletedProjects, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.deletedProjects, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      'subscription._id': 1,
+    },
+    name: 'subscription._id_1',
+  },
+  {
+    key: {
+      'subscription.admin_id': 1,
+    },
+    name: 'subscription.admin_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.deletedSubscriptions, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.deletedSubscriptions, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145004_create_docHistoryIndex_indexes.mjs

@@ -0,0 +1,36 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.docHistoryIndex, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.docHistoryIndex, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145005_create_docHistory_indexes.mjs

@@ -0,0 +1,64 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      doc_id: 1,
+      v: 1,
+    },
+    name: 'doc_id_1_v_1',
+  },
+  {
+    key: {
+      project_id: 1,
+      'meta.end_ts': 1,
+    },
+    name: 'project_id_1_meta.end_ts_1',
+  },
+  {
+    key: {
+      doc_id: 1,
+      project_id: 1,
+    },
+    name: 'doc_id_1_project_id_1',
+  },
+  {
+    key: {
+      expiresAt: 1,
+    },
+    name: 'expiresAt_1',
+    expireAfterSeconds: 0,
+  },
+  {
+    key: {
+      last_checked: 1,
+    },
+    name: 'last_checked_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.docHistory, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.docHistory, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145006_create_docOps_indexes.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+import { getCollectionInternal } from './lib/mongodb.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      doc_id: 1,
+    },
+    name: 'doc_id_1',
+  },
+]
+
+const migrate = async () => {
+  const docOps = await getCollectionInternal('docOps')
+  await Helpers.addIndexesToCollection(docOps, indexes)
+}
+
+const rollback = async () => {
+  const docOps = await getCollectionInternal('docOps')
+  await Helpers.dropIndexesFromCollection(docOps, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145007_create_docSnapshots_indexes.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+  {
+    key: {
+      ts: 1,
+    },
+    name: 'ts_1',
+    expireAfterSeconds: 2592000,
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.docSnapshots, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.docSnapshots, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145008_create_docs_indexes.mjs

@@ -0,0 +1,36 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.docs, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.docs, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145009_create_githubSyncEntityVersions_indexes.mjs

@@ -0,0 +1,47 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      pid: 1,
+      eid: 1,
+    },
+    name: 'pid_1_eid_1',
+  },
+  {
+    key: {
+      c: 1,
+    },
+    name: 'c_1',
+    expireAfterSeconds: 2592000,
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.githubSyncEntityVersions, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(
+      db.githubSyncEntityVersions,
+      indexes
+    )
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145010_create_githubSyncProjectStates_indexes.mjs

@@ -0,0 +1,42 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+  {
+    key: {
+      owner_id: 1,
+    },
+    name: 'owner_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.githubSyncProjectStates, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.githubSyncProjectStates, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145011_create_githubSyncUserCredentials_indexes.mjs

@@ -0,0 +1,39 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.githubSyncUserCredentials, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(
+      db.githubSyncUserCredentials,
+      indexes
+    )
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145012_create_institutions_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      v1Id: 1,
+    },
+    name: 'v1Id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.institutions, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.institutions, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145013_create_messages_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      room_id: 1,
+      timestamp: -1,
+    },
+    name: 'room_id_1_timestamp_-1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.messages, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.messages, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145014_create_notifications_indexes.mjs

@@ -0,0 +1,49 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      expires: 1,
+    },
+    name: 'expires_1',
+    expireAfterSeconds: 10,
+  },
+  {
+    key: {
+      key: 1,
+    },
+    name: 'key_1',
+  },
+  {
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.notifications, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.notifications, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145015_create_oauthAccessTokens_indexes.mjs

@@ -0,0 +1,44 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      accessToken: 1,
+    },
+    name: 'accessToken_1',
+  },
+  {
+    unique: true,
+    key: {
+      refreshToken: 1,
+    },
+    name: 'refreshToken_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145016_create_oauthApplications_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      id: 1,
+    },
+    name: 'id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.oauthApplications, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.oauthApplications, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145017_create_oauthAuthorizationCodes_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      authorizationCode: 1,
+    },
+    name: 'authorizationCode_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.oauthAuthorizationCodes, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.oauthAuthorizationCodes, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145018_create_projectHistoryFailures_indexes.mjs

@@ -0,0 +1,36 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.projectHistoryFailures, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.projectHistoryFailures, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145019_create_projectHistoryLabels_indexes.mjs

@@ -0,0 +1,42 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+  {
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.projectHistoryLabels, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.projectHistoryLabels, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs

@@ -0,0 +1,39 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+]
+
+const migrate = async () => {
+  await Helpers.addIndexesToCollection(
+    await mongodb.getCollectionInternal('projectHistoryMetaData'),
+    indexes
+  )
+}
+
+const rollback = async () => {
+  try {
+    await Helpers.dropIndexesFromCollection(
+      await mongodb.getCollectionInternal('projectHistoryMetaData'),
+      indexes
+    )
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145021_create_projectHistorySyncState_indexes.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+  {
+    key: {
+      expiresAt: 1,
+    },
+    name: 'expiresAt_1',
+    expireAfterSeconds: 0,
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.projectHistorySyncState, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.projectHistorySyncState, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145022_create_projectImportFailures_indexes.mjs

@@ -0,0 +1,41 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      v1_project_id: 1,
+    },
+    name: 'v1_project_id_1',
+  },
+]
+
+async function getCollection() {
+  // NOTE: This is a stale collection, it will get dropped in a later migration.
+  return await getCollectionInternal('projectImportFailures')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+  try {
+    await Helpers.dropIndexesFromCollection(collection, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145023_create_projectInvites_indexes.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      expires: 1,
+    },
+    name: 'expires_1',
+    expireAfterSeconds: 10,
+  },
+  {
+    key: {
+      projectId: 1,
+    },
+    name: 'projectId_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.projectInvites, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.projectInvites, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145024_create_projects_indexes.mjs

@@ -0,0 +1,146 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      active: 1,
+      lastOpened: 1,
+    },
+    name: 'active_1_lastOpened_1',
+  },
+  {
+    unique: true,
+    key: {
+      'tokens.readOnly': 1,
+    },
+    name: 'tokens.readOnly_1',
+    partialFilterExpression: {
+      'tokens.readOnly': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      'overleaf.history.id': 1,
+    },
+    name: 'overleaf.history.id_1',
+    partialFilterExpression: {
+      'overleaf.history.id': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      'tokens.readAndWritePrefix': 1,
+    },
+    name: 'tokens.readAndWritePrefix_1',
+    partialFilterExpression: {
+      'tokens.readAndWritePrefix': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    key: {
+      publicAccesLevel: 1,
+    },
+    name: 'publicAccesLevel_1',
+  },
+  {
+    key: {
+      owner_ref: 1,
+    },
+    name: 'owner_ref_1',
+  },
+  {
+    key: {
+      tokenAccessReadAndWrite_refs: 1,
+    },
+    name: 'tokenAccessReadAndWrite_refs_1',
+  },
+  {
+    key: {
+      readOnly_refs: 1,
+    },
+    name: 'readOnly_refs_1',
+  },
+  {
+    key: {
+      tokenAccessReadOnly_refs: 1,
+    },
+    name: 'tokenAccessReadOnly_refs_1',
+  },
+  {
+    unique: true,
+    key: {
+      'overleaf.id': 1,
+    },
+    name: 'overleaf.id_1',
+    partialFilterExpression: {
+      'overleaf.id': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    key: {
+      collaberator_refs: 1,
+    },
+    name: 'collaberator_refs_1',
+  },
+  {
+    key: {
+      name: 1,
+    },
+    name: 'name_1',
+  },
+  {
+    unique: true,
+    key: {
+      'tokens.readAndWrite': 1,
+    },
+    name: 'tokens.readAndWrite_1',
+    partialFilterExpression: {
+      'tokens.readAndWrite': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    key: {
+      'collabratecUsers.user_id': 1,
+    },
+    name: 'collabratecUsers.user_id_1',
+    sparse: true,
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.projects, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.projects, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145025_create_publishers_indexes.mjs

@@ -0,0 +1,37 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      slug: 1,
+    },
+    name: 'slug_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.publishers, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.publishers, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145026_create_rooms_indexes.mjs

@@ -0,0 +1,36 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.rooms, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.rooms, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145027_create_spellingPreferences_indexes.mjs

@@ -0,0 +1,36 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      token: 1,
+    },
+    name: 'token',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.spellingPreferences, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.spellingPreferences, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145028_create_subscriptions_indexes.mjs

@@ -0,0 +1,91 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      'teamInvites.token': 1,
+    },
+    name: 'teamInvites.token_1',
+    partialFilterExpression: {
+      'teamInvites.token': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      manager_ids: 1,
+    },
+    name: 'manager_ids_1',
+    partialFilterExpression: {
+      manager_ids: {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      admin_id: 1,
+    },
+    name: 'admin_id_1',
+  },
+  {
+    key: {
+      'freeTrial.downgraded': 1,
+      'freeTrial.expiresAt': 1,
+    },
+    name: 'free_trial',
+  },
+  {
+    key: {
+      member_ids: 1,
+    },
+    name: 'member_ids_1',
+  },
+  {
+    key: {
+      invited_emails: 1,
+    },
+    name: 'invited_emails_1',
+  },
+  {
+    unique: true,
+    key: {
+      'overleaf.id': 1,
+    },
+    name: 'overleaf.id_1',
+    partialFilterExpression: {
+      'overleaf.id': {
+        $exists: true,
+      },
+    },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.subscriptions, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.subscriptions, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145029_create_tags_indexes.mjs

@@ -0,0 +1,44 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+  {
+    unique: true,
+    key: {
+      user_id: 1,
+      name: 1,
+    },
+    name: 'user_id_1_name_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.tags, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.tags, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145030_create_templates_indexes.mjs

@@ -0,0 +1,55 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['server-pro', 'saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      project_id: 1,
+    },
+    name: 'project_id_1',
+  },
+  {
+    key: {
+      user_id: 1,
+    },
+    name: 'user_id_1',
+  },
+  {
+    key: {
+      name: 1,
+    },
+    name: 'name_1',
+  },
+]
+
+async function getCollection() {
+  // NOTE: This is a stale collection, it will get dropped in a later migration.
+  return await getCollectionInternal('templates')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+
+  try {
+    await Helpers.dropIndexesFromCollection(collection, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145031_create_tokens_indexes.mjs

@@ -0,0 +1,38 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      token: 1,
+      use: 1,
+    },
+    name: 'token_1_use_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.tokens, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.tokens, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145032_create_users_indexes.mjs

@@ -0,0 +1,170 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    unique: true,
+    key: {
+      email: 1,
+    },
+    name: 'email_case_insensitive',
+    collation: {
+      locale: 'en',
+      caseLevel: false,
+      caseFirst: 'off',
+      strength: 2,
+      numericOrdering: false,
+      alternate: 'non-ignorable',
+      maxVariable: 'punct',
+      normalization: false,
+      backwards: false,
+      version: '57.1',
+    },
+  },
+  {
+    key: {
+      'dropbox.access_token.oauth_token_secret': 1,
+    },
+    name: 'has dropbox',
+  },
+  {
+    unique: true,
+    key: {
+      'overleaf.id': 1,
+    },
+    name: 'overleaf.id_1',
+    partialFilterExpression: {
+      'overleaf.id': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      'thirdPartyIdentifiers.externalUserId': 1,
+      'thirdPartyIdentifiers.providerId': 1,
+    },
+    name: 'thirdPartyIdentifiers.externalUserId_1_thirdPartyIdentifiers.providerId_1',
+    sparse: true,
+  },
+  {
+    key: {
+      'subscription.freeTrialDowngraded': 1,
+    },
+    name: 'subscription.freeTrialDowngraded_1',
+  },
+  {
+    key: {
+      signUpDate: 1,
+    },
+    name: 'signUpDate',
+  },
+  {
+    unique: true,
+    key: {
+      'emails.email': 1,
+    },
+    name: 'emails_email_1',
+    partialFilterExpression: {
+      'emails.email': {
+        $exists: true,
+      },
+    },
+  },
+  {
+    unique: true,
+    key: {
+      'emails.email': 1,
+    },
+    name: 'emails_email_case_insensitive',
+    partialFilterExpression: {
+      'emails.email': {
+        $exists: true,
+      },
+    },
+    collation: {
+      locale: 'en',
+      caseLevel: false,
+      caseFirst: 'off',
+      strength: 2,
+      numericOrdering: false,
+      alternate: 'non-ignorable',
+      maxVariable: 'punct',
+      normalization: false,
+      backwards: false,
+      version: '57.1',
+    },
+  },
+  {
+    unique: true,
+    key: {
+      'dropbox.access_token.uid': 1,
+    },
+    name: 'dropbox.access_token.uid_unique',
+    sparse: true,
+  },
+  {
+    key: {
+      password: 1,
+      email: 1,
+    },
+    name: 'password_and_email',
+  },
+  {
+    key: {
+      referal_id: 1,
+    },
+    name: 'referal_id',
+  },
+  {
+    key: {
+      'subscription.freeTrialExpiresAt': 1,
+    },
+    name: 'subscription.freeTrialExpiresAt_1',
+  },
+  {
+    key: {
+      auth_token: 1,
+    },
+    name: 'auth_token_1',
+  },
+  {
+    unique: true,
+    key: {
+      email: 1,
+    },
+    name: 'email_1',
+  },
+  {
+    key: {
+      'emails.reversedHostname': 1,
+    },
+    name: 'emails.reversedHostname_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.users, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.users, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20190912145033_create_userstubs_indexes.mjs

@@ -0,0 +1,43 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      'overleaf.id': 1,
+    },
+    name: 'overleaf.id_1',
+  },
+]
+
+async function getCollection() {
+  // NOTE: This is a stale collection, it will get dropped in a later migration.
+  return await getCollectionInternal('userstubs')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+
+  try {
+    await Helpers.dropIndexesFromCollection(collection, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20191106102104_saml-log-indexes.mjs

@@ -0,0 +1,61 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      providerId: 1,
+    },
+    name: 'providerId_1',
+  },
+  {
+    key: {
+      sessionId: 1,
+    },
+    name: 'sessionId_1',
+  },
+  {
+    // expire after 30 days
+    expireAfterSeconds: 60 * 60 * 24 * 30,
+    key: {
+      createdAt: 1,
+    },
+    name: 'createdAt_1',
+  },
+]
+
+// Export indexes for use in the fix-up migration 20220105130000_fix_saml_indexes.js.
+const samlLogsIndexes = indexes
+
+async function getCollection() {
+  // This collection was incorrectly named - it should have been `samlLogs`
+  //  instead of `samllog`. The error is corrected by the subsequent migration
+  //  20220105130000_fix_saml_indexes.js.
+  return await getCollectionInternal('samllog')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+  try {
+    await Helpers.dropIndexesFromCollection(collection, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+  samlLogsIndexes,
+}

tools/migrations/20191107191318_saml-indentifiers-index.mjs

@@ -0,0 +1,49 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      'samlIdentifiers.externalUserId': 1,
+      'samlIdentifiers.providerId': 1,
+    },
+    name: 'samlIdentifiers.externalUserId_1_samlIdentifiers.providerId_1',
+    sparse: true,
+  },
+]
+
+// Export indexes for use in the fix-up migration 20220105130000_fix_saml_indexes.js.
+const usersIndexes = indexes
+
+async function getCollection() {
+  // This collection was incorrectly named - it should have been `users` instead
+  //  of `user`. The error is corrected by the subsequent migration
+  //  20220105130000_fix_saml_indexes.js.
+  return await getCollectionInternal('user')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+  try {
+    await Helpers.dropIndexesFromCollection(collection, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+  usersIndexes,
+}

tools/migrations/20200110183327_brandVarationIdIndex.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      brandVariationId: 1,
+    },
+    name: 'brandVariationId_1',
+  },
+]
+
+const migrate = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.projects, indexes)
+}
+
+const rollback = async ({ db }) => {
+  try {
+    await Helpers.dropIndexesFromCollection(db.projects, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200120163346_atlas_recommended_indexes.mjs

@@ -0,0 +1,42 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = {
+  docSnapshots: [{ key: { project_id: 1, ts: 1 }, name: 'project_id_1_ts_1' }],
+  deletedProjects: [
+    {
+      key: { 'deleterData.deletedProjectOwnerId': 1 },
+      name: 'deleterdata_deletedProjectOwnerId_1',
+    },
+  ],
+  docs: [{ key: { project_id: 1, inS3: 1 }, name: 'project_id_1_inS3_1' }],
+}
+
+const migrate = async client => {
+  const { db } = client
+
+  await Promise.all(
+    Object.keys(indexes).map(key =>
+      Helpers.addIndexesToCollection(db[key], indexes[key])
+    )
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  await Promise.all(
+    Object.keys(indexes).map(key =>
+      Helpers.dropIndexesFromCollection(db[key], indexes[key])
+    )
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200210084301_remove-duplicate-deleted-things.mjs

@@ -0,0 +1,48 @@
+/* eslint-disable no-unused-vars */
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+async function removeDuplicates(collection, field) {
+  const duplicates = await collection.aggregate(
+    [
+      {
+        $group: {
+          _id: { projectId: `$deleterData.${field}` },
+          dups: { $addToSet: '$_id' },
+          count: { $sum: 1 },
+        },
+      },
+      { $match: { count: { $gt: 1 } } },
+    ],
+    { allowDiskUse: true }
+  )
+  let duplicate
+  while ((duplicate = await duplicates.next())) {
+    // find duplicate items, ignore the most recent and delete the rest
+    const items = await collection
+      .find(
+        { _id: { $in: duplicate.dups } },
+        { projection: { _id: 1 }, sort: { 'deleterData.deletedAt': -1 } }
+      )
+      .toArray()
+    items.pop()
+    const ids = items.map(item => item._id)
+    await collection.deleteMany({ _id: { $in: ids } })
+  }
+}
+
+const migrate = async client => {
+  const { db } = client
+  await removeDuplicates(db.deletedProjects, 'deletedProjectId')
+  await removeDuplicates(db.deletedUsers, 'deletedUserId')
+}
+
+const rollback = async client => {
+  // can't really do anything here
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200210121103_uniqueify-deletedthings-indexes.mjs

@@ -0,0 +1,76 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  const { db } = client
+  // deletedUsers did not have an index before
+  await Helpers.dropIndexesFromCollection(db.deletedProjects, [
+    {
+      key: {
+        'deleterData.deletedProjectId': 1,
+      },
+      name: 'deleterData.deletedProjectId_1',
+    },
+  ])
+  // deletedUsers did not have an index before
+
+  await Helpers.addIndexesToCollection(db.deletedProjects, [
+    {
+      key: {
+        'deleterData.deletedProjectId': 1,
+      },
+      unique: true,
+      name: 'deleterData.deletedProjectId_1',
+    },
+  ])
+  await Helpers.addIndexesToCollection(db.deletedUsers, [
+    {
+      key: {
+        'deleterData.deletedUserId': 1,
+      },
+      unique: true,
+      name: 'deleterData.deleteUserId_1',
+    },
+  ])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.deletedProjects, [
+    {
+      key: {
+        'deleterData.deletedProjectId': 1,
+      },
+      unique: true,
+      name: 'deleterData.deletedProjectId_1',
+    },
+  ])
+  await Helpers.dropIndexesFromCollection(db.deletedUsers, [
+    {
+      key: {
+        'deleterData.deletedUserId': 1,
+      },
+      unique: true,
+      name: 'deleterData.deleteUserId_1',
+    },
+  ])
+
+  await Helpers.addIndexesToCollection(db.deletedProjects, [
+    {
+      key: {
+        'deleterData.deletedProjectId': 1,
+      },
+      name: 'deleterData.deletedProjectId_1',
+    },
+  ])
+  // deletedUsers did not have an index before
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200302143624_users_affiliationUnchecked_index.mjs

@@ -0,0 +1,31 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      'emails.affiliationUnchecked': 1,
+    },
+    name: 'affiliationUnchecked_1',
+    sparse: true,
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.users, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.users, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200522145727_dropProjectImportFailures.mjs

@@ -0,0 +1,17 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  await Helpers.dropCollection('projectImportFailures')
+}
+
+const rollback = async client => {
+  // can't really do anything here
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200522145741_dropProjectImportBatchRecords.mjs

@@ -0,0 +1,17 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  await Helpers.dropCollection('projectImportBatchRecords')
+}
+
+const rollback = async client => {
+  // can't really do anything here
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200608213302_saml-cache-indexes.mjs

@@ -0,0 +1,44 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      requestId: 1,
+    },
+    name: 'requestId_1',
+  },
+  {
+    // expire after 24 hours
+    expireAfterSeconds: 60 * 60 * 24,
+    key: {
+      createdAt: 1,
+    },
+    name: 'createdAt_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.addIndexesToCollection(db.samlCache, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.samlCache, indexes)
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20200729120824_update_subscriptions_manager_ids_index.mjs

@@ -0,0 +1,50 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const oldIndex = {
+  unique: true,
+  key: {
+    manager_ids: 1,
+  },
+  name: 'manager_ids_1',
+  partialFilterExpression: {
+    manager_ids: {
+      $exists: true,
+    },
+  },
+}
+
+const newIndex = {
+  key: {
+    manager_ids: 1,
+  },
+  name: 'manager_ids_1',
+  sparse: true,
+}
+
+const migrate = async client => {
+  const { db } = client
+
+  await Helpers.dropIndexesFromCollection(db.subscriptions, [oldIndex])
+  await Helpers.addIndexesToCollection(db.subscriptions, [newIndex])
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  try {
+    await Helpers.dropIndexesFromCollection(db.subscriptions, [newIndex])
+    await Helpers.addIndexesToCollection(db.subscriptions, [oldIndex])
+  } catch (err) {
+    console.error('Something went wrong rolling back the migrations', err)
+  }
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20201106094956_active-projects-index-with-id.mjs

@@ -0,0 +1,27 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const index = {
+  key: { _id: 1, lastOpened: 1, active: 1 },
+  name: '_id_1_lastOpened_1_active_1',
+  partialFilterExpression: {
+    active: true,
+  },
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projects, [index])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.projects, [index])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210310111225_create_deletedFiles_projectId_index.mjs

@@ -0,0 +1,32 @@
+import Helpers from './lib/helpers.mjs'
+import { getCollectionInternal } from './lib/mongodb.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      projectId: 1,
+    },
+    name: 'projectId_1',
+  },
+]
+
+async function getCollection() {
+  // NOTE: The deletedFiles collection is not available to the application as it has been retired. Fetch it here.
+  return await getCollectionInternal('deletedFiles')
+}
+
+const migrate = async () => {
+  await Helpers.addIndexesToCollection(await getCollection(), indexes)
+}
+
+const rollback = async () => {
+  await Helpers.dropIndexesFromCollection(await getCollection(), indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210407085118_token-expiry-with-ttl-index.mjs

@@ -0,0 +1,42 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = {
+  tokens: [
+    {
+      // expire all tokens 90 days after they are created
+      expireAfterSeconds: 90 * 24 * 60 * 60,
+      key: {
+        createdAt: 1,
+      },
+      name: 'createdAt_1',
+    },
+  ],
+}
+
+const migrate = async client => {
+  const { db } = client
+
+  await Promise.all(
+    Object.keys(indexes).map(key =>
+      Helpers.addIndexesToCollection(db[key], indexes[key])
+    )
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+
+  await Promise.all(
+    Object.keys(indexes).map(key =>
+      Helpers.dropIndexesFromCollection(db[key], indexes[key])
+    )
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210408123210_create_docs_project_id_deleted_deletedAt_index.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: {
+      project_id: 1,
+      deleted: 1,
+      deletedAt: -1,
+    },
+    name: 'project_id_deleted_deletedAt_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.docs, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.docs, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210721081758_create_history_display_index.mjs

@@ -0,0 +1,28 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      'overleaf.history.display': 1,
+    },
+    name: 'overleaf.history.display_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projects, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.projects, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs

@@ -0,0 +1,40 @@
+import { db } from './lib/mongodb.mjs'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async () => {
+  await batchedUpdate(
+    db.users,
+    { 'emails.confirmedAt': { $type: 'string' } },
+    async function (batch) {
+      for (const user of batch) {
+        for (const email of user.emails) {
+          if (typeof email.confirmedAt === 'string') {
+            await db.users.updateOne(
+              { _id: user._id, 'emails.email': email.email },
+              {
+                $set: {
+                  'emails.$.confirmedAt': new Date(
+                    email.confirmedAt.replace(/ UTC$/, '')
+                  ),
+                },
+              }
+            )
+          }
+        }
+      }
+    },
+    { emails: 1 }
+  )
+}
+
+const rollback = async () => {
+  /* nothing to do */
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs

@@ -0,0 +1,34 @@
+import { db } from './lib/mongodb.mjs'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async () => {
+  await batchedUpdate(
+    db.users,
+    { splitTests: { $exists: true } },
+    async function (batch) {
+      for (const user of batch) {
+        const splitTests = user.splitTests
+        for (const splitTest of Object.values(user.splitTests)) {
+          for (const variant of splitTest) {
+            variant.assignedAt = new Date(variant.assignedAt)
+          }
+        }
+
+        await db.users.updateOne({ _id: user._id }, { $set: { splitTests } })
+      }
+    },
+    { splitTests: 1 }
+  )
+}
+
+const rollback = async () => {
+  /* nothing to do */
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs

@@ -0,0 +1,14 @@
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  // Skip back-filling. The deletedFiles collection will be deleted in a following migration.
+  // The projects.deletedFiles array will be purged as part of the later migration as well.
+}
+
+const rollback = async client => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs

@@ -0,0 +1,50 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import { db } from './lib/mongodb.mjs'
+import { promiseMapWithLimit } from '@overleaf/promise-utils'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  await batchedUpdate(
+    db.projects,
+    // array is not empty ~ array has one item
+    { 'deletedDocs.0': { $exists: true } },
+    async projects => {
+      await processBatch(projects)
+    },
+    { _id: 1, deletedDocs: 1 }
+  )
+}
+
+async function processBatch(projects) {
+  await promiseMapWithLimit(10, projects, async project => {
+    await processProject(project)
+  })
+}
+
+async function processProject(project) {
+  for (const doc of project.deletedDocs) {
+    await backFillDoc(doc)
+  }
+  await cleanupProject(project)
+}
+
+async function backFillDoc(doc) {
+  const { name, deletedAt } = doc
+  await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
+}
+
+async function cleanupProject(project) {
+  await db.projects.updateOne(
+    { _id: project._id },
+    { $set: { deletedDocs: [] } }
+  )
+}
+
+const rollback = async client => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs

@@ -0,0 +1,24 @@
+import runScript from './scripts/back_fill_dummy_doc_meta.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  const { db } = client
+  const [firstProject] = await db.projects
+    .find()
+    .sort({ _id: 1 })
+    .limit(1)
+    .toArray()
+  if (!firstProject) {
+    return
+  }
+  await runScript(firstProject._id)
+}
+
+const rollback = async client => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20210924140139_splittests-name-index.mjs

@@ -0,0 +1,36 @@
+import Helpers from './lib/helpers.mjs'
+import mongodb from './lib/mongodb.mjs'
+const { getCollectionInternal } = mongodb
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      name: 1,
+    },
+    unique: true,
+    name: 'name_1',
+  },
+]
+
+async function getCollection() {
+  // NOTE: We do not access the splittests collection directly. Fetch it here.
+  return await getCollectionInternal('splittests')
+}
+
+const migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
+}
+
+const rollback = async client => {
+  const collection = await getCollection()
+  await Helpers.dropIndexesFromCollection(collection, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220105123000_cleanup_unused_collections.mjs

@@ -0,0 +1,35 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const unusedCollections = [
+  'collaberatorcount',
+  'db.subscriptions',
+  'projectsDeletedByMigration',
+  'readonlycount',
+  'samllog',
+  'sharelatex-production.docOps.2013-12-17T02-26-49.0',
+  'sharelatex-production.projects.2013-12-17T02-26-49.1',
+  'sharelatex-production.users.2013-12-17T02-26-49.2',
+  'sharelatex_production.users',
+  'totalwords',
+  'user',
+  'usersDeletedByMigration',
+  'usersEmailDomains',
+]
+
+const migrate = async () => {
+  for (const name of unusedCollections) {
+    await Helpers.dropCollection(name)
+  }
+}
+
+const rollback = async () => {
+  // We lost the indexes. There is no way back.
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220105130000_fix_saml_indexes.mjs

@@ -0,0 +1,28 @@
+import Helpers from './lib/helpers.mjs'
+import SamlLogsIndexesMigration from './20191106102104_saml-log-indexes.mjs'
+import SamlIndentifiersIndexMigration from './20191107191318_saml-indentifiers-index.mjs'
+
+const { samlLogsIndexes } = SamlLogsIndexesMigration
+const { usersIndexes } = SamlIndentifiersIndexMigration
+
+const tags = ['saas']
+
+const migrate = async ({ db }) => {
+  // Fix-up the previous SAML migrations that were operating on collections with
+  //  typos in their names.
+  await Helpers.addIndexesToCollection(
+    db.users,
+    usersIndexes.map(index => {
+      return Object.assign({}, index, { unique: true })
+    })
+  )
+  await Helpers.addIndexesToCollection(db.samlLogs, samlLogsIndexes)
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220222095146_split_tests_analytics_enabled.mjs

@@ -0,0 +1,57 @@
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  await db.splittests.updateMany(
+    {},
+    { $set: { 'versions.$[version].analyticsEnabled': true } },
+    {
+      arrayFilters: [
+        {
+          'version.active': true,
+          'version.analyticsEnabled': { $exists: false },
+        },
+      ],
+    }
+  )
+  await db.splittests.updateMany(
+    {},
+    { $set: { 'versions.$[version].analyticsEnabled': false } },
+    {
+      arrayFilters: [
+        {
+          'version.active': false,
+          'version.analyticsEnabled': { $exists: false },
+        },
+      ],
+    }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  await db.splittests.updateMany(
+    {},
+    { $unset: { 'versions.$[version].analyticsEnabled': 1 } },
+    {
+      arrayFilters: [
+        { 'version.active': true, 'version.analyticsEnabled': true },
+      ],
+    }
+  )
+  await db.splittests.updateMany(
+    {},
+    { $unset: { 'versions.$[version].analyticsEnabled': 1 } },
+    {
+      arrayFilters: [
+        { 'version.active': false, 'version.analyticsEnabled': false },
+      ],
+    }
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220811111800_create_dropboxEntities_index.mjs

@@ -0,0 +1,28 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      entityId: 1,
+    },
+    name: 'entityId_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.dropboxEntities, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.dropboxEntities, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220815105500_create_dropboxProjects_index.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      userId: 1,
+      projectId: 1,
+    },
+    unique: true,
+    name: 'projectId_userId_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.dropboxProjects, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.dropboxProjects, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220817120900_create_dropboxProjects_index.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: {
+      userId: 1,
+      dropboxId: 1,
+    },
+    unique: true,
+    name: 'userId_dropboxId_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.dropboxProjects, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.dropboxProjects, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220825160708_recreate_dropboxEntities.mjs

@@ -0,0 +1,38 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: { 'overleaf.userId': 1, 'dropbox.id': 1 },
+    name: 'overleaf.userId_1_dropbox.id_1',
+    unique: true,
+    partialFilterExpression: { 'dropbox.id': { $exists: true } },
+  },
+  {
+    key: { 'overleaf.userId': 1, 'overleaf.id': 1 },
+    name: 'overleaf.userId_1_overleaf.id_1',
+    unique: true,
+    partialFilterExpression: { 'overleaf.id': { $exists: true } },
+  },
+  { key: { 'overleaf.userId': 1, 'dropbox.pathLower': 'hashed' } },
+]
+
+const migrate = async client => {
+  const { db } = client
+  // Forcibly drop the dropboxEntities collection. The new structure is
+  // different and we don't want to keep the data with the old structure around.
+  await db.dropboxEntities.drop()
+  await Helpers.addIndexesToCollection(db.dropboxEntities, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.dropboxEntities, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220826104236_disable_alpha_beta_program.mjs

@@ -0,0 +1,17 @@
+const tags = ['server-ce', 'server-pro']
+
+const migrate = async client => {
+  const { db } = client
+  await db.users.updateMany(
+    {},
+    { $set: { alphaProgram: false, betaProgram: false } }
+  )
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220830140459_create_index_user_labsProgram_labsProgramGalileo.mjs

@@ -0,0 +1,32 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: { labsProgram: 1 },
+    name: 'labsProgram_1',
+  },
+  {
+    key: { labsProgramGalileo: 1 },
+    name: 'labsProgramGalileo_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.users, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.users, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220913105500_create_auditLog_indexes.mjs

@@ -0,0 +1,34 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    // expire after 2.5 years
+    expireAfterSeconds: 60 * 60 * 24 * 365 * 2.5,
+    key: {
+      timestamp: 1,
+    },
+    name: 'timestamp_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projectAuditLogEntries, indexes)
+  await Helpers.addIndexesToCollection(db.userAuditLogEntries, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Promise.all([
+    Helpers.dropIndexesFromCollection(db.userAuditLogEntries, indexes),
+    Helpers.dropIndexesFromCollection(db.projectAuditLogEntries, indexes),
+  ])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220913125500_migrate_auditLog_to_collections.mjs

@@ -0,0 +1,74 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import { db } from './lib/mongodb.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async () => {
+  await batchedUpdate(
+    db.users,
+    { auditLog: { $exists: true } },
+    async users => {
+      await processUsersBatch(users)
+    },
+    { _id: 1, auditLog: 1 }
+  )
+
+  await batchedUpdate(
+    db.projects,
+    { auditLog: { $exists: true } },
+    async projects => {
+      await processProjectsBatch(projects)
+    },
+    { _id: 1, auditLog: 1 }
+  )
+}
+
+async function processUsersBatch(users) {
+  if (!users || users.length <= 0) {
+    return
+  }
+
+  const entries = users
+    .map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
+    .flat()
+
+  if (entries?.length > 0) {
+    await db.userAuditLogEntries.insertMany(entries)
+  }
+
+  const userIds = users.map(user => user._id)
+  await db.users.updateMany(
+    { _id: { $in: userIds } },
+    { $unset: { auditLog: 1 } }
+  )
+}
+
+async function processProjectsBatch(projects) {
+  if (!projects || projects.length <= 0) {
+    return
+  }
+
+  const entries = projects
+    .map(project =>
+      project.auditLog.map(log => ({ ...log, projectId: project._id }))
+    )
+    .flat()
+
+  if (entries?.length > 0) {
+    await db.projectAuditLogEntries.insertMany(entries)
+  }
+
+  const projectIds = projects.map(project => project._id)
+  await db.projects.updateMany(
+    { _id: { $in: projectIds } },
+    { $unset: { auditLog: 1 } }
+  )
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20220929193200_add_auditLog_indexes.mjs

@@ -0,0 +1,47 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const projectIndexes = [
+  {
+    key: {
+      projectId: 1,
+    },
+    name: 'project_id_1',
+  },
+]
+
+const userIndexes = [
+  {
+    key: {
+      userId: 1,
+    },
+    name: 'user_id_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(
+    db.projectAuditLogEntries,
+    projectIndexes
+  )
+  await Helpers.addIndexesToCollection(db.userAuditLogEntries, userIndexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Promise.all([
+    Helpers.dropIndexesFromCollection(
+      db.projectAuditLogEntries,
+      projectIndexes
+    ),
+    Helpers.dropIndexesFromCollection(db.userAuditLogEntries, userIndexes),
+  ])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20221111111111_ce_sp_convert_archived_state.mjs

@@ -0,0 +1,64 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+import { promiseMapWithLimit } from '@overleaf/promise-utils'
+import { db, ObjectId } from './lib/mongodb.mjs'
+
+const tags = ['server-ce', 'server-pro']
+
+const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
+
+function getAllUserIds(project) {
+  return Array.from(
+    new Set(
+      [
+        project.owner_ref,
+        ...(project.collaberator_refs || []),
+        ...(project.readOnly_refs || []),
+        ...(project.tokenAccessReadAndWrite_refs || []),
+        ...(project.tokenAccessReadOnly_refs || []),
+      ].map(id => id.toString())
+    )
+  ).map(id => new ObjectId(id))
+}
+
+async function migrateField(field) {
+  await batchedUpdate(
+    db.projects,
+    { [field]: false },
+    { $set: { [field]: [] } }
+  )
+
+  await batchedUpdate(
+    db.projects,
+    { [field]: true },
+    async nextBatch => {
+      await promiseMapWithLimit(WRITE_CONCURRENCY, nextBatch, async project => {
+        await db.projects.updateOne(
+          { _id: project._id },
+          { $set: { [field]: getAllUserIds(project) } }
+        )
+      })
+    },
+    {
+      _id: 1,
+      owner_ref: 1,
+      collaberator_refs: 1,
+      readOnly_refs: 1,
+      tokenAccessReadAndWrite_refs: 1,
+      tokenAccessReadOnly_refs: 1,
+    }
+  )
+}
+
+const migrate = async () => {
+  for (const field of ['archived', 'trashed']) {
+    await migrateField(field)
+  }
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20221122191857_project_history_chunks_indexes.mjs

@@ -0,0 +1,33 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: { projectId: 1, startVersion: 1 },
+    name: 'projectId_1_startVersion_1',
+    partialFilterExpression: { state: 'active' },
+    unique: true,
+  },
+  {
+    key: { state: 1 },
+    name: 'state_1',
+    partialFilterExpression: { state: 'deleted' },
+  },
+]
+
+const migrate = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.projectHistoryChunks, indexes)
+}
+
+const rollback = async ({ db }) => {
+  await Helpers.dropIndexesFromCollection(db.projectHistoryChunks, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230110140452_rename_recurly_cached_status.mjs

@@ -0,0 +1,61 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  // 'recurly' -> 'recurlyStatus'
+  await batchedUpdate(
+    db.subscriptions,
+    {
+      $and: [
+        { recurlyStatus: { $exists: false } },
+        { recurly: { $exists: true } },
+      ],
+    },
+    { $rename: { recurly: 'recurlyStatus' } }
+  )
+  // some records may have already recached the recurly status, discard old cache
+  await batchedUpdate(
+    db.subscriptions,
+    {
+      $and: [
+        { recurlyStatus: { $exists: true } },
+        { recurly: { $exists: true } },
+      ],
+    },
+    { $unset: { recurly: 1 } }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  // 'recurlyStatus' -> 'recurly'
+  await batchedUpdate(
+    db.subscriptions,
+    {
+      $and: [
+        { recurly: { $exists: false } },
+        { recurlyStatus: { $exists: true } },
+      ],
+    },
+    { $rename: { recurlyStatus: 'recurly' } }
+  )
+  // some records may have already recached the recurly status, discard old cache
+  await batchedUpdate(
+    db.subscriptions,
+    {
+      $and: [
+        { recurlyStatus: { $exists: true } },
+        { recurly: { $exists: true } },
+      ],
+    },
+    { $unset: { recurlyStatus: 1 } }
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230124092607_clear_old_2fa_setup.mjs

@@ -0,0 +1,37 @@
+import { db } from './lib/mongodb.mjs'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const batchedUpdateOptions = {
+  VERBOSE_LOGGING: 'true',
+  BATCH_SIZE: '1',
+}
+
+const migrate = async () => {
+  await batchedUpdate(
+    db.users,
+    { 'twoFactorAuthentication.secret': { $exists: true } },
+    { $unset: { twoFactorAuthentication: true } },
+    null,
+    null,
+    batchedUpdateOptions
+  )
+}
+
+const rollback = async () => {
+  await batchedUpdate(
+    db.users,
+    { 'twoFactorAuthentication.secretEncrypted': { $exists: true } },
+    { $unset: { twoFactorAuthentication: true } },
+    null,
+    null,
+    batchedUpdateOptions
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230207134844_group_invite_emails_to_lowercase.mjs

@@ -0,0 +1,46 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.subscriptions,
+    {
+      'teamInvites.0': {
+        $exists: true,
+      },
+    },
+    [
+      {
+        $set: {
+          teamInvites: {
+            $map: {
+              input: '$teamInvites',
+              in: {
+                $mergeObjects: [
+                  '$$this',
+                  {
+                    email: {
+                      $toLower: '$$this.email',
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        },
+      },
+    ]
+  )
+}
+
+const rollback = async () => {
+  // There is no way back.
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230217205311_fix_deleted_history_chunks_index.mjs

@@ -0,0 +1,30 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    // The { state: -1 } sort order works around a restriction of Mongo 4.4
+    // where it doesn't allow multiple indexes with the same keys and different
+    // options. The restriction has been lifted in Mongo 5.0
+    //
+    // See https://www.mongodb.com/docs/manual/core/index-partial/#restrictions
+    key: { state: -1 },
+    name: 'state_pending',
+    partialFilterExpression: { state: 'pending' },
+  },
+]
+
+const migrate = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.projectHistoryChunks, indexes)
+}
+
+const rollback = async ({ db }) => {
+  await Helpers.dropIndexesFromCollection(db.projectHistoryChunks, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230315170739_back_fill_doc_rev.mjs

@@ -0,0 +1,15 @@
+import runScript from './scripts/back_fill_doc_rev.mjs'
+
+const tags = ['server-ce', 'server-pro']
+
+const migrate = async () => {
+  await runScript()
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230405190240_oauth_refresh_tokens_index.mjs

@@ -0,0 +1,53 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const OLD_INDEX = {
+  key: { refreshToken: 1 },
+  name: 'refreshToken_1',
+  unique: true,
+}
+const NEW_INDEX = {
+  key: { refreshToken: 1 },
+  name: 'refreshToken_1',
+  unique: true,
+  partialFilterExpression: { refreshToken: { $exists: true } },
+}
+const TMP_INDEX = {
+  key: { refreshToken: 1, dummyField: 1 },
+  name: 'refreshToken_tmp',
+}
+
+const migrate = async client => {
+  const { db } = client
+  // Create a temporary index so that the refresh tokens are not left unindexed
+  // while we drop the index and recreate it.
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [TMP_INDEX])
+
+  // Drop and recreate the index with different options
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [OLD_INDEX])
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [NEW_INDEX])
+
+  // Drop the temporary index
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [TMP_INDEX])
+}
+
+const rollback = async client => {
+  const { db } = client
+  // Create a temporary index so that the refresh tokens are not left unindexed
+  // while we drop the index and recreate it.
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [TMP_INDEX])
+
+  // Drop and recreate the index with different options
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [NEW_INDEX])
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [OLD_INDEX])
+
+  // Drop the temporary index
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [TMP_INDEX])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230406125632_oauth_tokens_ttl.mjs

@@ -0,0 +1,41 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const ACCESS_TOKENS_INDEX = {
+  name: 'expiresAt_1',
+  key: { expiresAt: 1 },
+  partialFilterExpression: { expiresAt: { $exists: true } },
+  expireAfterSeconds: 0,
+}
+
+const AUTHORIZATION_CODES_INDEX = {
+  name: 'expiresAt_1',
+  key: { expiresAt: 1 },
+  partialFilterExpression: { expiresAt: { $exists: true } },
+  expireAfterSeconds: 0,
+}
+
+const migrate = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [
+    ACCESS_TOKENS_INDEX,
+  ])
+  await Helpers.addIndexesToCollection(db.oauthAuthorizationCodes, [
+    AUTHORIZATION_CODES_INDEX,
+  ])
+}
+
+const rollback = async ({ db }) => {
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [
+    ACCESS_TOKENS_INDEX,
+  ])
+  await Helpers.dropIndexesFromCollection(db.oauthAuthorizationCodes, [
+    AUTHORIZATION_CODES_INDEX,
+  ])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230426095212_personal_oauth_tokens_user_index.mjs

@@ -0,0 +1,27 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const indexes = [
+  {
+    key: { user_id: 1 },
+    name: 'pat_user_id_1',
+    partialFilterExpression: { type: 'pat' },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, indexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230428154643_history_chunks_garbage_collection_index.mjs

@@ -0,0 +1,64 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const oldIndexes = [
+  {
+    key: {
+      state: 1,
+    },
+    name: 'state_1',
+    partialFilterExpression: {
+      state: 'deleted',
+    },
+  },
+  {
+    key: {
+      state: -1,
+    },
+    name: 'state_pending',
+    partialFilterExpression: {
+      state: 'pending',
+    },
+  },
+]
+const newIndexes = [
+  {
+    key: {
+      updatedAt: 1,
+    },
+    name: 'deleted_updated_at',
+    partialFilterExpression: {
+      state: 'deleted',
+    },
+  },
+  {
+    key: {
+      updatedAt: -1,
+    },
+    name: 'pending_updated_at',
+    partialFilterExpression: {
+      state: 'pending',
+    },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projectHistoryChunks, newIndexes)
+  await Helpers.dropIndexesFromCollection(db.projectHistoryChunks, oldIndexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projectHistoryChunks, oldIndexes)
+  await Helpers.dropIndexesFromCollection(db.projectHistoryChunks, newIndexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230502180757_server_pro_oauth_indexes.mjs

@@ -0,0 +1,53 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-pro']
+
+const OAUTH_APPLICATIONS_INDEX = {
+  key: { id: 1 },
+  unique: true,
+  name: 'id_1',
+}
+
+const OAUTH_ACCESS_TOKENS_INDEX = {
+  key: { accessToken: 1 },
+  unique: true,
+  name: 'accessToken_1',
+}
+
+const OAUTH_AUTHORIZATION_CODES_INDEX = {
+  key: { authorizationCode: 1 },
+  unique: true,
+  name: 'authorizationCode_1',
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.oauthApplications, [
+    OAUTH_APPLICATIONS_INDEX,
+  ])
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [
+    OAUTH_ACCESS_TOKENS_INDEX,
+  ])
+  await Helpers.addIndexesToCollection(db.oauthAuthorizationCodes, [
+    OAUTH_AUTHORIZATION_CODES_INDEX,
+  ])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.oauthApplications, [
+    OAUTH_APPLICATIONS_INDEX,
+  ])
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [
+    OAUTH_ACCESS_TOKENS_INDEX,
+  ])
+  await Helpers.dropIndexesFromCollection(db.oauthAuthorizationCodes, [
+    OAUTH_AUTHORIZATION_CODES_INDEX,
+  ])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230512100122_ensure_history_migration.mjs

@@ -0,0 +1,60 @@
+import util from 'node:util'
+const { promisify } = util
+const sleep = promisify(setTimeout)
+
+const tags = ['server-ce', 'server-pro']
+
+const migrate = async client => {
+  const { db } = client
+  const count = await db.projects.countDocuments({
+    'overleaf.history.display': { $ne: true },
+  })
+  if (count > 0) {
+    console.error(`
+-----------------------------------------------------------------------
+
+  Full Project History migration not completed for ${count} projects.
+
+  Starting with Server Pro/Community Edition version 4.0,
+  all projects must use the full project history feature.
+
+  Release 3.5 includes a migration process. Please go back to version
+   3.5 and run through the migration process:
+
+    Overleaf Toolkit setups:
+
+      toolkit$ echo "3.5.13" > config/version
+      toolkit$ bin/up
+
+    Legacy docker compose setups/Horizontal scaling setups:
+
+      Update the image tag for "services -> sharelatex" to
+        Server Pro:        quay.io/sharelatex/sharelatex-pro:3.5.13.
+        Community Edition: sharelatex/sharelatex:3.5.13
+      Then use "docker compose up" to apply the changes.
+
+  Documentation for the migration process:
+    https://github.com/overleaf/overleaf/wiki/Full-Project-History-Migration
+
+
+  Refusing to start up, exiting in 10s.
+
+-----------------------------------------------------------------------
+`)
+    await sleep(10_000)
+
+    throw new Error(
+      `Found ${count} projects not migrated to Full Project History`
+    )
+  }
+}
+
+const rollback = async client => {
+  // Not applicable
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230616153016_user_features_updated_at_index.mjs

@@ -0,0 +1,28 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const index = {
+  key: {
+    featuresUpdatedAt: 1,
+  },
+  name: 'featuresUpdatedAt_1',
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.users, [index])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.users, [index])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs

@@ -0,0 +1,14 @@
+import { db } from './lib/mongodb.mjs'
+const tags = ['server-ce', 'server-pro']
+
+const migrate = async () => {
+  await db.users.updateMany({}, { $set: { 'features.gitBridge': true } })
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20230928092537_backfill_subscriptions_managed_users_feature_flag.mjs

@@ -0,0 +1,27 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.subscriptions,
+    { 'features.managedUsers': { $ne: true } },
+    { $set: { 'features.managedUsers': null } }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.subscriptions,
+    { 'features.managedUsers': { $eq: null } },
+    { $set: { 'features.managedUsers': false } }
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231016101457_drop_history_display_index.mjs

@@ -0,0 +1,24 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const indexes = [
+  {
+    key: { 'overleaf.history.display': 1 },
+    name: 'overleaf.history.display_1',
+  },
+]
+
+const migrate = async ({ db }) => {
+  await Helpers.dropIndexesFromCollection(db.projects, indexes)
+}
+
+const rollback = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.projects, indexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231025094810_sso_config_certificates_array.mjs

@@ -0,0 +1,47 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.ssoConfigs,
+    { certificate: { $exists: true }, certificates: { $exists: false } },
+    [
+      { $set: { certificates: ['$certificate'] } },
+      {
+        $unset: 'certificate',
+      },
+    ]
+  )
+  await batchedUpdate(
+    db.ssoConfigs,
+    { userFirstNameAttribute: null },
+    { $unset: { userFirstNameAttribute: true } }
+  )
+  await batchedUpdate(
+    db.ssoConfigs,
+    { userLastNameAttribute: null },
+    { $unset: { userLastNameAttribute: true } }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.ssoConfigs,
+    { certificate: { $exists: false }, certificates: { $exists: true } },
+    [
+      { $set: { certificate: { $arrayElemAt: ['$certificates', 0] } } },
+      {
+        $unset: 'certificates',
+      },
+    ]
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231030160030_managed_users_enabled.mjs

@@ -0,0 +1,27 @@
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['saas']
+
+const migrate = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.subscriptions,
+    { groupPolicy: { $exists: true } },
+    { $set: { managedUsersEnabled: true } }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  await batchedUpdate(
+    db.subscriptions,
+    { groupPolicy: { $exists: true } },
+    { $unset: { managedUsersEnabled: '' } }
+  )
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231031164732_drop_redundant_indexes.mjs

@@ -0,0 +1,45 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const DOCS_INDEXES = [
+  {
+    key: { project_id: 1 },
+    name: 'project_id_1',
+  },
+]
+
+const TAGS_INDEXES = [
+  {
+    key: { user_id: 1 },
+    name: 'user_id_1',
+  },
+]
+
+const PROJECTS_INDEXES = [
+  {
+    key: { _id: 1, lastOpened: 1, active: 1 },
+    name: '_id_1_lastOpened_1_active_1',
+    partialFilterExpression: { active: true },
+  },
+]
+
+const migrate = async ({ db }) => {
+  await Helpers.dropIndexesFromCollection(db.docs, DOCS_INDEXES)
+  await Helpers.dropIndexesFromCollection(db.tags, TAGS_INDEXES)
+  await Helpers.dropIndexesFromCollection(db.projects, PROJECTS_INDEXES)
+}
+
+const rollback = async ({ db }) => {
+  await Helpers.addIndexesToCollection(db.docs, DOCS_INDEXES)
+  await Helpers.addIndexesToCollection(db.tags, TAGS_INDEXES)
+  await Helpers.addIndexesToCollection(db.projects, PROJECTS_INDEXES)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231101153447_partial_last_opened_index.mjs

@@ -0,0 +1,32 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const OLD_INDEX = {
+  name: 'active_1_lastOpened_1',
+  key: { active: 1, lastOpened: 1 },
+}
+
+const NEW_INDEX = {
+  name: 'lastOpened_1',
+  key: { lastOpened: 1 },
+  partialFilterExpression: { active: true },
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projects, [NEW_INDEX])
+  await Helpers.dropIndexesFromCollection(db.projects, [OLD_INDEX])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projects, [OLD_INDEX])
+  await Helpers.dropIndexesFromCollection(db.projects, [NEW_INDEX])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231101191643_drop_unused_indexes.mjs

@@ -0,0 +1,79 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const PROJECTS_INDEXES = [
+  {
+    name: 'publicAccesLevel_1',
+    key: { publicAccesLevel: 1 },
+  },
+  {
+    name: 'name_1',
+    key: { name: 1 },
+  },
+  {
+    name: 'brandVariationId_1',
+    key: { brandVariationId: 1 },
+  },
+]
+
+const DOC_SNAPSHOTS_INDEXES = [
+  {
+    name: 'project_id_1',
+    key: { project_id: 1 },
+  },
+]
+
+const USERS_INDEXES = [
+  {
+    name: 'owner_ref',
+    key: { owner_ref: 1 },
+  },
+  {
+    name: 'has dropbox',
+    key: { 'dropbox.access_token.oauth_token_secret': 1 },
+  },
+  {
+    name: 'holdingAccount_1',
+    key: { holdingAccount: 1 },
+  },
+  {
+    name: 'subscription.freeTrialDowngraded_1',
+    key: { 'subscription.freeTrialDowngraded': 1 },
+  },
+  {
+    name: 'password_and_email',
+    key: { password: 1, email: 1 },
+  },
+  {
+    name: 'subscription.freeTrialExpiresAt_1',
+    key: { 'subscription.freeTrialExpiresAt': 1 },
+  },
+  {
+    name: 'auth_token_1',
+    key: { auth_token: 1 },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.projects, PROJECTS_INDEXES)
+  await Helpers.dropIndexesFromCollection(
+    db.docSnapshots,
+    DOC_SNAPSHOTS_INDEXES
+  )
+  await Helpers.dropIndexesFromCollection(db.users, USERS_INDEXES)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projects, PROJECTS_INDEXES)
+  await Helpers.addIndexesToCollection(db.docSnapshots, DOC_SNAPSHOTS_INDEXES)
+  await Helpers.addIndexesToCollection(db.users, USERS_INDEXES)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231101204352_drop_unused_saas_indexes.mjs

@@ -0,0 +1,42 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const SUBSCRIPTIONS_INDEXES = [
+  {
+    name: 'free_trial',
+    key: { 'freeTrial.downgraded': 1, 'freeTrial.expiresAt': 1 },
+  },
+]
+
+const USERS_INDEXES = [
+  {
+    name: 'labsProgram_1',
+    key: { labsProgram: 1 },
+  },
+  {
+    name: 'labsProgramGalileo_1',
+    key: { labsProgramGalileo: 1 },
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(
+    db.subscriptions,
+    SUBSCRIPTIONS_INDEXES
+  )
+  await Helpers.dropIndexesFromCollection(db.users, USERS_INDEXES)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.subscriptions, SUBSCRIPTIONS_INDEXES)
+  await Helpers.addIndexesToCollection(db.users, USERS_INDEXES)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs

@@ -0,0 +1,60 @@
+import mongodbLegacy from 'mongodb'
+import { db, getCollectionInternal } from './lib/mongodb.mjs'
+const { ObjectId, ReadPreference } = mongodbLegacy
+
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '1000', 10)
+const MIN_ID = process.env.MIN_ID
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async () => {
+  const docOps = await getCollectionInternal('docOps')
+
+  const filter = {}
+  if (MIN_ID) {
+    filter._id = { $gte: new ObjectId(MIN_ID) }
+  }
+  const records = docOps
+    .find(filter, { readPreference: ReadPreference.secondaryPreferred })
+    .sort({ _id: 1 })
+
+  let docsProcessed = 0
+  let batch = []
+  for await (const record of records) {
+    const docId = record.doc_id
+    const version = record.version
+    batch.push({
+      updateOne: {
+        filter: {
+          _id: docId,
+          version: { $exists: false },
+        },
+        update: { $set: { version } },
+      },
+    })
+    if (batch.length >= BATCH_SIZE) {
+      await db.docs.bulkWrite(batch, { ordered: false })
+      batch = []
+    }
+    docsProcessed += 1
+    if (docsProcessed % 100000 === 0) {
+      console.log(`${docsProcessed} docs processed - last id: ${docId}`)
+    }
+  }
+  if (batch.length > 0) {
+    await db.docs.bulkWrite(batch, { ordered: false })
+  }
+  console.log(`DONE - ${docsProcessed} docs processed`)
+}
+
+const rollback = async ({ db }) => {
+  // Nothing to do on rollback. We don't want to remove versions from the docs
+  // collection because they might be more current than the ones in the docOps
+  // collection.
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231110192205_drop_docops_collection.mjs

@@ -0,0 +1,18 @@
+import Helper from './lib/helpers.mjs'
+const { dropCollection } = Helper
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async () => {
+  await dropCollection('docOps')
+}
+
+const rollback = async client => {
+  // there's no rollback: we can't recover the data
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20231113173237_tokens_user_id_index.mjs

@@ -0,0 +1,28 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const index = {
+  key: {
+    'data.user_id': 1,
+  },
+  name: 'data.user_id_1',
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.tokens, [index])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.tokens, [index])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20240115172206_oauth_issuer_index.mjs

@@ -0,0 +1,31 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const index = {
+  key: {
+    oauthApplication_id: 1,
+  },
+  name: 'oauthApplication_id_1',
+  partialFilterExpression: {
+    oauthApplication_id: {
+      $exists: true,
+    },
+  },
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.oauthAccessTokens, [index])
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.oauthAccessTokens, [index])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs

@@ -0,0 +1,15 @@
+import fixProjectsWithInvalidTokenAccessRefsIds from './scripts/remove_deleted_users_from_token_access_refs.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async () => {
+  await fixProjectsWithInvalidTokenAccessRefsIds()
+}
+
+const rollback = async () => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs

@@ -0,0 +1,53 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+import Crypto from 'node:crypto'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+// Copied from services/web/app/src/Features/Collaborators/CollaboratorsInviteHelper.js
+function hashInviteToken(token) {
+  return Crypto.createHmac('sha256', 'overleaf-token-invite')
+    .update(token)
+    .digest('hex')
+}
+
+const index = {
+  key: {
+    tokenHmac: 1,
+  },
+  name: 'tokenHmac_1',
+}
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.projectInvites, [index])
+
+  await batchedUpdate(
+    db.projectInvites,
+    { tokenHmac: { $exists: false } },
+    async invites => {
+      for (const invite of invites) {
+        const tokenHmac = hashInviteToken(invite.token)
+
+        await db.projectInvites.updateOne(
+          { _id: invite._id },
+          { $set: { tokenHmac } }
+        )
+      }
+    },
+    { token: 1 }
+  )
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.projectInvites, [index])
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}

tools/migrations/20240531082910_remove_project_invite_tokens.mjs

@@ -0,0 +1,21 @@
+/* eslint-disable no-unused-vars */
+
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['server-ce', 'server-pro', 'saas']
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.assertDependency(
+    '20240524135408_add_token_hmac_project_invite_tokens'
+  )
+  await db.projectInvites.updateMany({}, { $unset: { token: 1 } })
+}
+
+const rollback = async client => {}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}