move migrations to shared location (#28306)

* fix: correct typedef for Document in helpers.mjs

* add move-migrations codemod

* update migration paths to use shared migrations directory

* move migrations to shared location

* fix: update Dockerfile and docker-compose.ci.yml to include migrations directory

* feat: add migrations tool to workspaces in package.json

* [monorepo] Fix order of docker ignore rules

* [web] remove unused docker ignore file

* [monorepo] replace old references to migrations folder

* [server-ce] copy migrations from new place

* [migrations] Inline web scripts

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] move three web scripts over

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] add missing collection

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] remove lodash dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] avoid mongodb-legacy dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [monorepo] run migrations from tools/migrations

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] simplify migration for adding gitBridge feature to users

* [monorepo] run migrations from tests in all the services

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] fixup running web migrations everywhere

* [monorepo] trigger Jenkins builds on changes to mongo migrations

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] build scripts: update devDependencies before deps scanning

* [monorepo] build scripts: formerly depend on tools/migrations

* [monorepo] run eslint on .mjs files

* [migrations] enable more eslint rules and fix all the errors

* [rake] fix migrations:list task

---------

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614

services/history-v1/.jenkinsIncludeFile

@@ -16,3 +16,4 @@ package-lock.json
 package.json
 patches/**
 services/history-v1/**
+tools/migrations/**

services/history-v1/Dockerfile

@@ -30,7 +30,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
 COPY services/history-v1/package.json /overleaf/services/history-v1/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -45,6 +47,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY services/history-v1/ /overleaf/services/history-v1/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/history-v1/Makefile

@@ -26,6 +26,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
 	$(MONOREPO)/services/history-v1/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/history-v1/docker-compose.yml

@@ -47,6 +47,7 @@ services:
       - ../../libraries:/overleaf/libraries
       - minio-certs:/certs
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/history-v1
     environment:
       ELASTIC_SEARCH_DSN: es:9200
@@ -54,7 +55,7 @@ services:
       HISTORY_REDIS_HOST: redis_test
       QUEUES_REDIS_HOST: redis_test
       ANALYTICS_QUEUES_REDIS_HOST: redis_test
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: https://minio:9000
       AWS_S3_PATH_STYLE: 'true'

services/history-v1/package.json

@@ -47,6 +47,7 @@
     "utf-8-validate": "^5.0.4"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "benny": "^3.7.1",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",

services/history-v1/test/setup.js

@@ -2,7 +2,9 @@ const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
 const config = require('config')
 const fetch = require('node-fetch')
-const { knex, mongodb, redis } = require('../storage')
+const { knex, redis } = require('../storage')
+const { exec } = require('node:child_process')
+const { promisify } = require('node:util')
 
 // ensure every ObjectId has the id string as a property for correct comparisons
 require('mongodb').ObjectId.cacheHexString = true
@@ -17,19 +19,9 @@ async function setupPostgresDatabase() {
 
 async function setupMongoDatabase() {
   this.timeout(60_000)
-  await mongodb.db.collection('projectHistoryChunks').createIndexes([
-    {
-      key: { projectId: 1, startVersion: 1 },
-      name: 'projectId_1_startVersion_1',
-      partialFilterExpression: { state: { $in: ['active', 'closed'] } },
-      unique: true,
-    },
-    {
-      key: { state: 1 },
-      name: 'state_1',
-      partialFilterExpression: { state: 'deleted' },
-    },
-  ])
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
 }
 
 async function createGcsBuckets() {