diff --git a/libraries/access-token-encryptor/package.json b/libraries/access-token-encryptor/package.json index 4eb55c299d..fda3d2341f 100644 --- a/libraries/access-token-encryptor/package.json +++ b/libraries/access-token-encryptor/package.json @@ -5,8 +5,8 @@ "main": "index.js", "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/fetch-utils/package.json b/libraries/fetch-utils/package.json index d106957245..b742161209 100644 --- a/libraries/fetch-utils/package.json +++ b/libraries/fetch-utils/package.json @@ -5,8 +5,8 @@ "main": "index.js", "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/logger/package.json b/libraries/logger/package.json index 653ea9403c..c12ae4e12a 100644 --- a/libraries/logger/package.json +++ b/libraries/logger/package.json @@ -13,8 +13,8 @@ "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "test:ci": "npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", "types:check": "tsc --noEmit" diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index 295b24caa9..23eaccb869 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -31,8 +31,8 @@ "typescript": "^5.0.4" }, "scripts": { - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "test:unit": "mocha --exit test/**/*.{js,cjs}", "test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance", "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", diff --git a/libraries/mongo-utils/package.json b/libraries/mongo-utils/package.json index 055fb9b811..db038ad942 100644 --- a/libraries/mongo-utils/package.json +++ b/libraries/mongo-utils/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/o-error/package.json b/libraries/o-error/package.json index b455c226cf..3676dfa9a1 100644 --- a/libraries/o-error/package.json +++ b/libraries/o-error/package.json @@ -18,8 +18,8 @@ ], "scripts": { "build": "npm run --silent test", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", diff --git a/libraries/object-persistor/package.json b/libraries/object-persistor/package.json index ec0215f906..d60f33e489 100644 --- a/libraries/object-persistor/package.json +++ b/libraries/object-persistor/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/overleaf-editor-core/package.json b/libraries/overleaf-editor-core/package.json index 03d1834ee4..be09da6554 100644 --- a/libraries/overleaf-editor-core/package.json +++ b/libraries/overleaf-editor-core/package.json @@ -7,8 +7,8 @@ "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "test:ci": "npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", "types:check": "tsc --noEmit" diff --git a/libraries/promise-utils/package.json b/libraries/promise-utils/package.json index f7b3cb8400..e19538422d 100644 --- a/libraries/promise-utils/package.json +++ b/libraries/promise-utils/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/ranges-tracker/package.json b/libraries/ranges-tracker/package.json index ad502cd5b2..7ea19adca1 100644 --- a/libraries/ranges-tracker/package.json +++ b/libraries/ranges-tracker/package.json @@ -9,8 +9,8 @@ "author": "Overleaf (https://www.overleaf.com)", "private": true, "scripts": { - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", diff --git a/libraries/redis-wrapper/package.json b/libraries/redis-wrapper/package.json index 39bcbb454c..62e2f3642e 100644 --- a/libraries/redis-wrapper/package.json +++ b/libraries/redis-wrapper/package.json @@ -13,8 +13,8 @@ "repository": "github:overleaf/redis-wrapper", "license": "ISC", "scripts": { - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", diff --git a/libraries/settings/package.json b/libraries/settings/package.json index c0ed118391..bfbd717039 100644 --- a/libraries/settings/package.json +++ b/libraries/settings/package.json @@ -5,8 +5,8 @@ "repository": "overleaf/settings-module", "main": "index.js", "scripts": { - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", diff --git a/libraries/stream-utils/package.json b/libraries/stream-utils/package.json index c197ff767e..82fcd98883 100644 --- a/libraries/stream-utils/package.json +++ b/libraries/stream-utils/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "test:unit": "mocha --exit test/**/*.{js,cjs}", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", "test:ci": "npm run test:unit", diff --git a/libraries/validation-tools/package.json b/libraries/validation-tools/package.json index a82bfc2519..3aa85ff7bb 100644 --- a/libraries/validation-tools/package.json +++ b/libraries/validation-tools/package.json @@ -13,8 +13,8 @@ "test": "npm run lint && npm run format && npm run types:check && npm run test:unit", "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'", "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'", - "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .", - "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", "test:ci": "npm run test:unit", "test:unit": "vitest --config vitest.config.ts", "types:check": "tsc --noEmit" diff --git a/package-lock.json b/package-lock.json index 2d837588f4..77329c85e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,8 @@ "services/third-party-references", "services/tpdsworker", "services/web", - "tools/saas-e2e" + "tools/saas-e2e", + "tools/migrations" ], "dependencies": { "patch-package": "^8.0.0" @@ -14867,6 +14868,10 @@ "resolved": "libraries/metrics", "link": true }, + "node_modules/@overleaf/migrations": { + "resolved": "tools/migrations", + "link": true + }, "node_modules/@overleaf/mongo-utils": { "resolved": "libraries/mongo-utils", "link": true @@ -15248,7 +15253,6 @@ "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "dev": true, "optional": true, - "peer": true, "engines": { "node": ">=14" } @@ -51368,6 +51372,7 @@ "mongodb": "6.12.0" }, "devDependencies": { + "@overleaf/migrations": "*", "acorn": "^7.1.1", "ajv": "^6.12.0", "chai": "^4.3.6", @@ -51531,6 +51536,7 @@ "underscore": "~1.13.1" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "esmock": "^2.6.3", @@ -51605,6 +51611,7 @@ }, "devDependencies": { "@google-cloud/storage": "^6.10.1", + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", @@ -51682,6 +51689,7 @@ "requestretry": "^7.1.0" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "cluster-key-slot": "^1.0.5", @@ -51815,6 +51823,7 @@ }, "devDependencies": { "@overleaf/fetch-utils": "*", + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", @@ -51949,6 +51958,7 @@ "utf-8-validate": "^5.0.4" }, "devDependencies": { + "@overleaf/migrations": "*", "benny": "^3.7.1", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", @@ -52689,6 +52699,7 @@ "zod-validation-error": "^4.0.1" }, "devDependencies": { + "@overleaf/migrations": "*", "@types/method-override": "^3.0.0", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", @@ -52724,6 +52735,7 @@ "request": "^2.88.2" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", @@ -52896,6 +52908,7 @@ "request": "^2.88.2" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", @@ -53150,6 +53163,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/migrations": "*", "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", @@ -53185,7 +53199,6 @@ "csurf": "^1.11.0", "csv": "^6.2.5", "dateformat": "1.0.4-1.2.3", - "east": "^2.0.2", "ejs": "^3.1.10", "email-addresses": "^5.0.0", "eventsource-parser": "^1.1.2", @@ -54545,6 +54558,18 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "tools/migrations": { + "name": "@overleaf/migrations", + "dependencies": { + "@overleaf/logger": "*", + "@overleaf/mongo-utils": "*", + "@overleaf/o-error": "*", + "@overleaf/promise-utils": "*", + "@overleaf/settings": "*", + "east": "2.0.3", + "mongodb": "6.12.0" + } + }, "tools/saas-e2e": { "name": "@overleaf/saas-e2e", "devDependencies": { diff --git a/package.json b/package.json index fe20b06f4e..d492778d95 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "services/third-party-references", "services/tpdsworker", "services/web", - "tools/saas-e2e" + "tools/saas-e2e", + "tools/migrations" ] } diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index 5f32eb67cb..b7ed9cce74 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -14,6 +14,7 @@ ADD server-ce/services.js /overleaf/services.js ADD package.json package-lock.json /overleaf/ ADD libraries/ /overleaf/libraries/ ADD services/ /overleaf/services/ +ADD tools/migrations/ /overleaf/tools/migrations/ # Add npm patches # ----------------------- diff --git a/server-ce/init_scripts/900_run_web_migrations.sh b/server-ce/init_scripts/900_run_web_migrations.sh index cc206a528b..c1ddb5deda 100755 --- a/server-ce/init_scripts/900_run_web_migrations.sh +++ b/server-ce/init_scripts/900_run_web_migrations.sh @@ -8,6 +8,6 @@ else fi echo "Running migrations for $environment" -cd /overleaf/services/web +cd /overleaf/tools/migrations /sbin/setuser www-data npm run migrations -- migrate -t "$environment" echo "Finished migrations" diff --git a/server-ce/test/.jenkinsIncludeFile b/server-ce/test/.jenkinsIncludeFile index ef1d5a6dbf..dccf0a5db9 100644 --- a/server-ce/test/.jenkinsIncludeFile +++ b/server-ce/test/.jenkinsIncludeFile @@ -68,7 +68,6 @@ services/web/config/settings.defaults.js services/web/config/settings.overrides.server-pro.js services/web/frontend/** services/web/locales/** -services/web/migrations/** services/web/modules/*/* services/web/modules/*/app/** services/web/modules/*/frontend/** @@ -78,6 +77,8 @@ services/web/public/** services/web/types/** services/web/webpack-plugins/** +tools/migrations/** + .dockerignore .eslint* .pretter* diff --git a/services/chat/.jenkinsIncludeFile b/services/chat/.jenkinsIncludeFile index c9fada213f..d1b28ff396 100644 --- a/services/chat/.jenkinsIncludeFile +++ b/services/chat/.jenkinsIncludeFile @@ -6,8 +6,10 @@ libraries/logger/** libraries/metrics/** libraries/mongo-utils/** libraries/o-error/** +libraries/promise-utils/** libraries/settings/** package-lock.json package.json patches/** services/chat/** +tools/migrations/** diff --git a/services/chat/Dockerfile b/services/chat/Dockerfile index e29c5c4989..df54c83daf 100644 --- a/services/chat/Dockerfile +++ b/services/chat/Dockerfile @@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json +COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY services/chat/package.json /overleaf/services/chat/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/ COPY libraries/metrics/ /overleaf/libraries/metrics/ COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/ COPY libraries/o-error/ /overleaf/libraries/o-error/ +COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY services/chat/ /overleaf/services/chat/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/chat/Makefile b/services/chat/Makefile index 40a2f9fe2c..60a67c50ff 100644 --- a/services/chat/Makefile +++ b/services/chat/Makefile @@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/metrics/package.json \ $(MONOREPO)/libraries/mongo-utils/package.json \ $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/services/chat/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index 32de3bf570..b7f7aa2653 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -31,10 +31,11 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/chat environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/chat/package.json b/services/chat/package.json index 1a81f0aaa1..0bdfa3e4c8 100644 --- a/services/chat/package.json +++ b/services/chat/package.json @@ -29,6 +29,7 @@ "mongodb": "6.12.0" }, "devDependencies": { + "@overleaf/migrations": "*", "acorn": "^7.1.1", "ajv": "^6.12.0", "chai": "^4.3.6", diff --git a/services/chat/test/acceptance/js/helpers/ChatApp.js b/services/chat/test/acceptance/js/helpers/ChatApp.js index 3a0baf5ce0..32e6a25711 100644 --- a/services/chat/test/acceptance/js/helpers/ChatApp.js +++ b/services/chat/test/acceptance/js/helpers/ChatApp.js @@ -1,5 +1,6 @@ import { createServer } from '../../../../app/js/server.js' import { promisify } from 'node:util' +import './MongoHelper.js' export { db } from '../../../../app/js/mongodb.js' diff --git a/services/chat/test/acceptance/js/helpers/MongoHelper.js b/services/chat/test/acceptance/js/helpers/MongoHelper.js new file mode 100644 index 0000000000..cf969c6210 --- /dev/null +++ b/services/chat/test/acceptance/js/helpers/MongoHelper.js @@ -0,0 +1,10 @@ +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}) diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 8c5ffa59f9..21cde4076e 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -34,7 +34,7 @@ services: working_dir: /overleaf/services/clsi environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/contacts/.jenkinsIncludeFile b/services/contacts/.jenkinsIncludeFile index 50cb21ab4e..3b87ed779c 100644 --- a/services/contacts/.jenkinsIncludeFile +++ b/services/contacts/.jenkinsIncludeFile @@ -6,8 +6,10 @@ libraries/logger/** libraries/metrics/** libraries/mongo-utils/** libraries/o-error/** +libraries/promise-utils/** libraries/settings/** package-lock.json package.json patches/** services/contacts/** +tools/migrations/** diff --git a/services/contacts/Dockerfile b/services/contacts/Dockerfile index f72764db69..13ec7097d2 100644 --- a/services/contacts/Dockerfile +++ b/services/contacts/Dockerfile @@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json +COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY services/contacts/package.json /overleaf/services/contacts/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/ COPY libraries/metrics/ /overleaf/libraries/metrics/ COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/ COPY libraries/o-error/ /overleaf/libraries/o-error/ +COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY services/contacts/ /overleaf/services/contacts/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/contacts/Makefile b/services/contacts/Makefile index b4f68097de..8d92635a83 100644 --- a/services/contacts/Makefile +++ b/services/contacts/Makefile @@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/metrics/package.json \ $(MONOREPO)/libraries/mongo-utils/package.json \ $(MONOREPO)/libraries/o-error/package.json \ + $(MONOREPO)/libraries/promise-utils/package.json \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/services/contacts/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index f68c5d2bde..59ecadc38a 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -31,10 +31,11 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/contacts environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/contacts/package.json b/services/contacts/package.json index 00e4859d81..ed894f5391 100644 --- a/services/contacts/package.json +++ b/services/contacts/package.json @@ -31,6 +31,7 @@ "underscore": "~1.13.1" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "esmock": "^2.6.3", diff --git a/services/contacts/test/acceptance/js/MongoHelper.js b/services/contacts/test/acceptance/js/MongoHelper.js new file mode 100644 index 0000000000..cf969c6210 --- /dev/null +++ b/services/contacts/test/acceptance/js/MongoHelper.js @@ -0,0 +1,10 @@ +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}) diff --git a/services/docstore/.jenkinsIncludeFile b/services/docstore/.jenkinsIncludeFile index a9eee55286..0bdf856796 100644 --- a/services/docstore/.jenkinsIncludeFile +++ b/services/docstore/.jenkinsIncludeFile @@ -14,3 +14,4 @@ package-lock.json package.json patches/** services/docstore/** +tools/migrations/** diff --git a/services/docstore/Dockerfile b/services/docstore/Dockerfile index 524855ebf7..19b0b60768 100644 --- a/services/docstore/Dockerfile +++ b/services/docstore/Dockerfile @@ -23,7 +23,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json COPY services/docstore/package.json /overleaf/services/docstore/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -36,6 +38,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY services/docstore/ /overleaf/services/docstore/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/docstore/Makefile b/services/docstore/Makefile index b5d051f2e0..9ec95321f1 100644 --- a/services/docstore/Makefile +++ b/services/docstore/Makefile @@ -24,6 +24,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/libraries/stream-utils/package.json \ $(MONOREPO)/services/docstore/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index ce09176420..aa1a282764 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -31,10 +31,11 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/docstore environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres GCS_API_ENDPOINT: http://gcs:9090 GCS_PROJECT_ID: fake diff --git a/services/docstore/package.json b/services/docstore/package.json index e00b829194..359d73aa24 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@google-cloud/storage": "^6.10.1", + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js index 27bced4699..b6a7b3b8a8 100644 --- a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js +++ b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js @@ -1,5 +1,6 @@ const app = require('../../../../app') const Settings = require('@overleaf/settings') +require('./MongoHelper') function startApp() { return new Promise((resolve, reject) => { diff --git a/services/docstore/test/acceptance/js/helpers/MongoHelper.js b/services/docstore/test/acceptance/js/helpers/MongoHelper.js new file mode 100644 index 0000000000..cf969c6210 --- /dev/null +++ b/services/docstore/test/acceptance/js/helpers/MongoHelper.js @@ -0,0 +1,10 @@ +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}) diff --git a/services/document-updater/.jenkinsIncludeFile b/services/document-updater/.jenkinsIncludeFile index 9867b1b0d4..2023a28626 100644 --- a/services/document-updater/.jenkinsIncludeFile +++ b/services/document-updater/.jenkinsIncludeFile @@ -15,3 +15,4 @@ package-lock.json package.json patches/** services/document-updater/** +tools/migrations/** diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 147655394e..48e2be6218 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -24,7 +24,9 @@ COPY libraries/ranges-tracker/package.json /overleaf/libraries/ranges-tracker/pa COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY services/document-updater/package.json /overleaf/services/document-updater/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -38,6 +40,7 @@ COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY services/document-updater/ /overleaf/services/document-updater/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index 70abc0318a..3e25c71ad2 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/redis-wrapper/package.json \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/services/document-updater/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 816a344f35..92227a6a8c 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -37,6 +37,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/document-updater environment: ELASTIC_SEARCH_DSN: es:9200 @@ -44,7 +45,7 @@ services: HISTORY_REDIS_HOST: redis_test QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_REDIS_HOST: redis_test - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/document-updater/package.json b/services/document-updater/package.json index 9d419cbf1d..158fbb3faf 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -41,6 +41,7 @@ "requestretry": "^7.1.0" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "cluster-key-slot": "^1.0.5", diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js index aa06e5f3ec..7eda7984d3 100644 --- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js +++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js @@ -1,4 +1,5 @@ const app = require('../../../../app') +require('./MongoHelper') function startApp() { return new Promise((resolve, reject) => { diff --git a/services/document-updater/test/acceptance/js/helpers/MongoHelper.js b/services/document-updater/test/acceptance/js/helpers/MongoHelper.js new file mode 100644 index 0000000000..2dc6bcaedc --- /dev/null +++ b/services/document-updater/test/acceptance/js/helpers/MongoHelper.js @@ -0,0 +1,10 @@ +const { exec } = require('node:child_process') +const { promisify } = require('node:util') + +before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}) diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 1e92c9b464..74ef2df8de 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -38,7 +38,7 @@ services: working_dir: /overleaf/services/filestore environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres AWS_S3_ENDPOINT: https://minio:9000 AWS_S3_PATH_STYLE: 'true' diff --git a/services/history-v1/.jenkinsIncludeFile b/services/history-v1/.jenkinsIncludeFile index cfd84c7e38..d50f947739 100644 --- a/services/history-v1/.jenkinsIncludeFile +++ b/services/history-v1/.jenkinsIncludeFile @@ -16,3 +16,4 @@ package-lock.json package.json patches/** services/history-v1/** +tools/migrations/** diff --git a/services/history-v1/Dockerfile b/services/history-v1/Dockerfile index fb23158ca7..18105972c4 100644 --- a/services/history-v1/Dockerfile +++ b/services/history-v1/Dockerfile @@ -30,7 +30,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json COPY services/history-v1/package.json /overleaf/services/history-v1/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -45,6 +47,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY services/history-v1/ /overleaf/services/history-v1/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile index 632786cd21..1d4256288f 100644 --- a/services/history-v1/Makefile +++ b/services/history-v1/Makefile @@ -26,6 +26,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/libraries/stream-utils/package.json \ $(MONOREPO)/services/history-v1/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 0713873909..8b27e8a17d 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -47,6 +47,7 @@ services: - ../../libraries:/overleaf/libraries - minio-certs:/certs - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/history-v1 environment: ELASTIC_SEARCH_DSN: es:9200 @@ -54,7 +55,7 @@ services: HISTORY_REDIS_HOST: redis_test QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_REDIS_HOST: redis_test - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres AWS_S3_ENDPOINT: https://minio:9000 AWS_S3_PATH_STYLE: 'true' diff --git a/services/history-v1/package.json b/services/history-v1/package.json index c647c3088f..7cedcbef51 100644 --- a/services/history-v1/package.json +++ b/services/history-v1/package.json @@ -47,6 +47,7 @@ "utf-8-validate": "^5.0.4" }, "devDependencies": { + "@overleaf/migrations": "*", "benny": "^3.7.1", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", diff --git a/services/history-v1/test/setup.js b/services/history-v1/test/setup.js index 60974173de..4d168f17a6 100644 --- a/services/history-v1/test/setup.js +++ b/services/history-v1/test/setup.js @@ -2,7 +2,9 @@ const chai = require('chai') const chaiAsPromised = require('chai-as-promised') const config = require('config') const fetch = require('node-fetch') -const { knex, mongodb, redis } = require('../storage') +const { knex, redis } = require('../storage') +const { exec } = require('node:child_process') +const { promisify } = require('node:util') // ensure every ObjectId has the id string as a property for correct comparisons require('mongodb').ObjectId.cacheHexString = true @@ -17,19 +19,9 @@ async function setupPostgresDatabase() { async function setupMongoDatabase() { this.timeout(60_000) - await mongodb.db.collection('projectHistoryChunks').createIndexes([ - { - key: { projectId: 1, startVersion: 1 }, - name: 'projectId_1_startVersion_1', - partialFilterExpression: { state: { $in: ['active', 'closed'] } }, - unique: true, - }, - { - key: { state: 1 }, - name: 'state_1', - partialFilterExpression: { state: 'deleted' }, - }, - ]) + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) } async function createGcsBuckets() { diff --git a/services/notifications/.jenkinsIncludeFile b/services/notifications/.jenkinsIncludeFile index fd805630af..207bd1395d 100644 --- a/services/notifications/.jenkinsIncludeFile +++ b/services/notifications/.jenkinsIncludeFile @@ -13,3 +13,4 @@ package-lock.json package.json patches/** services/notifications/** +tools/migrations/** diff --git a/services/notifications/Dockerfile b/services/notifications/Dockerfile index 9dfd883422..fab27da9a7 100644 --- a/services/notifications/Dockerfile +++ b/services/notifications/Dockerfile @@ -22,7 +22,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json COPY services/notifications/package.json /overleaf/services/notifications/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -34,6 +36,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/ COPY services/notifications/ /overleaf/services/notifications/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 1d0c35554e..9477ad9221 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -23,6 +23,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/libraries/validation-tools/package.json \ $(MONOREPO)/services/notifications/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index 96a6bb7c8e..cfaf888467 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -32,11 +32,12 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json working_dir: /overleaf/services/notifications environment: ELASTIC_SEARCH_DSN: es:9200 - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/notifications/package.json b/services/notifications/package.json index 7ca3a8fc90..e28504c3e7 100644 --- a/services/notifications/package.json +++ b/services/notifications/package.json @@ -38,6 +38,7 @@ "zod-validation-error": "^4.0.1" }, "devDependencies": { + "@overleaf/migrations": "*", "@types/method-override": "^3.0.0", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", diff --git a/services/notifications/test/acceptance/js/HealthCheck.test.ts b/services/notifications/test/acceptance/js/HealthCheck.test.ts index 1ffa60a5ef..1d35759700 100644 --- a/services/notifications/test/acceptance/js/HealthCheck.test.ts +++ b/services/notifications/test/acceptance/js/HealthCheck.test.ts @@ -2,6 +2,7 @@ import { beforeAll, describe, it, expect } from 'vitest' import { fetchStringWithResponse } from '@overleaf/fetch-utils' import app from '../../../app.ts' import logger from '@overleaf/logger' +import './MongoHelper.ts' let runAppPromise: Promise | null = null diff --git a/services/notifications/test/acceptance/js/MongoHelper.ts b/services/notifications/test/acceptance/js/MongoHelper.ts new file mode 100644 index 0000000000..f0ee6ce5c6 --- /dev/null +++ b/services/notifications/test/acceptance/js/MongoHelper.ts @@ -0,0 +1,9 @@ +import { beforeAll } from 'vitest' +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +beforeAll(async function () { + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}, 60_000) diff --git a/services/project-history/.jenkinsIncludeFile b/services/project-history/.jenkinsIncludeFile index 522a7ce2ee..593a538fa2 100644 --- a/services/project-history/.jenkinsIncludeFile +++ b/services/project-history/.jenkinsIncludeFile @@ -15,3 +15,4 @@ package-lock.json package.json patches/** services/project-history/** +tools/migrations/** diff --git a/services/project-history/Dockerfile b/services/project-history/Dockerfile index 15bc7cab73..5966fca8cd 100644 --- a/services/project-history/Dockerfile +++ b/services/project-history/Dockerfile @@ -24,7 +24,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json COPY services/project-history/package.json /overleaf/services/project-history/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ RUN cd /overleaf && npm ci --quiet COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ @@ -38,6 +40,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY services/project-history/ /overleaf/services/project-history/ +COPY tools/migrations/ /overleaf/tools/migrations/ FROM app USER node diff --git a/services/project-history/Makefile b/services/project-history/Makefile index ba939f8e0a..4f18c7df40 100644 --- a/services/project-history/Makefile +++ b/services/project-history/Makefile @@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \ $(MONOREPO)/libraries/settings/package.json \ $(MONOREPO)/libraries/stream-utils/package.json \ $(MONOREPO)/services/project-history/package.json \ + $(MONOREPO)/tools/migrations/package.json \ $(MONOREPO)/patches/* \ | sha256sum | cut -d '-' -f1) diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index c92c43f9e4..9bea08c8d2 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -37,6 +37,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/project-history environment: ELASTIC_SEARCH_DSN: es:9200 @@ -44,7 +45,7 @@ services: HISTORY_REDIS_HOST: redis_test QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_REDIS_HOST: redis_test - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/project-history/package.json b/services/project-history/package.json index 120f6a6898..8163799e75 100644 --- a/services/project-history/package.json +++ b/services/project-history/package.json @@ -42,6 +42,7 @@ "request": "^2.88.2" }, "devDependencies": { + "@overleaf/migrations": "*", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "mocha": "^11.1.0", diff --git a/services/project-history/test/acceptance/js/helpers/MongoHelper.js b/services/project-history/test/acceptance/js/helpers/MongoHelper.js new file mode 100644 index 0000000000..cf969c6210 --- /dev/null +++ b/services/project-history/test/acceptance/js/helpers/MongoHelper.js @@ -0,0 +1,10 @@ +import { exec } from 'node:child_process' +import { promisify } from 'node:util' + +before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + ) +}) diff --git a/services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js b/services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js index df8ced4da7..31b848b2fe 100644 --- a/services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js +++ b/services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js @@ -1,5 +1,6 @@ import { app } from '../../../../app/js/server.js' import { mongoClient } from '../../../../app/js/mongodb.js' +import './MongoHelper.js' let running = false let initPromise = null diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index c368a904c8..2678561151 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -39,7 +39,7 @@ services: HISTORY_REDIS_HOST: redis_test QUEUES_REDIS_HOST: redis_test ANALYTICS_QUEUES_REDIS_HOST: redis_test - MONGO_HOST: mongo + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} diff --git a/services/web/.dockerignore b/services/web/.dockerignore deleted file mode 120000 index 3e4e48b0b5..0000000000 --- a/services/web/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/services/web/.eslintrc.js b/services/web/.eslintrc.js index 52e25ca43e..8056ffd48b 100644 --- a/services/web/.eslintrc.js +++ b/services/web/.eslintrc.js @@ -286,10 +286,8 @@ module.exports = { 'scripts/add_subscription_members_csv.mjs', 'scripts/analytics/helpers/GoogleBigQueryHelper.mjs', 'scripts/attach_dangling_comments_to_doc.mjs', - 'scripts/back_fill_doc_rev.mjs', 'scripts/backfill_mixpanel_user_properties.mjs', 'scripts/backfill_project_image_name.mjs', - 'scripts/backfill_project_invites_token_hmac.mjs', 'scripts/backfill_user_properties.mjs', 'scripts/backfill_users_sso_attribute.mjs', 'scripts/bench_bcrypt.mjs', diff --git a/services/web/.jenkinsIncludeFile b/services/web/.jenkinsIncludeFile index 79b7675130..1bff84f00e 100644 --- a/services/web/.jenkinsIncludeFile +++ b/services/web/.jenkinsIncludeFile @@ -20,3 +20,4 @@ package-lock.json package.json patches/** services/web/** +tools/migrations/** diff --git a/services/web/Dockerfile b/services/web/Dockerfile index e33e56ed67..c5f1824189 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -39,6 +39,7 @@ COPY libraries/settings/package.json /overleaf/libraries/settings/package.json COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json COPY services/web/package.json /overleaf/services/web/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json COPY patches/ /overleaf/patches/ RUN cd /overleaf && NODE_ENV=production npm ci --quiet @@ -72,6 +73,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/ COPY services/web/ /overleaf/services/web/ +COPY tools/migrations/ /overleaf/tools/migrations/ # Build the latex parser RUN cd /overleaf/services/web && npm run 'lezer-latex:generate' @@ -108,6 +110,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/ COPY services/web/ /overleaf/services/web/ +COPY tools/migrations/ /overleaf/tools/migrations/ # Omit Server Pro/CE specific scripts from SaaS image RUN rm /overleaf/services/web/modules/server-ce-scripts -rf diff --git a/services/web/Makefile b/services/web/Makefile index 9a5526cdfa..3a836f7be7 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -146,7 +146,7 @@ test_unit_module: mongo_migrations_for_tests $(MAKE) modules/$(MODULE_NAME)/test_unit mongo_migrations_for_tests: - $(DOCKER_COMPOSE) run --rm test_unit npm run migrations -- migrate -t saas + $(DOCKER_COMPOSE) run --rm --workdir /overleaf/tools/migrations test_unit npm run migrations -- migrate -t saas # # Frontend tests diff --git a/services/web/buildscript.txt b/services/web/buildscript.txt index 7d0529c3ec..25bec39fa0 100644 --- a/services/web/buildscript.txt +++ b/services/web/buildscript.txt @@ -1,5 +1,5 @@ web ---dependencies= +--dependencies=mongo --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker --env-add= --env-pass-through= diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index 0ec8e5c06b..d766b9113f 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/web env_file: docker-compose.common.env environment: @@ -37,6 +38,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations user: node working_dir: /overleaf/services/web env_file: docker-compose.common.env diff --git a/services/web/migrations/20190730093801_script_example.mjs b/services/web/migrations/20190730093801_script_example.mjs deleted file mode 100644 index fda971a242..0000000000 --- a/services/web/migrations/20190730093801_script_example.mjs +++ /dev/null @@ -1,28 +0,0 @@ -/* eslint-disable no-unused-vars */ - -/* - * Example migration for a script: - * - * This migration demonstrates how to run a script. In this case, the example - * script will print "hello world" if there are no users in the users collection - * or "hello ", when User.findOne() finds something. - */ - -import runScript from '../scripts/example/script_for_migration.mjs' - -const tags = [] - -const migrate = async client => { - const { db } = client - await runScript() -} - -const rollback = async client => { - const { db } = client -} - -export default { - tags, - migrate, - rollback, -} diff --git a/services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs b/services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs deleted file mode 100644 index 7016fd6fb5..0000000000 --- a/services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs +++ /dev/null @@ -1,19 +0,0 @@ -import runScript from '../scripts/back_fill_doc_name_for_deleted_docs.mjs' - -const tags = ['server-ce', 'server-pro', 'saas'] - -const migrate = async client => { - const options = { - performCleanup: true, - letUserDoubleCheckInputsFor: 10, - } - await runScript(options) -} - -const rollback = async client => {} - -export default { - tags, - migrate, - rollback, -} diff --git a/services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs b/services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs deleted file mode 100644 index 7fc776e4d5..0000000000 --- a/services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import runScript from '../scripts/migrate_audit_logs.mjs' - -const tags = ['server-ce', 'server-pro', 'saas'] - -const migrate = async () => { - const options = { - letUserDoubleCheckInputsFor: 10, - writeConcurrency: 5, - dryRun: false, - } - await runScript(options) -} - -const rollback = async () => {} - -export default { - tags, - migrate, - rollback, -} diff --git a/services/web/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs b/services/web/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs deleted file mode 100644 index 2325394c76..0000000000 --- a/services/web/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs +++ /dev/null @@ -1,19 +0,0 @@ -const tags = ['server-ce', 'server-pro'] - -const migrate = async () => { - // Run-time import as SaaS does not ship with the server-ce-scripts module - const { default: runScript } = await import( - '../modules/server-ce-scripts/scripts/upgrade-user-features.mjs' - ) - await runScript(false, { - gitBridge: 1, - }) -} - -const rollback = async () => {} - -export default { - tags, - migrate, - rollback, -} diff --git a/services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs b/services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs deleted file mode 100644 index 5577a7ff32..0000000000 --- a/services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs +++ /dev/null @@ -1,30 +0,0 @@ -/* eslint-disable no-unused-vars */ - -import Helpers from './lib/helpers.mjs' -import runScript from '../scripts/backfill_project_invites_token_hmac.mjs' - -const tags = ['server-ce', 'server-pro', 'saas'] - -const index = { - key: { - tokenHmac: 1, - }, - name: 'tokenHmac_1', -} - -const migrate = async client => { - const { db } = client - await Helpers.addIndexesToCollection(db.projectInvites, [index]) - await runScript(false) -} - -const rollback = async client => { - const { db } = client - await Helpers.dropIndexesFromCollection(db.projectInvites, [index]) -} - -export default { - tags, - migrate, - rollback, -} diff --git a/services/web/package.json b/services/web/package.json index 1873edd580..57eaf343e1 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -40,7 +40,6 @@ "type-check": "tsc --noEmit", "type-check:backend": "tsc -p tsconfig.backend.json --noEmit", "extract-translations": "i18next-scanner", - "migrations": "MONGO_SOCKET_TIMEOUT=0 east --es-modules", "convert-themes": "node frontend/js/features/source-editor/themes/scripts/convert.js", "cypress:open-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress open --component", "cypress:run-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress run --component --browser chrome", @@ -91,6 +90,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/migrations": "*", "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", @@ -126,7 +126,6 @@ "csurf": "^1.11.0", "csv": "^6.2.5", "dateformat": "1.0.4-1.2.3", - "east": "^2.0.2", "ejs": "^3.1.10", "email-addresses": "^5.0.0", "eventsource-parser": "^1.1.2", @@ -194,6 +193,7 @@ "zod-validation-error": "^4.0.1" }, "devDependencies": { + "5to6-codemod": "^1.8.0", "@babel/cli": "^7.27.0", "@babel/core": "^7.26.10", "@babel/plugin-proposal-decorators": "^7.27.0", @@ -278,7 +278,6 @@ "@writefull/core": "^1.27.27", "@writefull/ui": "^1.27.27", "@writefull/utils": "^1.27.27", - "5to6-codemod": "^1.8.0", "abort-controller": "^3.0.0", "acorn": "^7.1.1", "acorn-walk": "^7.1.1", @@ -354,8 +353,8 @@ "mock-fs": "^5.1.2", "nock": "^13.5.6", "nvd3": "^1.8.6", - "path-browserify": "^1.0.1", "p-reflect": "^3.1.0", + "path-browserify": "^1.0.1", "pdfjs-dist": "5.1.91", "pirates": "^4.0.1", "postcss": "^8.4.31", diff --git a/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs b/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs deleted file mode 100644 index b311176184..0000000000 --- a/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs +++ /dev/null @@ -1,97 +0,0 @@ -import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils' -import { db } from '../app/src/infrastructure/mongodb.js' -import { fileURLToPath } from 'node:url' -import _ from 'lodash' -import { scriptRunner } from './lib/ScriptRunner.mjs' - -const sleep = promisify(setTimeout) - -async function main(options, trackProgress) { - if (!options) { - options = {} - } - _.defaults(options, { - writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10, - performCleanup: process.argv.pop() === '--perform-cleanup', - letUserDoubleCheckInputsFor: parseInt( - process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000, - 10 - ), - }) - - await letUserDoubleCheckInputs(options) - - await batchedUpdate( - db.projects, - // array is not empty ~ array has one item - { 'deletedDocs.0': { $exists: true } }, - async projects => { - await processBatch(projects, options) - }, - { _id: 1, deletedDocs: 1 }, - undefined, - { trackProgress } - ) -} - -async function processBatch(projects, options) { - await promiseMapWithLimit( - options.writeConcurrency, - projects, - async project => { - await processProject(project, options) - } - ) -} - -async function processProject(project, options) { - for (const doc of project.deletedDocs) { - await backFillDoc(doc) - } - if (options.performCleanup) { - await cleanupProject(project) - } -} - -async function backFillDoc(doc) { - const { name, deletedAt } = doc - await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } }) -} - -async function cleanupProject(project) { - await db.projects.updateOne( - { _id: project._id }, - { $set: { deletedDocs: [] } } - ) -} - -async function letUserDoubleCheckInputs(options) { - if (options.performCleanup) { - console.error('BACK FILLING AND PERFORMING CLEANUP') - } else { - console.error( - 'BACK FILLING ONLY - You will need to rerun with --perform-cleanup' - ) - } - console.error( - 'Waiting for you to double check inputs for', - options.letUserDoubleCheckInputsFor, - 'ms' - ) - await sleep(options.letUserDoubleCheckInputsFor) -} - -export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - try { - await scriptRunner( - async trackProgress => await main(undefined, trackProgress) - ) - process.exit(0) - } catch (error) { - console.error({ error }) - process.exit(1) - } -} diff --git a/services/web/scripts/backfill_project_invites_token_hmac.mjs b/services/web/scripts/backfill_project_invites_token_hmac.mjs deleted file mode 100644 index 64a0b405a9..0000000000 --- a/services/web/scripts/backfill_project_invites_token_hmac.mjs +++ /dev/null @@ -1,79 +0,0 @@ -import { db } from '../app/src/infrastructure/mongodb.js' -import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import minimist from 'minimist' -import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js' -import { fileURLToPath } from 'node:url' - -const argv = minimist(process.argv.slice(2), { - boolean: ['dry-run', 'help'], - default: { - 'dry-run': true, - }, -}) - -const DRY_RUN = argv['dry-run'] - -async function addTokenHmacField(DRY_RUN) { - const query = { tokenHmac: { $exists: false } } - - await batchedUpdate( - db.projectInvites, - query, - async invites => { - for (const invite of invites) { - console.log( - `=> Missing "tokenHmac" token in invitation: ${invite._id.toString()}` - ) - - if (DRY_RUN) { - console.log( - `=> DRY RUN - would add "tokenHmac" token to invitation ${invite._id.toString()}` - ) - continue - } - - const tokenHmac = CollaboratorsInviteHelper.hashInviteToken( - invite.token - ) - - await db.projectInvites.updateOne( - { _id: invite._id }, - { $set: { tokenHmac } } - ) - - console.log( - `=> Added "tokenHmac" token to invitation ${invite._id.toString()}` - ) - } - }, - { token: 1 } - ) -} - -async function main(DRY_RUN) { - await addTokenHmacField(DRY_RUN) -} - -export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - if (argv.help || argv._.length > 1) { - console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs - Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record. - - Options: - --dry-run finds invitations without HMAC token but does not do any updates - `) - - process.exit(1) - } - - try { - await main(DRY_RUN) - console.error('Done') - process.exit(0) - } catch (error) { - console.error(error) - process.exit(1) - } -} diff --git a/services/web/scripts/mark_migration.mjs b/services/web/scripts/mark_migration.mjs index cf592140a3..2ee8f7de63 100644 --- a/services/web/scripts/mark_migration.mjs +++ b/services/web/scripts/mark_migration.mjs @@ -1,4 +1,4 @@ -import Adapter from '../migrations/lib/adapter.mjs' +import Adapter from '../../../tools/migrations/lib/adapter.mjs' import { promises as fs } from 'node:fs' import { join, dirname } from 'node:path' import { fileURLToPath } from 'node:url' diff --git a/services/web/scripts/migrate_audit_logs.mjs b/services/web/scripts/migrate_audit_logs.mjs deleted file mode 100644 index 9591275fca..0000000000 --- a/services/web/scripts/migrate_audit_logs.mjs +++ /dev/null @@ -1,169 +0,0 @@ -import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils' -import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' -import _ from 'lodash' -import { fileURLToPath } from 'node:url' -import { scriptRunner } from './lib/ScriptRunner.mjs' - -const sleep = promisify(setTimeout) - -async function main(options, trackProgress) { - if (!options) { - options = {} - } - _.defaults(options, { - dryRun: process.env.DRY_RUN !== 'false', - projectId: process.env.PROJECT_ID, - userId: process.env.USER_ID, - skipUsersMigration: process.env.SKIP_USERS_MIGRATION === 'true', - writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10, - letUserDoubleCheckInputsFor: parseInt( - process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000, - 10 - ), - }) - - await letUserDoubleCheckInputs(options) - - if (options.projectId) { - console.log('migrating projectId=' + options.projectId) - const project = await db.projects.findOne( - { _id: new ObjectId(options.projectId) }, - { _id: 1, auditLog: 1 } - ) - if (!project || !project.auditLog) { - console.error('unable to process project', project) - return - } - await processProjectsBatch([project], options) - } else if (options.userId) { - console.log('migrating userId=' + options.userId) - const user = await db.users.findOne( - { _id: new ObjectId(options.userId) }, - { _id: 1, auditLog: 1 } - ) - if (!user || !user.auditLog) { - console.error('unable to process user', user) - return - } - await processUsersBatch([user], options) - } else { - if (!options.skipUsersMigration) { - await batchedUpdate( - db.users, - { auditLog: { $exists: true } }, - async users => { - await processUsersBatch(users, options) - }, - { _id: 1, auditLog: 1 }, - undefined, - { trackProgress } - ) - } - - // most projects are processed after its owner has been processed, but only those - // users with an existing `auditLog` have been taken into consideration, leaving - // some projects orphan. This batched update processes all remaining projects. - await batchedUpdate( - db.projects, - { auditLog: { $exists: true } }, - async projects => { - await processProjectsBatch(projects, options) - }, - { _id: 1, auditLog: 1 }, - undefined, - { trackProgress } - ) - } -} - -async function processUsersBatch(users, options) { - if (!users || users.length <= 0) { - return - } - - const entries = users - .map(user => user.auditLog.map(log => ({ ...log, userId: user._id }))) - .flat() - - if (!options.dryRun && entries?.length > 0) { - await db.userAuditLogEntries.insertMany(entries) - } - - if (!options.dryRun) { - const userIds = users.map(user => user._id) - await db.users.updateMany( - { _id: { $in: userIds } }, - { $unset: { auditLog: 1 } } - ) - } - - await promiseMapWithLimit(options.writeConcurrency, users, async user => { - const projects = await db.projects - .find( - { owner_ref: user._id, auditLog: { $exists: true } }, - { _id: 1, auditLog: 1 } - ) - .toArray() - await processProjectsBatch(projects, options) - }) -} - -async function processProjectsBatch(projects, options) { - if (!projects || projects.length <= 0) { - return - } - - const entries = projects - .map(project => - project.auditLog.map(log => ({ ...log, projectId: project._id })) - ) - .flat() - - if (!options.dryRun && entries?.length > 0) { - await db.projectAuditLogEntries.insertMany(entries) - } - - if (!options.dryRun) { - const projectIds = projects.map(project => project._id) - await db.projects.updateMany( - { _id: { $in: projectIds } }, - { $unset: { auditLog: 1 } } - ) - } -} - -async function letUserDoubleCheckInputs(options) { - const allOptions = { - ...options, - // batchedUpdate() environment variables - BATCH_DESCENDING: process.env.BATCH_DESCENDING, - BATCH_SIZE: process.env.BATCH_SIZE, - VERBOSE_LOGGING: process.env.VERBOSE_LOGGING, - BATCH_LAST_ID: process.env.BATCH_LAST_ID, - BATCH_RANGE_END: process.env.BATCH_RANGE_END, - SKIP_USERS_MIGRATION: process.env.SKIP_USERS_MIGRATION, - } - console.error('Options:', JSON.stringify(allOptions, null, 2)) - console.error( - 'Waiting for you to double check inputs for', - options.letUserDoubleCheckInputsFor, - 'ms' - ) - await sleep(options.letUserDoubleCheckInputsFor) -} - -export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - try { - await scriptRunner( - async trackProgress => await main(undefined, trackProgress) - ) - console.log('Done.') - process.exit(0) - } catch (error) { - console.error({ error }) - process.exit(1) - } -} diff --git a/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs b/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs index 97e1e1f54d..599e684447 100644 --- a/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs +++ b/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs @@ -63,14 +63,11 @@ describe('BackFillDocNameForDeletedDocs', function () { await setDeletedDocs(projectId2, deletedDocs2) }) - async function runScript(args = []) { + async function runScript() { let result try { result = await promisify(exec)( - ['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1'] - .concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs.mjs']) - .concat(args) - .join(' ') + 'cd ../../tools/migrations && east migrate -t saas --force 20210727150530_ce_sp_backfill_deleted_docs' ) } catch (error) { // dump details like exit code, stdErr and stdOut @@ -95,20 +92,9 @@ describe('BackFillDocNameForDeletedDocs', function () { }) } - describe('back fill only', function () { - beforeEach('run script', runScript) - - checkDocsBackFilled() - - it('should leave the deletedDocs as is', async function () { - expect(await getDeletedDocs(projectId1)).to.deep.equal(deletedDocs1) - expect(await getDeletedDocs(projectId2)).to.deep.equal(deletedDocs2) - }) - }) - describe('back fill and cleanup', function () { beforeEach('run script with cleanup flag', async function () { - await runScript(['--perform-cleanup']) + await runScript() }) checkDocsBackFilled() diff --git a/services/web/test/acceptance/src/BackFillDocRevTests.mjs b/services/web/test/acceptance/src/BackFillDocRevTests.mjs index df62d1276f..ad15f2fed5 100644 --- a/services/web/test/acceptance/src/BackFillDocRevTests.mjs +++ b/services/web/test/acceptance/src/BackFillDocRevTests.mjs @@ -17,16 +17,11 @@ describe('BackFillDocRevTests', function () { ]) }) - async function runScript(dryRun) { + async function runScript() { let result try { result = await promisify(exec)( - [ - 'VERBOSE_LOGGING=true', - 'node', - 'scripts/back_fill_doc_rev.mjs', - dryRun, - ].join(' ') + 'cd ../../tools/migrations && VERBOSE_LOGGING=true east migrate -t server-ce --force 20230315170739_back_fill_doc_rev' ) } catch (error) { // dump details like exit code, stdErr and stdOut @@ -47,24 +42,9 @@ describe('BackFillDocRevTests', function () { ) } - describe('dry-run=true', function () { - beforeEach('run script', async function () { - await runScript('--dry-run=true') - }) - - it('should not back fill the rev', async function () { - const docs = await db.docs.find({}, { $sort: { _id: 1 } }).toArray() - expect(docs).to.deep.equal([ - { _id: docId1, deleted: true }, - { _id: docId2 }, - { _id: docId3, rev: 42 }, - ]) - }) - }) - describe('dry-run=false', function () { beforeEach('run script', async function () { - await runScript('--dry-run=false') + await runScript() }) it('should back fill the rev', async function () { diff --git a/services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs b/services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs index 852e19ec2c..1b82b129d3 100644 --- a/services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs +++ b/services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs @@ -7,8 +7,6 @@ import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js' const DUMMY_NAME = 'unknown.tex' const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z') -const ONE_DAY_IN_S = 60 * 60 * 24 -const BATCH_SIZE = 3 function getObjectIdFromDate(date) { const seconds = new Date(date).getTime() / 1000 @@ -18,7 +16,6 @@ function getObjectIdFromDate(date) { describe('BackFillDummyDocMeta', function () { let docIds let projectIds - let stopAtSeconds beforeEach('create docs', async function () { docIds = [] docIds[0] = getObjectIdFromDate('2021-04-01T00:00:00.000Z') @@ -48,11 +45,15 @@ describe('BackFillDummyDocMeta', function () { // two docs in the same project projectIds[10] = projectIds[9] projectIds[11] = projectIds[4] - - stopAtSeconds = new Date('2021-04-17T00:00:00.000Z').getTime() / 1000 }) const now = new Date() beforeEach('insert doc stubs into docs collection', async function () { + // don't look here, just drop duplicates from the list of projectIds :) + await db.projects.insertMany( + Array.from(new Set(projectIds.map(id => id.toString()))).map(_id => ({ + _id: new ObjectId(_id), + })) + ) await db.docs.insertMany([ // incomplete, without deletedDocs context { _id: docIds[0], project_id: projectIds[0], deleted: true }, @@ -104,33 +105,18 @@ describe('BackFillDummyDocMeta', function () { ]) }) - let options - async function runScript(dryRun) { - options = { - BATCH_SIZE, - CACHE_SIZE: 100, - DRY_RUN: dryRun, - FIRST_PROJECT_ID: projectIds[0].toString(), - INCREMENT_BY_S: ONE_DAY_IN_S, - STOP_AT_S: stopAtSeconds, - // start right away - LET_USER_DOUBLE_CHECK_INPUTS_FOR: 1, - } + async function runScript() { let result try { result = await promisify(exec)( - Object.entries(options) - .map(([key, value]) => `${key}=${value}`) - .concat(['node', 'scripts/back_fill_dummy_doc_meta.mjs']) - .join(' ') + 'cd ../../tools/migrations && east migrate -t saas --force 20210728115327_ce_sp_backfill_dummy_doc_meta' ) } catch (error) { // dump details like exit code, stdErr and stdOut logger.error({ error }, 'script failed') throw error } - let { stderr: stdErr, stdout: stdOut } = result - stdErr = stdErr.split('\n') + let { stdout: stdOut } = result stdOut = stdOut.split('\n').filter(filterOutput) expect(stdOut.filter(filterOutput)).to.include.members([ @@ -145,42 +131,8 @@ describe('BackFillDummyDocMeta', function () { `Orphaned deleted doc ${docIds[9]} (no deletedProjects entry)`, `Orphaned deleted doc ${docIds[10]} (no deletedProjects entry)`, ]) - expect(stdErr.filter(filterOutput)).to.include.members([ - `Processed 9 until ${projectIds[9]}`, - 'Done.', - ]) } - describe('DRY_RUN=true', function () { - beforeEach('run script', async function () { - await runScript(true) - }) - - it('should leave docs as is', async function () { - const docs = await db.docs.find({}).toArray() - expect(docs).to.deep.equal([ - { _id: docIds[0], project_id: projectIds[0], deleted: true }, - { _id: docIds[1], project_id: projectIds[1], deleted: true }, - { _id: docIds[2], project_id: projectIds[2], deleted: true }, - { _id: docIds[3], project_id: projectIds[3], deleted: true }, - { _id: docIds[4], project_id: projectIds[4], deleted: true }, - { - _id: docIds[5], - project_id: projectIds[5], - deleted: true, - name: 'foo.tex', - deletedAt: now, - }, - { _id: docIds[6], project_id: projectIds[6] }, - { _id: docIds[7], project_id: projectIds[7], deleted: true }, - { _id: docIds[8], project_id: projectIds[8], deleted: true }, - { _id: docIds[9], project_id: projectIds[9], deleted: true }, - { _id: docIds[10], project_id: projectIds[10], deleted: true }, - { _id: docIds[11], project_id: projectIds[11], deleted: true }, - ]) - }) - }) - describe('DRY_RUN=false', function () { beforeEach('run script', async function () { await runScript(false) diff --git a/services/web/test/acceptance/src/ConvertArchivedState.mjs b/services/web/test/acceptance/src/ConvertArchivedState.mjs index 06ce5e3b15..29c1608355 100644 --- a/services/web/test/acceptance/src/ConvertArchivedState.mjs +++ b/services/web/test/acceptance/src/ConvertArchivedState.mjs @@ -120,7 +120,7 @@ describe('ConvertArchivedState', function () { beforeEach(function (done) { exec( - 'east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state', + 'cd ../../tools/migrations && east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state', error => { if (error) { return done(error) diff --git a/services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js b/services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js index da5f842bb6..5fa88b6fb3 100644 --- a/services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js +++ b/services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js @@ -28,7 +28,7 @@ describe('ConvertEmailConfirmedAtToDates', function () { beforeEach('run migration', function (done) { exec( - 'east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates', + 'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates', done ) }) diff --git a/services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js b/services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js index bfec1d1fb2..1f27726876 100644 --- a/services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js +++ b/services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js @@ -65,7 +65,7 @@ describe('ConvertSplitTestAssignedAtToDates', function () { beforeEach('run migration', function (done) { exec( - 'east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates', + 'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates', done ) }) diff --git a/services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs b/services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs index ce596a1837..7aa0c895d6 100644 --- a/services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs +++ b/services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs @@ -22,9 +22,8 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () { const projectId3 = new ObjectId('65d726e807c024c8db43be24') const projectId4 = new ObjectId('65d726e807c024c8db43be25') - let insertedProjects beforeEach('insert projects', async function () { - insertedProjects = await db.projects.insertMany([ + await db.projects.insertMany([ { _id: projectId1, tokenAccessReadAndWrite_refs: [userId1], @@ -47,17 +46,11 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () { let stdOut - const runScript = async (dryRun, projectsList) => { + const runScript = async () => { let result try { result = await promisify(exec)( - [ - 'VERBOSE_LOGGING=true', - 'node', - 'scripts/remove_deleted_users_from_token_access_refs.mjs', - dryRun, - projectsList, - ].join(' ') + 'cd ../../tools/migrations && east migrate -t saas --force 20240220130452_remove_deleted_users_from_token_access_refs' ) } catch (error) { // dump details like exit code, stdErr and stdOut @@ -70,91 +63,6 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () { expect(stdOut).to.match(new RegExp(`User ids count: ${insertedUsersCount}`)) } - describe('dry-run=true', function () { - beforeEach('run script', async function () { - await runScript('--dry-run=true') - expect(stdOut).to.match(/doing dry run/i) - }) - - it('should show current user id to be removed', function () { - expect(stdOut).to.match( - new RegExp( - `Found deleted user id: ${userId2.toString()} in project: ${projectId2.toString()}` - ) - ) - expect(stdOut).to.match( - new RegExp( - `DRY RUN - would remove deleted ${userId2.toString()} from all projects \\(found in project ${projectId2.toString()}\\)` - ) - ) - expect(stdOut).to.match( - new RegExp( - `Found deleted user id: ${userId3.toString()} in project: ${projectId3.toString()}` - ) - ) - expect(stdOut).to.match( - new RegExp( - `DRY RUN - would remove deleted ${userId3.toString()} from all projects \\(found in project ${projectId3.toString()}\\)` - ) - ) - }) - - it('should show projects with non-existing token access fields', function () { - expect(stdOut) - .to.match( - new RegExp( - `DRY RUN - would fix non-existing token access fields in project ${projectId3.toString()}` - ) - ) - .and.match( - new RegExp( - `DRY RUN - would fix non-existing token access fields in project ${projectId4.toString()}` - ) - ) - }) - - it('should show the user ids (and their count) to be deleted', function () { - expect(stdOut).to.match( - new RegExp( - `DRY RUN - would delete user ids \\(2\\)\\n${userId2.toString()}\\n${userId3.toString()}` - ) - ) - }) - - it('should show the project ids (and their count) that needs fixing', function () { - expect(stdOut).to.match( - new RegExp( - `Projects with deleted user ids \\(2\\)\\n${projectId2.toString()}\\n${projectId3.toString()}` - ) - ) - }) - - it('should not fix the token access fields of projects', async function () { - const projects = await db.projects - .find({}, { $sort: { _id: 1 } }) - .toArray() - expect(projects).to.deep.equal([ - { - _id: projectId1, - tokenAccessReadAndWrite_refs: [userId1], - tokenAccessReadOnly_refs: [], - }, - { - _id: projectId2, - tokenAccessReadAndWrite_refs: [userId2], - tokenAccessReadOnly_refs: [], - }, - { - _id: projectId3, - tokenAccessReadAndWrite_refs: [userId3], - }, - { - _id: projectId4, - }, - ]) - }) - }) - describe('dry-run=false', function () { beforeEach('run script', async function () { await runScript('--dry-run=false') @@ -237,20 +145,4 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () { ]) }) }) - - describe('projects=projectId2', function () { - beforeEach('run script', async function () { - const projectId2 = insertedProjects.insertedIds[1] - await runScript('--dry-run=false', `--projects=${projectId2.toString()}`) - }) - - it('should fix only the projects provided', async function () { - const [project1, project2, project3] = await db.projects - .find({}, { $sort: { _id: 1 } }) - .toArray() - expect(project1.tokenAccessReadAndWrite_refs.length).to.be.gt(0) - expect(project2.tokenAccessReadAndWrite_refs.length).to.eq(0) // deleted user removed - expect(project3.tokenAccessReadAndWrite_refs.length).to.be.gt(0) - }) - }) }) diff --git a/services/web/test/acceptance/src/helpers/MongoHelper.mjs b/services/web/test/acceptance/src/helpers/MongoHelper.mjs index fc4a0feda2..0351c09a31 100644 --- a/services/web/test/acceptance/src/helpers/MongoHelper.mjs +++ b/services/web/test/acceptance/src/helpers/MongoHelper.mjs @@ -1,10 +1,11 @@ -import { execFile } from 'node:child_process' +import { exec } from 'node:child_process' import { connectionPromise, cleanupTestDatabase, dropTestDatabase, } from '../../../../app/src/infrastructure/mongodb.js' import Settings from '@overleaf/settings' +import { promisify } from 'node:util' const DEFAULT_ENV = 'saas' @@ -17,21 +18,12 @@ export default { before('drop test database', dropTestDatabase) } - before('run migrations', function (done) { - const args = [ - 'run', - 'migrations', - '--', - 'migrate', - '-t', - Settings.env || DEFAULT_ENV, - ] - execFile('npm', args, (error, stdout, stderr) => { - if (error) { - throw error - } - done() - }) + before('run migrations', async function () { + this.timeout(60_000) + + await promisify(exec)( + `cd ../../tools/migrations && npm run migrations -- migrate -t ${Settings.env || DEFAULT_ENV}` + ) }) afterEach('purge mongo data', cleanupTestDatabase) diff --git a/services/web/.eastrc b/tools/migrations/.eastrc similarity index 60% rename from services/web/.eastrc rename to tools/migrations/.eastrc index ac757c1dd9..e3e1124d70 100644 --- a/services/web/.eastrc +++ b/tools/migrations/.eastrc @@ -1,5 +1,6 @@ { - "adapter": "./migrations/lib/adapter.mjs", + "adapter": "./lib/adapter.mjs", + "dir": ".", "migrationNumberFormat": "dateTime", "migrationExtension": "mjs" } diff --git a/tools/migrations/.jenkinsIncludeFile b/tools/migrations/.jenkinsIncludeFile new file mode 100644 index 0000000000..7fa3cb2325 --- /dev/null +++ b/tools/migrations/.jenkinsIncludeFile @@ -0,0 +1,6 @@ +tools/migrations/** + +package.json +package-lock.json + +Makefile diff --git a/services/web/migrations/20190720165251_create_migrations.mjs b/tools/migrations/20190720165251_create_migrations.mjs similarity index 100% rename from services/web/migrations/20190720165251_create_migrations.mjs rename to tools/migrations/20190720165251_create_migrations.mjs diff --git a/services/web/migrations/20190912145001_create_contacts_indexes.mjs b/tools/migrations/20190912145001_create_contacts_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145001_create_contacts_indexes.mjs rename to tools/migrations/20190912145001_create_contacts_indexes.mjs diff --git a/services/web/migrations/20190912145002_create_deletedProjects_indexes.mjs b/tools/migrations/20190912145002_create_deletedProjects_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145002_create_deletedProjects_indexes.mjs rename to tools/migrations/20190912145002_create_deletedProjects_indexes.mjs diff --git a/services/web/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs b/tools/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs rename to tools/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs diff --git a/services/web/migrations/20190912145004_create_docHistoryIndex_indexes.mjs b/tools/migrations/20190912145004_create_docHistoryIndex_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145004_create_docHistoryIndex_indexes.mjs rename to tools/migrations/20190912145004_create_docHistoryIndex_indexes.mjs diff --git a/services/web/migrations/20190912145005_create_docHistory_indexes.mjs b/tools/migrations/20190912145005_create_docHistory_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145005_create_docHistory_indexes.mjs rename to tools/migrations/20190912145005_create_docHistory_indexes.mjs diff --git a/services/web/migrations/20190912145006_create_docOps_indexes.mjs b/tools/migrations/20190912145006_create_docOps_indexes.mjs similarity index 87% rename from services/web/migrations/20190912145006_create_docOps_indexes.mjs rename to tools/migrations/20190912145006_create_docOps_indexes.mjs index f01bde0151..c05155f569 100644 --- a/services/web/migrations/20190912145006_create_docOps_indexes.mjs +++ b/tools/migrations/20190912145006_create_docOps_indexes.mjs @@ -1,5 +1,5 @@ import Helpers from './lib/helpers.mjs' -import { getCollectionInternal } from '../app/src/infrastructure/mongodb.js' +import { getCollectionInternal } from './lib/mongodb.mjs' const tags = ['server-ce', 'server-pro', 'saas'] diff --git a/services/web/migrations/20190912145007_create_docSnapshots_indexes.mjs b/tools/migrations/20190912145007_create_docSnapshots_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145007_create_docSnapshots_indexes.mjs rename to tools/migrations/20190912145007_create_docSnapshots_indexes.mjs diff --git a/services/web/migrations/20190912145008_create_docs_indexes.mjs b/tools/migrations/20190912145008_create_docs_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145008_create_docs_indexes.mjs rename to tools/migrations/20190912145008_create_docs_indexes.mjs diff --git a/services/web/migrations/20190912145009_create_githubSyncEntityVersions_indexes.mjs b/tools/migrations/20190912145009_create_githubSyncEntityVersions_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145009_create_githubSyncEntityVersions_indexes.mjs rename to tools/migrations/20190912145009_create_githubSyncEntityVersions_indexes.mjs diff --git a/services/web/migrations/20190912145010_create_githubSyncProjectStates_indexes.mjs b/tools/migrations/20190912145010_create_githubSyncProjectStates_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145010_create_githubSyncProjectStates_indexes.mjs rename to tools/migrations/20190912145010_create_githubSyncProjectStates_indexes.mjs diff --git a/services/web/migrations/20190912145011_create_githubSyncUserCredentials_indexes.mjs b/tools/migrations/20190912145011_create_githubSyncUserCredentials_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145011_create_githubSyncUserCredentials_indexes.mjs rename to tools/migrations/20190912145011_create_githubSyncUserCredentials_indexes.mjs diff --git a/services/web/migrations/20190912145012_create_institutions_indexes.mjs b/tools/migrations/20190912145012_create_institutions_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145012_create_institutions_indexes.mjs rename to tools/migrations/20190912145012_create_institutions_indexes.mjs diff --git a/services/web/migrations/20190912145013_create_messages_indexes.mjs b/tools/migrations/20190912145013_create_messages_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145013_create_messages_indexes.mjs rename to tools/migrations/20190912145013_create_messages_indexes.mjs diff --git a/services/web/migrations/20190912145014_create_notifications_indexes.mjs b/tools/migrations/20190912145014_create_notifications_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145014_create_notifications_indexes.mjs rename to tools/migrations/20190912145014_create_notifications_indexes.mjs diff --git a/services/web/migrations/20190912145015_create_oauthAccessTokens_indexes.mjs b/tools/migrations/20190912145015_create_oauthAccessTokens_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145015_create_oauthAccessTokens_indexes.mjs rename to tools/migrations/20190912145015_create_oauthAccessTokens_indexes.mjs diff --git a/services/web/migrations/20190912145016_create_oauthApplications_indexes.mjs b/tools/migrations/20190912145016_create_oauthApplications_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145016_create_oauthApplications_indexes.mjs rename to tools/migrations/20190912145016_create_oauthApplications_indexes.mjs diff --git a/services/web/migrations/20190912145017_create_oauthAuthorizationCodes_indexes.mjs b/tools/migrations/20190912145017_create_oauthAuthorizationCodes_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145017_create_oauthAuthorizationCodes_indexes.mjs rename to tools/migrations/20190912145017_create_oauthAuthorizationCodes_indexes.mjs diff --git a/services/web/migrations/20190912145018_create_projectHistoryFailures_indexes.mjs b/tools/migrations/20190912145018_create_projectHistoryFailures_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145018_create_projectHistoryFailures_indexes.mjs rename to tools/migrations/20190912145018_create_projectHistoryFailures_indexes.mjs diff --git a/services/web/migrations/20190912145019_create_projectHistoryLabels_indexes.mjs b/tools/migrations/20190912145019_create_projectHistoryLabels_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145019_create_projectHistoryLabels_indexes.mjs rename to tools/migrations/20190912145019_create_projectHistoryLabels_indexes.mjs diff --git a/services/web/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs b/tools/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs similarity index 91% rename from services/web/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs rename to tools/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs index 9272afc2e7..895780504f 100644 --- a/services/web/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs +++ b/tools/migrations/20190912145020_create_projectHistoryMetaData_indexes.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const tags = ['saas'] diff --git a/services/web/migrations/20190912145021_create_projectHistorySyncState_indexes.mjs b/tools/migrations/20190912145021_create_projectHistorySyncState_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145021_create_projectHistorySyncState_indexes.mjs rename to tools/migrations/20190912145021_create_projectHistorySyncState_indexes.mjs diff --git a/services/web/migrations/20190912145022_create_projectImportFailures_indexes.mjs b/tools/migrations/20190912145022_create_projectImportFailures_indexes.mjs similarity index 93% rename from services/web/migrations/20190912145022_create_projectImportFailures_indexes.mjs rename to tools/migrations/20190912145022_create_projectImportFailures_indexes.mjs index 9ff7805a1c..418429bbd0 100644 --- a/services/web/migrations/20190912145022_create_projectImportFailures_indexes.mjs +++ b/tools/migrations/20190912145022_create_projectImportFailures_indexes.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['saas'] diff --git a/services/web/migrations/20190912145023_create_projectInvites_indexes.mjs b/tools/migrations/20190912145023_create_projectInvites_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145023_create_projectInvites_indexes.mjs rename to tools/migrations/20190912145023_create_projectInvites_indexes.mjs diff --git a/services/web/migrations/20190912145024_create_projects_indexes.mjs b/tools/migrations/20190912145024_create_projects_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145024_create_projects_indexes.mjs rename to tools/migrations/20190912145024_create_projects_indexes.mjs diff --git a/services/web/migrations/20190912145025_create_publishers_indexes.mjs b/tools/migrations/20190912145025_create_publishers_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145025_create_publishers_indexes.mjs rename to tools/migrations/20190912145025_create_publishers_indexes.mjs diff --git a/services/web/migrations/20190912145026_create_rooms_indexes.mjs b/tools/migrations/20190912145026_create_rooms_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145026_create_rooms_indexes.mjs rename to tools/migrations/20190912145026_create_rooms_indexes.mjs diff --git a/services/web/migrations/20190912145027_create_spellingPreferences_indexes.mjs b/tools/migrations/20190912145027_create_spellingPreferences_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145027_create_spellingPreferences_indexes.mjs rename to tools/migrations/20190912145027_create_spellingPreferences_indexes.mjs diff --git a/services/web/migrations/20190912145028_create_subscriptions_indexes.mjs b/tools/migrations/20190912145028_create_subscriptions_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145028_create_subscriptions_indexes.mjs rename to tools/migrations/20190912145028_create_subscriptions_indexes.mjs diff --git a/services/web/migrations/20190912145029_create_tags_indexes.mjs b/tools/migrations/20190912145029_create_tags_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145029_create_tags_indexes.mjs rename to tools/migrations/20190912145029_create_tags_indexes.mjs diff --git a/services/web/migrations/20190912145030_create_templates_indexes.mjs b/tools/migrations/20190912145030_create_templates_indexes.mjs similarity index 94% rename from services/web/migrations/20190912145030_create_templates_indexes.mjs rename to tools/migrations/20190912145030_create_templates_indexes.mjs index ceaf176e6b..d757d899f7 100644 --- a/services/web/migrations/20190912145030_create_templates_indexes.mjs +++ b/tools/migrations/20190912145030_create_templates_indexes.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['server-pro', 'saas'] diff --git a/services/web/migrations/20190912145031_create_tokens_indexes.mjs b/tools/migrations/20190912145031_create_tokens_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145031_create_tokens_indexes.mjs rename to tools/migrations/20190912145031_create_tokens_indexes.mjs diff --git a/services/web/migrations/20190912145032_create_users_indexes.mjs b/tools/migrations/20190912145032_create_users_indexes.mjs similarity index 100% rename from services/web/migrations/20190912145032_create_users_indexes.mjs rename to tools/migrations/20190912145032_create_users_indexes.mjs diff --git a/services/web/migrations/20190912145033_create_userstubs_indexes.mjs b/tools/migrations/20190912145033_create_userstubs_indexes.mjs similarity index 93% rename from services/web/migrations/20190912145033_create_userstubs_indexes.mjs rename to tools/migrations/20190912145033_create_userstubs_indexes.mjs index 35f6796387..f1006dd9ff 100644 --- a/services/web/migrations/20190912145033_create_userstubs_indexes.mjs +++ b/tools/migrations/20190912145033_create_userstubs_indexes.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['saas'] diff --git a/services/web/migrations/20191106102104_saml-log-indexes.mjs b/tools/migrations/20191106102104_saml-log-indexes.mjs similarity index 95% rename from services/web/migrations/20191106102104_saml-log-indexes.mjs rename to tools/migrations/20191106102104_saml-log-indexes.mjs index a68a3575f8..da403445dd 100644 --- a/services/web/migrations/20191106102104_saml-log-indexes.mjs +++ b/tools/migrations/20191106102104_saml-log-indexes.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['saas'] diff --git a/services/web/migrations/20191107191318_saml-indentifiers-index.mjs b/tools/migrations/20191107191318_saml-indentifiers-index.mjs similarity index 95% rename from services/web/migrations/20191107191318_saml-indentifiers-index.mjs rename to tools/migrations/20191107191318_saml-indentifiers-index.mjs index a39e88d245..ca8c24e29e 100644 --- a/services/web/migrations/20191107191318_saml-indentifiers-index.mjs +++ b/tools/migrations/20191107191318_saml-indentifiers-index.mjs @@ -1,7 +1,7 @@ /* eslint-disable no-unused-vars */ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['saas'] diff --git a/services/web/migrations/20200110183327_brandVarationIdIndex.mjs b/tools/migrations/20200110183327_brandVarationIdIndex.mjs similarity index 100% rename from services/web/migrations/20200110183327_brandVarationIdIndex.mjs rename to tools/migrations/20200110183327_brandVarationIdIndex.mjs diff --git a/services/web/migrations/20200120163346_atlas_recommended_indexes.mjs b/tools/migrations/20200120163346_atlas_recommended_indexes.mjs similarity index 100% rename from services/web/migrations/20200120163346_atlas_recommended_indexes.mjs rename to tools/migrations/20200120163346_atlas_recommended_indexes.mjs diff --git a/services/web/migrations/20200210084301_remove-duplicate-deleted-things.mjs b/tools/migrations/20200210084301_remove-duplicate-deleted-things.mjs similarity index 100% rename from services/web/migrations/20200210084301_remove-duplicate-deleted-things.mjs rename to tools/migrations/20200210084301_remove-duplicate-deleted-things.mjs diff --git a/services/web/migrations/20200210121103_uniqueify-deletedthings-indexes.mjs b/tools/migrations/20200210121103_uniqueify-deletedthings-indexes.mjs similarity index 100% rename from services/web/migrations/20200210121103_uniqueify-deletedthings-indexes.mjs rename to tools/migrations/20200210121103_uniqueify-deletedthings-indexes.mjs diff --git a/services/web/migrations/20200302143624_users_affiliationUnchecked_index.mjs b/tools/migrations/20200302143624_users_affiliationUnchecked_index.mjs similarity index 100% rename from services/web/migrations/20200302143624_users_affiliationUnchecked_index.mjs rename to tools/migrations/20200302143624_users_affiliationUnchecked_index.mjs diff --git a/services/web/migrations/20200522145727_dropProjectImportFailures.mjs b/tools/migrations/20200522145727_dropProjectImportFailures.mjs similarity index 100% rename from services/web/migrations/20200522145727_dropProjectImportFailures.mjs rename to tools/migrations/20200522145727_dropProjectImportFailures.mjs diff --git a/services/web/migrations/20200522145741_dropProjectImportBatchRecords.mjs b/tools/migrations/20200522145741_dropProjectImportBatchRecords.mjs similarity index 100% rename from services/web/migrations/20200522145741_dropProjectImportBatchRecords.mjs rename to tools/migrations/20200522145741_dropProjectImportBatchRecords.mjs diff --git a/services/web/migrations/20200608213302_saml-cache-indexes.mjs b/tools/migrations/20200608213302_saml-cache-indexes.mjs similarity index 100% rename from services/web/migrations/20200608213302_saml-cache-indexes.mjs rename to tools/migrations/20200608213302_saml-cache-indexes.mjs diff --git a/services/web/migrations/20200729120824_update_subscriptions_manager_ids_index.mjs b/tools/migrations/20200729120824_update_subscriptions_manager_ids_index.mjs similarity index 100% rename from services/web/migrations/20200729120824_update_subscriptions_manager_ids_index.mjs rename to tools/migrations/20200729120824_update_subscriptions_manager_ids_index.mjs diff --git a/services/web/migrations/20201106094956_active-projects-index-with-id.mjs b/tools/migrations/20201106094956_active-projects-index-with-id.mjs similarity index 100% rename from services/web/migrations/20201106094956_active-projects-index-with-id.mjs rename to tools/migrations/20201106094956_active-projects-index-with-id.mjs diff --git a/services/web/migrations/20210310111225_create_deletedFiles_projectId_index.mjs b/tools/migrations/20210310111225_create_deletedFiles_projectId_index.mjs similarity index 89% rename from services/web/migrations/20210310111225_create_deletedFiles_projectId_index.mjs rename to tools/migrations/20210310111225_create_deletedFiles_projectId_index.mjs index 57dcb4e21f..bfa70b65cd 100644 --- a/services/web/migrations/20210310111225_create_deletedFiles_projectId_index.mjs +++ b/tools/migrations/20210310111225_create_deletedFiles_projectId_index.mjs @@ -1,5 +1,5 @@ import Helpers from './lib/helpers.mjs' -import { getCollectionInternal } from '../app/src/infrastructure/mongodb.js' +import { getCollectionInternal } from './lib/mongodb.mjs' const tags = ['server-ce', 'server-pro', 'saas'] diff --git a/services/web/migrations/20210407085118_token-expiry-with-ttl-index.mjs b/tools/migrations/20210407085118_token-expiry-with-ttl-index.mjs similarity index 100% rename from services/web/migrations/20210407085118_token-expiry-with-ttl-index.mjs rename to tools/migrations/20210407085118_token-expiry-with-ttl-index.mjs diff --git a/services/web/migrations/20210408123210_create_docs_project_id_deleted_deletedAt_index.mjs b/tools/migrations/20210408123210_create_docs_project_id_deleted_deletedAt_index.mjs similarity index 100% rename from services/web/migrations/20210408123210_create_docs_project_id_deleted_deletedAt_index.mjs rename to tools/migrations/20210408123210_create_docs_project_id_deleted_deletedAt_index.mjs diff --git a/services/web/migrations/20210721081758_create_history_display_index.mjs b/tools/migrations/20210721081758_create_history_display_index.mjs similarity index 100% rename from services/web/migrations/20210721081758_create_history_display_index.mjs rename to tools/migrations/20210721081758_create_history_display_index.mjs diff --git a/services/web/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs b/tools/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs similarity index 93% rename from services/web/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs rename to tools/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs index 064e365a86..9f6facd0d0 100644 --- a/services/web/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs +++ b/tools/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs @@ -1,4 +1,4 @@ -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' const tags = ['saas'] diff --git a/services/web/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs b/tools/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs similarity index 92% rename from services/web/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs rename to tools/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs index b13d636d0f..65322b915c 100644 --- a/services/web/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs +++ b/tools/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs @@ -1,4 +1,4 @@ -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' const tags = ['saas'] diff --git a/services/web/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs b/tools/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs similarity index 100% rename from services/web/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs rename to tools/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs diff --git a/tools/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs b/tools/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs new file mode 100644 index 0000000000..ea47a9418d --- /dev/null +++ b/tools/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs @@ -0,0 +1,50 @@ +import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { db } from './lib/mongodb.mjs' +import { promiseMapWithLimit } from '@overleaf/promise-utils' + +const tags = ['server-ce', 'server-pro', 'saas'] + +const migrate = async client => { + await batchedUpdate( + db.projects, + // array is not empty ~ array has one item + { 'deletedDocs.0': { $exists: true } }, + async projects => { + await processBatch(projects) + }, + { _id: 1, deletedDocs: 1 } + ) +} + +async function processBatch(projects) { + await promiseMapWithLimit(10, projects, async project => { + await processProject(project) + }) +} + +async function processProject(project) { + for (const doc of project.deletedDocs) { + await backFillDoc(doc) + } + await cleanupProject(project) +} + +async function backFillDoc(doc) { + const { name, deletedAt } = doc + await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } }) +} + +async function cleanupProject(project) { + await db.projects.updateOne( + { _id: project._id }, + { $set: { deletedDocs: [] } } + ) +} + +const rollback = async client => {} + +export default { + tags, + migrate, + rollback, +} diff --git a/services/web/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs b/tools/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs similarity index 61% rename from services/web/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs rename to tools/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs index f58b819fc8..af2c5dacfb 100644 --- a/services/web/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs +++ b/tools/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs @@ -1,4 +1,4 @@ -import runScript from '../scripts/back_fill_dummy_doc_meta.mjs' +import runScript from './scripts/back_fill_dummy_doc_meta.mjs' const tags = ['server-ce', 'server-pro', 'saas'] @@ -12,12 +12,7 @@ const migrate = async client => { if (!firstProject) { return } - const options = { - firstProjectId: firstProject._id, - performCleanup: true, - letUserDoubleCheckInputsFor: 10, - } - await runScript(options) + await runScript(firstProject._id) } const rollback = async client => {} diff --git a/services/web/migrations/20210924140139_splittests-name-index.mjs b/tools/migrations/20210924140139_splittests-name-index.mjs similarity index 92% rename from services/web/migrations/20210924140139_splittests-name-index.mjs rename to tools/migrations/20210924140139_splittests-name-index.mjs index e6cdd7cd32..9f8905c476 100644 --- a/services/web/migrations/20210924140139_splittests-name-index.mjs +++ b/tools/migrations/20210924140139_splittests-name-index.mjs @@ -1,5 +1,5 @@ import Helpers from './lib/helpers.mjs' -import mongodb from '../app/src/infrastructure/mongodb.js' +import mongodb from './lib/mongodb.mjs' const { getCollectionInternal } = mongodb const tags = ['saas'] diff --git a/services/web/migrations/20220105123000_cleanup_unused_collections.mjs b/tools/migrations/20220105123000_cleanup_unused_collections.mjs similarity index 100% rename from services/web/migrations/20220105123000_cleanup_unused_collections.mjs rename to tools/migrations/20220105123000_cleanup_unused_collections.mjs diff --git a/services/web/migrations/20220105130000_fix_saml_indexes.mjs b/tools/migrations/20220105130000_fix_saml_indexes.mjs similarity index 100% rename from services/web/migrations/20220105130000_fix_saml_indexes.mjs rename to tools/migrations/20220105130000_fix_saml_indexes.mjs diff --git a/services/web/migrations/20220222095146_split_tests_analytics_enabled.mjs b/tools/migrations/20220222095146_split_tests_analytics_enabled.mjs similarity index 100% rename from services/web/migrations/20220222095146_split_tests_analytics_enabled.mjs rename to tools/migrations/20220222095146_split_tests_analytics_enabled.mjs diff --git a/services/web/migrations/20220811111800_create_dropboxEntities_index.mjs b/tools/migrations/20220811111800_create_dropboxEntities_index.mjs similarity index 100% rename from services/web/migrations/20220811111800_create_dropboxEntities_index.mjs rename to tools/migrations/20220811111800_create_dropboxEntities_index.mjs diff --git a/services/web/migrations/20220815105500_create_dropboxProjects_index.mjs b/tools/migrations/20220815105500_create_dropboxProjects_index.mjs similarity index 100% rename from services/web/migrations/20220815105500_create_dropboxProjects_index.mjs rename to tools/migrations/20220815105500_create_dropboxProjects_index.mjs diff --git a/services/web/migrations/20220817120900_create_dropboxProjects_index.mjs b/tools/migrations/20220817120900_create_dropboxProjects_index.mjs similarity index 100% rename from services/web/migrations/20220817120900_create_dropboxProjects_index.mjs rename to tools/migrations/20220817120900_create_dropboxProjects_index.mjs diff --git a/services/web/migrations/20220825160708_recreate_dropboxEntities.mjs b/tools/migrations/20220825160708_recreate_dropboxEntities.mjs similarity index 100% rename from services/web/migrations/20220825160708_recreate_dropboxEntities.mjs rename to tools/migrations/20220825160708_recreate_dropboxEntities.mjs diff --git a/services/web/migrations/20220826104236_disable_alpha_beta_program.mjs b/tools/migrations/20220826104236_disable_alpha_beta_program.mjs similarity index 100% rename from services/web/migrations/20220826104236_disable_alpha_beta_program.mjs rename to tools/migrations/20220826104236_disable_alpha_beta_program.mjs diff --git a/services/web/migrations/20220830140459_create_index_user_labsProgram_labsProgramGalileo.mjs b/tools/migrations/20220830140459_create_index_user_labsProgram_labsProgramGalileo.mjs similarity index 100% rename from services/web/migrations/20220830140459_create_index_user_labsProgram_labsProgramGalileo.mjs rename to tools/migrations/20220830140459_create_index_user_labsProgram_labsProgramGalileo.mjs diff --git a/services/web/migrations/20220913105500_create_auditLog_indexes.mjs b/tools/migrations/20220913105500_create_auditLog_indexes.mjs similarity index 100% rename from services/web/migrations/20220913105500_create_auditLog_indexes.mjs rename to tools/migrations/20220913105500_create_auditLog_indexes.mjs diff --git a/tools/migrations/20220913125500_migrate_auditLog_to_collections.mjs b/tools/migrations/20220913125500_migrate_auditLog_to_collections.mjs new file mode 100644 index 0000000000..734a79053d --- /dev/null +++ b/tools/migrations/20220913125500_migrate_auditLog_to_collections.mjs @@ -0,0 +1,74 @@ +import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { db } from './lib/mongodb.mjs' + +const tags = ['server-ce', 'server-pro', 'saas'] + +const migrate = async () => { + await batchedUpdate( + db.users, + { auditLog: { $exists: true } }, + async users => { + await processUsersBatch(users) + }, + { _id: 1, auditLog: 1 } + ) + + await batchedUpdate( + db.projects, + { auditLog: { $exists: true } }, + async projects => { + await processProjectsBatch(projects) + }, + { _id: 1, auditLog: 1 } + ) +} + +async function processUsersBatch(users) { + if (!users || users.length <= 0) { + return + } + + const entries = users + .map(user => user.auditLog.map(log => ({ ...log, userId: user._id }))) + .flat() + + if (entries?.length > 0) { + await db.userAuditLogEntries.insertMany(entries) + } + + const userIds = users.map(user => user._id) + await db.users.updateMany( + { _id: { $in: userIds } }, + { $unset: { auditLog: 1 } } + ) +} + +async function processProjectsBatch(projects) { + if (!projects || projects.length <= 0) { + return + } + + const entries = projects + .map(project => + project.auditLog.map(log => ({ ...log, projectId: project._id })) + ) + .flat() + + if (entries?.length > 0) { + await db.projectAuditLogEntries.insertMany(entries) + } + + const projectIds = projects.map(project => project._id) + await db.projects.updateMany( + { _id: { $in: projectIds } }, + { $unset: { auditLog: 1 } } + ) +} + +const rollback = async () => {} + +export default { + tags, + migrate, + rollback, +} diff --git a/services/web/migrations/20220929193200_add_auditLog_indexes.mjs b/tools/migrations/20220929193200_add_auditLog_indexes.mjs similarity index 100% rename from services/web/migrations/20220929193200_add_auditLog_indexes.mjs rename to tools/migrations/20220929193200_add_auditLog_indexes.mjs diff --git a/services/web/migrations/20221111111111_ce_sp_convert_archived_state.mjs b/tools/migrations/20221111111111_ce_sp_convert_archived_state.mjs similarity index 74% rename from services/web/migrations/20221111111111_ce_sp_convert_archived_state.mjs rename to tools/migrations/20221111111111_ce_sp_convert_archived_state.mjs index 284429cd2c..0238eae58a 100644 --- a/services/web/migrations/20221111111111_ce_sp_convert_archived_state.mjs +++ b/tools/migrations/20221111111111_ce_sp_convert_archived_state.mjs @@ -1,21 +1,23 @@ import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' import { promiseMapWithLimit } from '@overleaf/promise-utils' -import { db } from '../app/src/infrastructure/mongodb.js' -import _ from 'lodash' +import { db, ObjectId } from './lib/mongodb.mjs' const tags = ['server-ce', 'server-pro'] const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 function getAllUserIds(project) { - return _.unionWith( - [project.owner_ref], - project.collaberator_refs, - project.readOnly_refs, - project.tokenAccessReadAndWrite_refs, - project.tokenAccessReadOnly_refs, - (a, b) => a.toString() === b.toString() - ) + return Array.from( + new Set( + [ + project.owner_ref, + ...(project.collaberator_refs || []), + ...(project.readOnly_refs || []), + ...(project.tokenAccessReadAndWrite_refs || []), + ...(project.tokenAccessReadOnly_refs || []), + ].map(id => id.toString()) + ) + ).map(id => new ObjectId(id)) } async function migrateField(field) { diff --git a/services/web/migrations/20221122191857_project_history_chunks_indexes.mjs b/tools/migrations/20221122191857_project_history_chunks_indexes.mjs similarity index 100% rename from services/web/migrations/20221122191857_project_history_chunks_indexes.mjs rename to tools/migrations/20221122191857_project_history_chunks_indexes.mjs diff --git a/services/web/migrations/20230110140452_rename_recurly_cached_status.mjs b/tools/migrations/20230110140452_rename_recurly_cached_status.mjs similarity index 100% rename from services/web/migrations/20230110140452_rename_recurly_cached_status.mjs rename to tools/migrations/20230110140452_rename_recurly_cached_status.mjs diff --git a/services/web/migrations/20230124092607_clear_old_2fa_setup.mjs b/tools/migrations/20230124092607_clear_old_2fa_setup.mjs similarity index 92% rename from services/web/migrations/20230124092607_clear_old_2fa_setup.mjs rename to tools/migrations/20230124092607_clear_old_2fa_setup.mjs index 1ebded5d78..b6e6ed73ad 100644 --- a/services/web/migrations/20230124092607_clear_old_2fa_setup.mjs +++ b/tools/migrations/20230124092607_clear_old_2fa_setup.mjs @@ -1,4 +1,4 @@ -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' const tags = ['saas'] diff --git a/services/web/migrations/20230207134844_group_invite_emails_to_lowercase.mjs b/tools/migrations/20230207134844_group_invite_emails_to_lowercase.mjs similarity index 100% rename from services/web/migrations/20230207134844_group_invite_emails_to_lowercase.mjs rename to tools/migrations/20230207134844_group_invite_emails_to_lowercase.mjs diff --git a/services/web/migrations/20230217205311_fix_deleted_history_chunks_index.mjs b/tools/migrations/20230217205311_fix_deleted_history_chunks_index.mjs similarity index 100% rename from services/web/migrations/20230217205311_fix_deleted_history_chunks_index.mjs rename to tools/migrations/20230217205311_fix_deleted_history_chunks_index.mjs diff --git a/services/web/migrations/20230315170739_back_fill_doc_rev.mjs b/tools/migrations/20230315170739_back_fill_doc_rev.mjs similarity index 65% rename from services/web/migrations/20230315170739_back_fill_doc_rev.mjs rename to tools/migrations/20230315170739_back_fill_doc_rev.mjs index c435817d15..ac844fd1a7 100644 --- a/services/web/migrations/20230315170739_back_fill_doc_rev.mjs +++ b/tools/migrations/20230315170739_back_fill_doc_rev.mjs @@ -1,9 +1,9 @@ -import runScript from '../scripts/back_fill_doc_rev.mjs' +import runScript from './scripts/back_fill_doc_rev.mjs' const tags = ['server-ce', 'server-pro'] const migrate = async () => { - await runScript(false) + await runScript() } const rollback = async () => {} diff --git a/services/web/migrations/20230405190240_oauth_refresh_tokens_index.mjs b/tools/migrations/20230405190240_oauth_refresh_tokens_index.mjs similarity index 100% rename from services/web/migrations/20230405190240_oauth_refresh_tokens_index.mjs rename to tools/migrations/20230405190240_oauth_refresh_tokens_index.mjs diff --git a/services/web/migrations/20230406125632_oauth_tokens_ttl.mjs b/tools/migrations/20230406125632_oauth_tokens_ttl.mjs similarity index 100% rename from services/web/migrations/20230406125632_oauth_tokens_ttl.mjs rename to tools/migrations/20230406125632_oauth_tokens_ttl.mjs diff --git a/services/web/migrations/20230426095212_personal_oauth_tokens_user_index.mjs b/tools/migrations/20230426095212_personal_oauth_tokens_user_index.mjs similarity index 100% rename from services/web/migrations/20230426095212_personal_oauth_tokens_user_index.mjs rename to tools/migrations/20230426095212_personal_oauth_tokens_user_index.mjs diff --git a/services/web/migrations/20230428154643_history_chunks_garbage_collection_index.mjs b/tools/migrations/20230428154643_history_chunks_garbage_collection_index.mjs similarity index 100% rename from services/web/migrations/20230428154643_history_chunks_garbage_collection_index.mjs rename to tools/migrations/20230428154643_history_chunks_garbage_collection_index.mjs diff --git a/services/web/migrations/20230502180757_server_pro_oauth_indexes.mjs b/tools/migrations/20230502180757_server_pro_oauth_indexes.mjs similarity index 100% rename from services/web/migrations/20230502180757_server_pro_oauth_indexes.mjs rename to tools/migrations/20230502180757_server_pro_oauth_indexes.mjs diff --git a/services/web/migrations/20230512100122_ensure_history_migration.mjs b/tools/migrations/20230512100122_ensure_history_migration.mjs similarity index 100% rename from services/web/migrations/20230512100122_ensure_history_migration.mjs rename to tools/migrations/20230512100122_ensure_history_migration.mjs diff --git a/services/web/migrations/20230616153016_user_features_updated_at_index.mjs b/tools/migrations/20230616153016_user_features_updated_at_index.mjs similarity index 100% rename from services/web/migrations/20230616153016_user_features_updated_at_index.mjs rename to tools/migrations/20230616153016_user_features_updated_at_index.mjs diff --git a/tools/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs b/tools/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs new file mode 100644 index 0000000000..44668f26d7 --- /dev/null +++ b/tools/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs @@ -0,0 +1,14 @@ +import { db } from './lib/mongodb.mjs' +const tags = ['server-ce', 'server-pro'] + +const migrate = async () => { + await db.users.updateMany({}, { $set: { 'features.gitBridge': true } }) +} + +const rollback = async () => {} + +export default { + tags, + migrate, + rollback, +} diff --git a/services/web/migrations/20230928092537_backfill_subscriptions_managed_users_feature_flag.mjs b/tools/migrations/20230928092537_backfill_subscriptions_managed_users_feature_flag.mjs similarity index 100% rename from services/web/migrations/20230928092537_backfill_subscriptions_managed_users_feature_flag.mjs rename to tools/migrations/20230928092537_backfill_subscriptions_managed_users_feature_flag.mjs diff --git a/services/web/migrations/20231016101457_drop_history_display_index.mjs b/tools/migrations/20231016101457_drop_history_display_index.mjs similarity index 100% rename from services/web/migrations/20231016101457_drop_history_display_index.mjs rename to tools/migrations/20231016101457_drop_history_display_index.mjs diff --git a/services/web/migrations/20231025094810_sso_config_certificates_array.mjs b/tools/migrations/20231025094810_sso_config_certificates_array.mjs similarity index 100% rename from services/web/migrations/20231025094810_sso_config_certificates_array.mjs rename to tools/migrations/20231025094810_sso_config_certificates_array.mjs diff --git a/services/web/migrations/20231030160030_managed_users_enabled.mjs b/tools/migrations/20231030160030_managed_users_enabled.mjs similarity index 100% rename from services/web/migrations/20231030160030_managed_users_enabled.mjs rename to tools/migrations/20231030160030_managed_users_enabled.mjs diff --git a/services/web/migrations/20231031164732_drop_redundant_indexes.mjs b/tools/migrations/20231031164732_drop_redundant_indexes.mjs similarity index 100% rename from services/web/migrations/20231031164732_drop_redundant_indexes.mjs rename to tools/migrations/20231031164732_drop_redundant_indexes.mjs diff --git a/services/web/migrations/20231101153447_partial_last_opened_index.mjs b/tools/migrations/20231101153447_partial_last_opened_index.mjs similarity index 100% rename from services/web/migrations/20231101153447_partial_last_opened_index.mjs rename to tools/migrations/20231101153447_partial_last_opened_index.mjs diff --git a/services/web/migrations/20231101191643_drop_unused_indexes.mjs b/tools/migrations/20231101191643_drop_unused_indexes.mjs similarity index 100% rename from services/web/migrations/20231101191643_drop_unused_indexes.mjs rename to tools/migrations/20231101191643_drop_unused_indexes.mjs diff --git a/services/web/migrations/20231101204352_drop_unused_saas_indexes.mjs b/tools/migrations/20231101204352_drop_unused_saas_indexes.mjs similarity index 100% rename from services/web/migrations/20231101204352_drop_unused_saas_indexes.mjs rename to tools/migrations/20231101204352_drop_unused_saas_indexes.mjs diff --git a/services/web/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs b/tools/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs similarity index 92% rename from services/web/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs rename to tools/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs index acb974ffee..75977ce79c 100644 --- a/services/web/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs +++ b/tools/migrations/20231105000000_move_doc_versions_from_docops_to_docs.mjs @@ -1,5 +1,5 @@ -import mongodbLegacy from 'mongodb-legacy' -import { db, getCollectionInternal } from '../app/src/infrastructure/mongodb.js' +import mongodbLegacy from 'mongodb' +import { db, getCollectionInternal } from './lib/mongodb.mjs' const { ObjectId, ReadPreference } = mongodbLegacy const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '1000', 10) diff --git a/services/web/migrations/20231110192205_drop_docops_collection.mjs b/tools/migrations/20231110192205_drop_docops_collection.mjs similarity index 100% rename from services/web/migrations/20231110192205_drop_docops_collection.mjs rename to tools/migrations/20231110192205_drop_docops_collection.mjs diff --git a/services/web/migrations/20231113173237_tokens_user_id_index.mjs b/tools/migrations/20231113173237_tokens_user_id_index.mjs similarity index 100% rename from services/web/migrations/20231113173237_tokens_user_id_index.mjs rename to tools/migrations/20231113173237_tokens_user_id_index.mjs diff --git a/services/web/migrations/20240115172206_oauth_issuer_index.mjs b/tools/migrations/20240115172206_oauth_issuer_index.mjs similarity index 100% rename from services/web/migrations/20240115172206_oauth_issuer_index.mjs rename to tools/migrations/20240115172206_oauth_issuer_index.mjs diff --git a/services/web/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs b/tools/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs similarity index 50% rename from services/web/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs rename to tools/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs index 25292a7ea9..7a21b8d2e8 100644 --- a/services/web/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs +++ b/tools/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs @@ -1,9 +1,9 @@ -import runScript from '../scripts/remove_deleted_users_from_token_access_refs.mjs' +import fixProjectsWithInvalidTokenAccessRefsIds from './scripts/remove_deleted_users_from_token_access_refs.mjs' const tags = ['server-ce', 'server-pro', 'saas'] const migrate = async () => { - await runScript(false) + await fixProjectsWithInvalidTokenAccessRefsIds() } const rollback = async () => {} diff --git a/tools/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs b/tools/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs new file mode 100644 index 0000000000..a32cadabde --- /dev/null +++ b/tools/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs @@ -0,0 +1,53 @@ +/* eslint-disable no-unused-vars */ + +import Helpers from './lib/helpers.mjs' +import Crypto from 'node:crypto' +import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' + +const tags = ['server-ce', 'server-pro', 'saas'] + +// Copied from services/web/app/src/Features/Collaborators/CollaboratorsInviteHelper.js +function hashInviteToken(token) { + return Crypto.createHmac('sha256', 'overleaf-token-invite') + .update(token) + .digest('hex') +} + +const index = { + key: { + tokenHmac: 1, + }, + name: 'tokenHmac_1', +} + +const migrate = async client => { + const { db } = client + await Helpers.addIndexesToCollection(db.projectInvites, [index]) + + await batchedUpdate( + db.projectInvites, + { tokenHmac: { $exists: false } }, + async invites => { + for (const invite of invites) { + const tokenHmac = hashInviteToken(invite.token) + + await db.projectInvites.updateOne( + { _id: invite._id }, + { $set: { tokenHmac } } + ) + } + }, + { token: 1 } + ) +} + +const rollback = async client => { + const { db } = client + await Helpers.dropIndexesFromCollection(db.projectInvites, [index]) +} + +export default { + tags, + migrate, + rollback, +} diff --git a/services/web/migrations/20240531082910_remove_project_invite_tokens.mjs b/tools/migrations/20240531082910_remove_project_invite_tokens.mjs similarity index 100% rename from services/web/migrations/20240531082910_remove_project_invite_tokens.mjs rename to tools/migrations/20240531082910_remove_project_invite_tokens.mjs diff --git a/services/web/migrations/20240618125145_cleanup_user_features_templates.mjs b/tools/migrations/20240618125145_cleanup_user_features_templates.mjs similarity index 86% rename from services/web/migrations/20240618125145_cleanup_user_features_templates.mjs rename to tools/migrations/20240618125145_cleanup_user_features_templates.mjs index 50950209eb..d98facad72 100644 --- a/services/web/migrations/20240618125145_cleanup_user_features_templates.mjs +++ b/tools/migrations/20240618125145_cleanup_user_features_templates.mjs @@ -1,4 +1,4 @@ -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' const tags = ['server-ce', 'server-pro', 'saas'] diff --git a/services/web/migrations/20240625101055_add_ai_policy_to_group_policy.mjs b/tools/migrations/20240625101055_add_ai_policy_to_group_policy.mjs similarity index 100% rename from services/web/migrations/20240625101055_add_ai_policy_to_group_policy.mjs rename to tools/migrations/20240625101055_add_ai_policy_to_group_policy.mjs diff --git a/services/web/migrations/202407131109055_admin_reversed_hostname.mjs b/tools/migrations/202407131109055_admin_reversed_hostname.mjs similarity index 100% rename from services/web/migrations/202407131109055_admin_reversed_hostname.mjs rename to tools/migrations/202407131109055_admin_reversed_hostname.mjs diff --git a/services/web/migrations/20240730155209_create_project_imageName_index.mjs b/tools/migrations/20240730155209_create_project_imageName_index.mjs similarity index 100% rename from services/web/migrations/20240730155209_create_project_imageName_index.mjs rename to tools/migrations/20240730155209_create_project_imageName_index.mjs diff --git a/services/web/migrations/20240923131936_create_user_last_active_index.mjs b/tools/migrations/20240923131936_create_user_last_active_index.mjs similarity index 100% rename from services/web/migrations/20240923131936_create_user_last_active_index.mjs rename to tools/migrations/20240923131936_create_user_last_active_index.mjs diff --git a/services/web/migrations/20241002180623_drop_unused_collections.mjs b/tools/migrations/20241002180623_drop_unused_collections.mjs similarity index 100% rename from services/web/migrations/20241002180623_drop_unused_collections.mjs rename to tools/migrations/20241002180623_drop_unused_collections.mjs diff --git a/services/web/migrations/20241111133330_remove_null_managed_users_sso_from_subscriptions.mjs b/tools/migrations/20241111133330_remove_null_managed_users_sso_from_subscriptions.mjs similarity index 100% rename from services/web/migrations/20241111133330_remove_null_managed_users_sso_from_subscriptions.mjs rename to tools/migrations/20241111133330_remove_null_managed_users_sso_from_subscriptions.mjs diff --git a/services/web/migrations/20241204103349_create_reviewer_refs_index.mjs b/tools/migrations/20241204103349_create_reviewer_refs_index.mjs similarity index 100% rename from services/web/migrations/20241204103349_create_reviewer_refs_index.mjs rename to tools/migrations/20241204103349_create_reviewer_refs_index.mjs diff --git a/services/web/migrations/20250115110745_oauth_user_id_index_no_partial.mjs b/tools/migrations/20250115110745_oauth_user_id_index_no_partial.mjs similarity index 100% rename from services/web/migrations/20250115110745_oauth_user_id_index_no_partial.mjs rename to tools/migrations/20250115110745_oauth_user_id_index_no_partial.mjs diff --git a/services/web/migrations/20250116090000_onboardingDataCollection_updatedAt_index.mjs b/tools/migrations/20250116090000_onboardingDataCollection_updatedAt_index.mjs similarity index 100% rename from services/web/migrations/20250116090000_onboardingDataCollection_updatedAt_index.mjs rename to tools/migrations/20250116090000_onboardingDataCollection_updatedAt_index.mjs diff --git a/services/web/migrations/20250116090001_deletedUsers_non_expired_index.mjs b/tools/migrations/20250116090001_deletedUsers_non_expired_index.mjs similarity index 100% rename from services/web/migrations/20250116090001_deletedUsers_non_expired_index.mjs rename to tools/migrations/20250116090001_deletedUsers_non_expired_index.mjs diff --git a/services/web/migrations/20250121114712_add_chat_policy_to_group_policy.mjs b/tools/migrations/20250121114712_add_chat_policy_to_group_policy.mjs similarity index 100% rename from services/web/migrations/20250121114712_add_chat_policy_to_group_policy.mjs rename to tools/migrations/20250121114712_add_chat_policy_to_group_policy.mjs diff --git a/services/web/migrations/20250126195914_update_deleterData.deletedAt_index.mjs b/tools/migrations/20250126195914_update_deleterData.deletedAt_index.mjs similarity index 100% rename from services/web/migrations/20250126195914_update_deleterData.deletedAt_index.mjs rename to tools/migrations/20250126195914_update_deleterData.deletedAt_index.mjs diff --git a/services/web/migrations/20250130104049_create_dropboxEntities_userId_projectId_index.mjs b/tools/migrations/20250130104049_create_dropboxEntities_userId_projectId_index.mjs similarity index 100% rename from services/web/migrations/20250130104049_create_dropboxEntities_userId_projectId_index.mjs rename to tools/migrations/20250130104049_create_dropboxEntities_userId_projectId_index.mjs diff --git a/services/web/migrations/20250203164153_create_project_pendingChangeAt_index.mjs b/tools/migrations/20250203164153_create_project_pendingChangeAt_index.mjs similarity index 100% rename from services/web/migrations/20250203164153_create_project_pendingChangeAt_index.mjs rename to tools/migrations/20250203164153_create_project_pendingChangeAt_index.mjs diff --git a/services/web/migrations/20250205145327_drop_rooms_projectId_index.mjs b/tools/migrations/20250205145327_drop_rooms_projectId_index.mjs similarity index 100% rename from services/web/migrations/20250205145327_drop_rooms_projectId_index.mjs rename to tools/migrations/20250205145327_drop_rooms_projectId_index.mjs diff --git a/services/web/migrations/20250206103037_create_rooms_projectId_threadId_index.mjs b/tools/migrations/20250206103037_create_rooms_projectId_threadId_index.mjs similarity index 100% rename from services/web/migrations/20250206103037_create_rooms_projectId_threadId_index.mjs rename to tools/migrations/20250206103037_create_rooms_projectId_threadId_index.mjs diff --git a/services/web/migrations/20250212144722_clear_history_metadata.mjs b/tools/migrations/20250212144722_clear_history_metadata.mjs similarity index 100% rename from services/web/migrations/20250212144722_clear_history_metadata.mjs rename to tools/migrations/20250212144722_clear_history_metadata.mjs diff --git a/services/web/migrations/20250307120446_create_project_lastBackedUpVersion_index.mjs b/tools/migrations/20250307120446_create_project_lastBackedUpVersion_index.mjs similarity index 100% rename from services/web/migrations/20250307120446_create_project_lastBackedUpVersion_index.mjs rename to tools/migrations/20250307120446_create_project_lastBackedUpVersion_index.mjs diff --git a/services/web/migrations/20250318144744_add_subscription_and_deleted_subscription_indexes.mjs b/tools/migrations/20250318144744_add_subscription_and_deleted_subscription_indexes.mjs similarity index 100% rename from services/web/migrations/20250318144744_add_subscription_and_deleted_subscription_indexes.mjs rename to tools/migrations/20250318144744_add_subscription_and_deleted_subscription_indexes.mjs diff --git a/services/web/migrations/20250319110212_update_saml_logs_indexes.mjs b/tools/migrations/20250319110212_update_saml_logs_indexes.mjs similarity index 100% rename from services/web/migrations/20250319110212_update_saml_logs_indexes.mjs rename to tools/migrations/20250319110212_update_saml_logs_indexes.mjs diff --git a/services/web/migrations/20250320161029_update_inactive_project_index.mjs b/tools/migrations/20250320161029_update_inactive_project_index.mjs similarity index 100% rename from services/web/migrations/20250320161029_update_inactive_project_index.mjs rename to tools/migrations/20250320161029_update_inactive_project_index.mjs diff --git a/services/web/migrations/20250321092555_add_deleted_to_project_id_in_s3_index.mjs b/tools/migrations/20250321092555_add_deleted_to_project_id_in_s3_index.mjs similarity index 100% rename from services/web/migrations/20250321092555_add_deleted_to_project_id_in_s3_index.mjs rename to tools/migrations/20250321092555_add_deleted_to_project_id_in_s3_index.mjs diff --git a/services/web/migrations/20250321094339_update_saml_identifiers_indexes.mjs b/tools/migrations/20250321094339_update_saml_identifiers_indexes.mjs similarity index 100% rename from services/web/migrations/20250321094339_update_saml_identifiers_indexes.mjs rename to tools/migrations/20250321094339_update_saml_identifiers_indexes.mjs diff --git a/services/web/migrations/20250321112055_create_deletedUsers_email_indexes.mjs b/tools/migrations/20250321112055_create_deletedUsers_email_indexes.mjs similarity index 100% rename from services/web/migrations/20250321112055_create_deletedUsers_email_indexes.mjs rename to tools/migrations/20250321112055_create_deletedUsers_email_indexes.mjs diff --git a/services/web/migrations/20250321135735_add_email_to_emails_reversed_hostname_index.mjs b/tools/migrations/20250321135735_add_email_to_emails_reversed_hostname_index.mjs similarity index 100% rename from services/web/migrations/20250321135735_add_email_to_emails_reversed_hostname_index.mjs rename to tools/migrations/20250321135735_add_email_to_emails_reversed_hostname_index.mjs diff --git a/services/web/migrations/20250321160345_add_type_to_pat_user_id_index.mjs b/tools/migrations/20250321160345_add_type_to_pat_user_id_index.mjs similarity index 100% rename from services/web/migrations/20250321160345_add_type_to_pat_user_id_index.mjs rename to tools/migrations/20250321160345_add_type_to_pat_user_id_index.mjs diff --git a/services/web/migrations/20250331120946_add_email_to_teamInvites_index.mjs b/tools/migrations/20250331120946_add_email_to_teamInvites_index.mjs similarity index 100% rename from services/web/migrations/20250331120946_add_email_to_teamInvites_index.mjs rename to tools/migrations/20250331120946_add_email_to_teamInvites_index.mjs diff --git a/services/web/migrations/20250403133427_create_index_for_script_logs.mjs b/tools/migrations/20250403133427_create_index_for_script_logs.mjs similarity index 100% rename from services/web/migrations/20250403133427_create_index_for_script_logs.mjs rename to tools/migrations/20250403133427_create_index_for_script_logs.mjs diff --git a/services/web/migrations/20250409155536_group_audit_log_index.mjs b/tools/migrations/20250409155536_group_audit_log_index.mjs similarity index 100% rename from services/web/migrations/20250409155536_group_audit_log_index.mjs rename to tools/migrations/20250409155536_group_audit_log_index.mjs diff --git a/services/web/migrations/20250411200550_active_chunk_index_update.mjs b/tools/migrations/20250411200550_active_chunk_index_update.mjs similarity index 100% rename from services/web/migrations/20250411200550_active_chunk_index_update.mjs rename to tools/migrations/20250411200550_active_chunk_index_update.mjs diff --git a/services/web/migrations/20250519101127_drop_deletedFiles.mjs b/tools/migrations/20250519101127_drop_deletedFiles.mjs similarity index 93% rename from services/web/migrations/20250519101127_drop_deletedFiles.mjs rename to tools/migrations/20250519101127_drop_deletedFiles.mjs index 87f63f4405..c2d8dea496 100644 --- a/services/web/migrations/20250519101127_drop_deletedFiles.mjs +++ b/tools/migrations/20250519101127_drop_deletedFiles.mjs @@ -1,5 +1,5 @@ import Helpers from './lib/helpers.mjs' -import { getCollectionInternal, db } from '../app/src/infrastructure/mongodb.js' +import { getCollectionInternal, db } from './lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' const tags = ['server-ce', 'server-pro', 'saas'] diff --git a/services/web/migrations/20250519101128_binary_files_migration_check.mjs b/tools/migrations/20250519101128_binary_files_migration_check.mjs similarity index 95% rename from services/web/migrations/20250519101128_binary_files_migration_check.mjs rename to tools/migrations/20250519101128_binary_files_migration_check.mjs index 9720448b16..a54be208da 100644 --- a/services/web/migrations/20250519101128_binary_files_migration_check.mjs +++ b/tools/migrations/20250519101128_binary_files_migration_check.mjs @@ -1,5 +1,5 @@ import { setTimeout } from 'node:timers/promises' -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' import Helpers from './lib/helpers.mjs' const tags = ['server-ce', 'server-pro'] diff --git a/services/web/migrations/20250528141310_drop_projectHistoryMetaData_collection.mjs b/tools/migrations/20250528141310_drop_projectHistoryMetaData_collection.mjs similarity index 100% rename from services/web/migrations/20250528141310_drop_projectHistoryMetaData_collection.mjs rename to tools/migrations/20250528141310_drop_projectHistoryMetaData_collection.mjs diff --git a/services/web/migrations/20250604112908_add_dropbox_policy_to_group_policy.mjs b/tools/migrations/20250604112908_add_dropbox_policy_to_group_policy.mjs similarity index 100% rename from services/web/migrations/20250604112908_add_dropbox_policy_to_group_policy.mjs rename to tools/migrations/20250604112908_add_dropbox_policy_to_group_policy.mjs diff --git a/services/web/migrations/20250620152657_ensure_collaborator_arrays.mjs b/tools/migrations/20250620152657_ensure_collaborator_arrays.mjs similarity index 90% rename from services/web/migrations/20250620152657_ensure_collaborator_arrays.mjs rename to tools/migrations/20250620152657_ensure_collaborator_arrays.mjs index 3caa210d41..47aa1a6093 100644 --- a/services/web/migrations/20250620152657_ensure_collaborator_arrays.mjs +++ b/tools/migrations/20250620152657_ensure_collaborator_arrays.mjs @@ -1,5 +1,5 @@ import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from './lib/mongodb.mjs' const tags = ['server-ce', 'server-pro', 'saas'] diff --git a/services/web/migrations/20250702203054_recurly_subscription_id_index.mjs b/tools/migrations/20250702203054_recurly_subscription_id_index.mjs similarity index 100% rename from services/web/migrations/20250702203054_recurly_subscription_id_index.mjs rename to tools/migrations/20250702203054_recurly_subscription_id_index.mjs diff --git a/services/web/migrations/20250728103841_group_audit_log_expire_index.mjs b/tools/migrations/20250728103841_group_audit_log_expire_index.mjs similarity index 100% rename from services/web/migrations/20250728103841_group_audit_log_expire_index.mjs rename to tools/migrations/20250728103841_group_audit_log_expire_index.mjs diff --git a/services/web/migrations/20250827155732_optimise_lastBackedUpVersion_index.mjs b/tools/migrations/20250827155732_optimise_lastBackedUpVersion_index.mjs similarity index 100% rename from services/web/migrations/20250827155732_optimise_lastBackedUpVersion_index.mjs rename to tools/migrations/20250827155732_optimise_lastBackedUpVersion_index.mjs diff --git a/services/web/migrations/20250828095305_create_subscription_v1_id_index.mjs b/tools/migrations/20250828095305_create_subscription_v1_id_index.mjs similarity index 100% rename from services/web/migrations/20250828095305_create_subscription_v1_id_index.mjs rename to tools/migrations/20250828095305_create_subscription_v1_id_index.mjs diff --git a/services/web/migrations/20250828100935_drop_old_lastBackedUpVersion_index.mjs b/tools/migrations/20250828100935_drop_old_lastBackedUpVersion_index.mjs similarity index 100% rename from services/web/migrations/20250828100935_drop_old_lastBackedUpVersion_index.mjs rename to tools/migrations/20250828100935_drop_old_lastBackedUpVersion_index.mjs diff --git a/services/web/migrations/20250912102601_add_timestamp_to_projectAuditLogEntries_index.mjs b/tools/migrations/20250912102601_add_timestamp_to_projectAuditLogEntries_index.mjs similarity index 100% rename from services/web/migrations/20250912102601_add_timestamp_to_projectAuditLogEntries_index.mjs rename to tools/migrations/20250912102601_add_timestamp_to_projectAuditLogEntries_index.mjs diff --git a/services/web/migrations/20250922141050_project_history_sizes_indexes.mjs b/tools/migrations/20250922141050_project_history_sizes_indexes.mjs similarity index 100% rename from services/web/migrations/20250922141050_project_history_sizes_indexes.mjs rename to tools/migrations/20250922141050_project_history_sizes_indexes.mjs diff --git a/tools/migrations/Jenkinsfile b/tools/migrations/Jenkinsfile new file mode 100644 index 0000000000..eb064b9513 --- /dev/null +++ b/tools/migrations/Jenkinsfile @@ -0,0 +1,64 @@ +pipeline { + agent { + node { + label 'jenkins-agent-web' + customWorkspace '/workspace' + } + } + options { + timestamps() + parallelsAlwaysFailFast() + timeout(time: 15, unit: 'MINUTES') + } + environment { + BRANCH_NAME = "${env.CHANGE_BRANCH ? env.CHANGE_BRANCH : env.BRANCH_NAME}" + JENKINS_BUILD_NUMBER = "${BUILD_NUMBER}" + BUILD_NUMBER = "${SHORT_SHA}_${BUILD_NUMBER}" + COMMIT_SHA = "${GIT_COMMIT}" + SHORT_SHA = "${GIT_COMMIT.take(7)}" + } + stages { + stage('Stage 1') { + parallel { + stage('Install monorepo') { + steps { + sh 'make monorepo_setup' + } + } + stage('Create reports folder') { + steps { + sh 'mkdir tools/migrations/reports' + } + } + } + } + stage('Stage 2') { + parallel { + stage('Lint') { + steps { + sh 'bin/run -w /overleaf/tools/migrations monorepo npm run lint -- --format json --output-file reports/eslint.json' + } + post { + always { + sh """ + sed -i 's_"filePath":"/overleaf_"filePath":"/workspace_g' tools/migrations/reports/eslint.json + """ + recordIssues checksAnnotationScope: 'ALL', enabledForFailure: true, failOnError: true, id: 'migrations-eslint', name: 'migrations eslint', qualityGates: [[integerThreshold: 1, threshold: 1.0, type: 'TOTAL']], sourceCodeRetention: 'LAST_BUILD', tools: [esLint(pattern: 'tools/migrations/reports/eslint.json')] + } + } + } + stage('Format') { + steps { + sh 'bin/run -w /overleaf/tools/migrations monorepo npm run format' + } + } + } + } + } + post { + cleanup { + sh 'rm -rf tools/migrations/reports' + sh 'make clean_jenkins' + } + } +} \ No newline at end of file diff --git a/services/web/migrations/README.md b/tools/migrations/README.md similarity index 100% rename from services/web/migrations/README.md rename to tools/migrations/README.md diff --git a/tools/migrations/config/settings.defaults.js b/tools/migrations/config/settings.defaults.js new file mode 100644 index 0000000000..245cfd1acd --- /dev/null +++ b/tools/migrations/config/settings.defaults.js @@ -0,0 +1,21 @@ +module.exports = { + mongo: { + options: { + appname: 'migrations', + maxPoolSize: parseInt(process.env.MONGO_POOL_SIZE, 10) || 100, + serverSelectionTimeoutMS: + parseInt(process.env.MONGO_SERVER_SELECTION_TIMEOUT, 10) || 60000, + // Setting socketTimeoutMS to 0 means no timeout + socketTimeoutMS: parseInt( + process.env.MONGO_SOCKET_TIMEOUT ?? '60000', + 10 + ), + monitorCommands: true, + }, + url: + process.env.MONGO_CONNECTION_STRING || + process.env.MONGO_URL || + `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`, + hasSecondaries: process.env.MONGO_HAS_SECONDARIES === 'true', + }, +} diff --git a/services/web/migrations/lib/template.mjs b/tools/migrations/lib/20000000000000_template.mjs similarity index 100% rename from services/web/migrations/lib/template.mjs rename to tools/migrations/lib/20000000000000_template.mjs diff --git a/services/web/migrations/lib/adapter.mjs b/tools/migrations/lib/adapter.mjs similarity index 84% rename from services/web/migrations/lib/adapter.mjs rename to tools/migrations/lib/adapter.mjs index 0f4cecdd05..1b49796862 100644 --- a/services/web/migrations/lib/adapter.mjs +++ b/tools/migrations/lib/adapter.mjs @@ -1,5 +1,5 @@ import Path from 'node:path' -import { db } from '../../app/src/infrastructure/mongodb.js' +import { db } from './mongodb.mjs' import { fileURLToPath } from 'node:url' const __filename = fileURLToPath(import.meta.url) @@ -19,7 +19,7 @@ class Adapter { } getTemplatePath() { - return Path.resolve(__dirname, 'template.mjs') + return Path.resolve(__dirname, '20000000000000_template.mjs') } async connect() { @@ -38,14 +38,14 @@ class Adapter { } async markExecuted(name) { - return db.migrations.insertOne({ + return await db.migrations.insertOne({ name, migratedAt: new Date(), }) } async unmarkExecuted(name) { - return db.migrations.deleteOne({ + return await db.migrations.deleteOne({ name, }) } diff --git a/services/web/migrations/lib/helpers.mjs b/tools/migrations/lib/helpers.mjs similarity index 88% rename from services/web/migrations/lib/helpers.mjs rename to tools/migrations/lib/helpers.mjs index f342ee333e..ab39a74a93 100644 --- a/services/web/migrations/lib/helpers.mjs +++ b/tools/migrations/lib/helpers.mjs @@ -1,14 +1,10 @@ // @ts-check -import { - db, - getCollectionNames, - getCollectionInternal, -} from '../../app/src/infrastructure/mongodb.js' +import { db, getCollectionNames, getCollectionInternal } from './mongodb.mjs' /** - * @typedef {import('mongodb-legacy').Document} Collection - * @typedef {import('mongodb-legacy').Collection} Collection + * @typedef {import('mongodb').Document} Document + * @typedef {import('mongodb').Collection} Collection */ /** diff --git a/tools/migrations/lib/mongodb.mjs b/tools/migrations/lib/mongodb.mjs new file mode 100644 index 0000000000..993794463c --- /dev/null +++ b/tools/migrations/lib/mongodb.mjs @@ -0,0 +1,99 @@ +import { ObjectId, ReadPreference, MongoClient } from 'mongodb' +import Settings from '@overleaf/settings' + +export { ObjectId } from 'mongodb' + +export const READ_PREFERENCE_PRIMARY = ReadPreference.primary.mode +export const READ_PREFERENCE_SECONDARY = Settings.mongo.hasSecondaries + ? ReadPreference.secondary.mode + : ReadPreference.secondaryPreferred.mode + +const mongoClient = new MongoClient(Settings.mongo.url, Settings.mongo.options) + +const internalDb = mongoClient.db() +export const db = { + contacts: internalDb.collection('contacts'), + deletedProjects: internalDb.collection('deletedProjects'), + deletedSubscriptions: internalDb.collection('deletedSubscriptions'), + deletedUsers: internalDb.collection('deletedUsers'), + dropboxEntities: internalDb.collection('dropboxEntities'), + dropboxProjects: internalDb.collection('dropboxProjects'), + docHistory: internalDb.collection('docHistory'), + docHistoryIndex: internalDb.collection('docHistoryIndex'), + docSnapshots: internalDb.collection('docSnapshots'), + docs: internalDb.collection('docs'), + feedbacks: internalDb.collection('feedbacks'), + githubSyncEntityVersions: internalDb.collection('githubSyncEntityVersions'), + githubSyncProjectStates: internalDb.collection('githubSyncProjectStates'), + githubSyncUserCredentials: internalDb.collection('githubSyncUserCredentials'), + globalMetrics: internalDb.collection('globalMetrics'), + grouppolicies: internalDb.collection('grouppolicies'), + groupAuditLogEntries: internalDb.collection('groupAuditLogEntries'), + institutions: internalDb.collection('institutions'), + messages: internalDb.collection('messages'), + migrations: internalDb.collection('migrations'), + notifications: internalDb.collection('notifications'), + oauthAccessTokens: internalDb.collection('oauthAccessTokens'), + oauthApplications: internalDb.collection('oauthApplications'), + oauthAuthorizationCodes: internalDb.collection('oauthAuthorizationCodes'), + projectAuditLogEntries: internalDb.collection('projectAuditLogEntries'), + projectHistoryChunks: internalDb.collection('projectHistoryChunks'), + projectHistoryFailures: internalDb.collection('projectHistoryFailures'), + projectHistoryGlobalBlobs: internalDb.collection('projectHistoryGlobalBlobs'), + projectHistoryLabels: internalDb.collection('projectHistoryLabels'), + projectHistorySizes: internalDb.collection('projectHistorySizes'), + projectHistorySyncState: internalDb.collection('projectHistorySyncState'), + projectInvites: internalDb.collection('projectInvites'), + projects: internalDb.collection('projects'), + publishers: internalDb.collection('publishers'), + rooms: internalDb.collection('rooms'), + samlCache: internalDb.collection('samlCache'), + samlLogs: internalDb.collection('samlLogs'), + spellingPreferences: internalDb.collection('spellingPreferences'), + splittests: internalDb.collection('splittests'), + ssoConfigs: internalDb.collection('ssoConfigs'), + subscriptions: internalDb.collection('subscriptions'), + surveys: internalDb.collection('surveys'), + systemmessages: internalDb.collection('systemmessages'), + tags: internalDb.collection('tags'), + teamInvites: internalDb.collection('teamInvites'), + tokens: internalDb.collection('tokens'), + userAuditLogEntries: internalDb.collection('userAuditLogEntries'), + users: internalDb.collection('users'), + onboardingDataCollection: internalDb.collection('onboardingDataCollection'), + scriptLogs: internalDb.collection('scriptLogs'), +} + +export const connectionPromise = mongoClient.connect() + +export async function getCollectionNames() { + const internalDb = mongoClient.db() + + const collections = await internalDb.collections() + return collections.map(collection => collection.collectionName) +} + +/** + * WARNING: Consider using a pre-populated collection from `db` to avoid typos! + */ +export async function getCollectionInternal(name) { + const internalDb = mongoClient.db() + return internalDb.collection(name) +} + +export async function waitForDb() { + await connectionPromise +} + +const mongodb = { + db, + ObjectId, + connectionPromise, + waitForDb, + getCollectionNames, + getCollectionInternal, + READ_PREFERENCE_PRIMARY, + READ_PREFERENCE_SECONDARY, +} + +export default mongodb diff --git a/tools/migrations/package.json b/tools/migrations/package.json new file mode 100644 index 0000000000..7368dfeeda --- /dev/null +++ b/tools/migrations/package.json @@ -0,0 +1,19 @@ +{ + "name": "@overleaf/migrations", + "scripts": { + "migrations": "MONGO_SOCKET_TIMEOUT=0 east --es-modules", + "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .", + "format": "prettier --list-different $PWD/'**/*.{mjs,js,cjs,ts}'", + "format:fix": "prettier --write $PWD/'**/*.{mjs,js,cjs,ts}'" + }, + "dependencies": { + "@overleaf/logger": "*", + "@overleaf/mongo-utils": "*", + "@overleaf/o-error": "*", + "@overleaf/promise-utils": "*", + "@overleaf/settings": "*", + "east": "2.0.3", + "mongodb": "6.12.0" + } +} diff --git a/services/web/scripts/back_fill_doc_rev.mjs b/tools/migrations/scripts/back_fill_doc_rev.mjs similarity index 52% rename from services/web/scripts/back_fill_doc_rev.mjs rename to tools/migrations/scripts/back_fill_doc_rev.mjs index c80e4a9c62..cc11f56850 100644 --- a/services/web/scripts/back_fill_doc_rev.mjs +++ b/tools/migrations/scripts/back_fill_doc_rev.mjs @@ -1,11 +1,9 @@ -import { db } from '../app/src/infrastructure/mongodb.js' +import { db } from '../lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import { fileURLToPath } from 'node:url' -const DRY_RUN = !process.argv.includes('--dry-run=false') const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S, 10) || 5 -async function main(DRY_RUN) { +async function main() { let processed = 0 let deleted = 0 let lastLog = 0 @@ -18,15 +16,13 @@ async function main(DRY_RUN) { db.docs, { rev: { $exists: false } }, async docs => { - if (!DRY_RUN) { - await db.docs.updateMany( - { - _id: { $in: docs.map(doc => doc._id) }, - rev: { $exists: false }, - }, - { $set: { rev: 1 } } - ) - } + await db.docs.updateMany( + { + _id: { $in: docs.map(doc => doc._id) }, + rev: { $exists: false }, + }, + { $set: { rev: 1 } } + ) processed += docs.length deleted += docs.filter(doc => doc.deleted).length @@ -46,14 +42,3 @@ async function main(DRY_RUN) { } export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - try { - await main(DRY_RUN) - console.log('Done.') - process.exit(0) - } catch (error) { - console.error({ error }) - process.exit(1) - } -} diff --git a/services/web/scripts/back_fill_dummy_doc_meta.mjs b/tools/migrations/scripts/back_fill_dummy_doc_meta.mjs similarity index 66% rename from services/web/scripts/back_fill_dummy_doc_meta.mjs rename to tools/migrations/scripts/back_fill_dummy_doc_meta.mjs index e0684ea0be..c00adf9a23 100644 --- a/services/web/scripts/back_fill_dummy_doc_meta.mjs +++ b/tools/migrations/scripts/back_fill_dummy_doc_meta.mjs @@ -1,20 +1,10 @@ -import { promisify } from 'node:util' -import mongodb from 'mongodb-legacy' -import { - db, - READ_PREFERENCE_SECONDARY, -} from '../app/src/infrastructure/mongodb.js' -import _ from 'lodash' -import LRUCache from 'lru-cache' -import { fileURLToPath } from 'node:url' -import { scriptRunner } from './lib/ScriptRunner.mjs' +import mongodb from 'mongodb' +import { db, READ_PREFERENCE_SECONDARY } from '../lib/mongodb.mjs' const { ObjectId } = mongodb -const sleep = promisify(setTimeout) const NOW_IN_S = Date.now() / 1000 const ONE_WEEK_IN_S = 60 * 60 * 24 * 7 -const TEN_SECONDS = 10 * 1000 const DUMMY_NAME = 'unknown.tex' const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z') @@ -25,31 +15,16 @@ function getSecondsFromObjectId(id) { return id.getTimestamp().getTime() / 1000 } -async function main(options) { - if (!options) { - options = {} - } - _.defaults(options, { - dryRun: process.env.DRY_RUN === 'true', - cacheSize: parseInt(process.env.CACHE_SIZE, 10) || 100, - firstProjectId: new ObjectId(process.env.FIRST_PROJECT_ID), +async function main(firstProjectId) { + const options = { + firstProjectId, incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S, batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000, stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S, - letUserDoubleCheckInputsFor: - parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS, - }) - - if (!options.firstProjectId) { - console.error('Set FIRST_PROJECT_ID and re-run.') - process.exit(1) } - deletedProjectsCache = new LRUCache({ - max: options.cacheSize, - }) + deletedProjectsCache = new Map() - await letUserDoubleCheckInputs(options) let startId = options.firstProjectId let nProcessed = 0 @@ -133,30 +108,8 @@ async function processBatch(docs, options) { } else { console.log('Orphaned deleted doc %s (no deletedProjects entry)', docId) } - if (options.dryRun) continue await db.docs.updateOne({ _id: docId }, { $set: { name, deletedAt } }) } } -async function letUserDoubleCheckInputs(options) { - console.error('Options:', JSON.stringify(options, null, 2)) - console.error( - 'Waiting for you to double check inputs for', - options.letUserDoubleCheckInputsFor, - 'ms' - ) - await sleep(options.letUserDoubleCheckInputsFor) -} - export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - try { - await scriptRunner(async () => await main()) - console.error('Done.') - process.exit(0) - } catch (error) { - console.error({ error }) - process.exit(1) - } -} diff --git a/services/web/scripts/remove_deleted_users_from_token_access_refs.mjs b/tools/migrations/scripts/remove_deleted_users_from_token_access_refs.mjs similarity index 53% rename from services/web/scripts/remove_deleted_users_from_token_access_refs.mjs rename to tools/migrations/scripts/remove_deleted_users_from_token_access_refs.mjs index cfb0b9ddb1..8aaec30bba 100644 --- a/services/web/scripts/remove_deleted_users_from_token_access_refs.mjs +++ b/tools/migrations/scripts/remove_deleted_users_from_token_access_refs.mjs @@ -1,29 +1,11 @@ -import { - db, - READ_PREFERENCE_SECONDARY, -} from '../app/src/infrastructure/mongodb.js' +import { db, READ_PREFERENCE_SECONDARY } from '../lib/mongodb.mjs' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import mongodb from 'mongodb-legacy' -import minimist from 'minimist' -import CollaboratorsHandler from '../app/src/Features/Collaborators/CollaboratorsHandler.js' -import { fileURLToPath } from 'node:url' +import mongodb from 'mongodb' +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' const { ObjectId } = mongodb -const argv = minimist(process.argv.slice(2), { - string: ['projects'], - boolean: ['dry-run', 'help'], - alias: { - projects: 'p', - }, - default: { - 'dry-run': true, - }, -}) - -const DRY_RUN = argv['dry-run'] -const PROJECTS_LIST = argv.projects - async function findUserIds() { const userIds = new Set() const cursor = db.users.find( @@ -43,14 +25,7 @@ async function findUserIds() { return userIds } -async function fixProjectsWithInvalidTokenAccessRefsIds( - DRY_RUN, - PROJECTS_LIST -) { - if (DRY_RUN) { - console.log('=> Doing dry run') - } - +export default async function fixProjectsWithInvalidTokenAccessRefsIds() { const DELETED_USER_COLLABORATOR_IDS = new Set() const PROJECTS_WITH_DELETED_USER = new Set() @@ -58,7 +33,7 @@ async function fixProjectsWithInvalidTokenAccessRefsIds( const userIds = await findUserIds() // default query for finding all projects with non-existing/null or non-empty token access fields - let query = { + const query = { $or: [ { tokenAccessReadOnly_refs: { $not: { $type: 'array' } } }, { tokenAccessReadAndWrite_refs: { $not: { $type: 'array' } } }, @@ -67,15 +42,6 @@ async function fixProjectsWithInvalidTokenAccessRefsIds( ], } - const projectIds = PROJECTS_LIST?.split(',').map( - projectId => new ObjectId(projectId) - ) - - // query for finding projects passed in as args - if (projectIds) { - query = { $and: [{ _id: { $in: projectIds } }] } - } - await batchedUpdate( db.projects, query, @@ -89,28 +55,22 @@ async function fixProjectsWithInvalidTokenAccessRefsIds( // update the token access fields if necessary if (isTokenAccessFieldMissing) { - if (DRY_RUN) { - console.log( - `=> DRY RUN - would fix non-existing token access fields in project ${project._id.toString()}` - ) - } else { - const fields = [ - 'tokenAccessReadOnly_refs', - 'tokenAccessReadAndWrite_refs', - ] - for (const field of fields) { - await db.projects.updateOne( - { - _id: project._id, - [field]: { $not: { $type: 'array' } }, - }, - { $set: { [field]: [] } } - ) - } - console.log( - `=> Fixed non-existing token access fields in project ${project._id.toString()}` + const fields = [ + 'tokenAccessReadOnly_refs', + 'tokenAccessReadAndWrite_refs', + ] + for (const field of fields) { + await db.projects.updateOne( + { + _id: project._id, + [field]: { $not: { $type: 'array' } }, + }, + { $set: { [field]: [] } } ) } + console.log( + `=> Fixed non-existing token access fields in project ${project._id.toString()}` + ) } // find the set of user ids that are in the token access fields @@ -154,29 +114,17 @@ async function fixProjectsWithInvalidTokenAccessRefsIds( 'in project:', project._id.toString() ) - if (DRY_RUN) { - console.log( - `=> DRY RUN - would remove deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})` - ) - continue - } console.log( `=> Removing deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})` ) - await CollaboratorsHandler.promises.removeUserFromAllProjects( - new ObjectId(deletedUserId) - ) + await removeUserFromAllProjects(new ObjectId(deletedUserId)) } } }, { tokenAccessReadOnly_refs: 1, tokenAccessReadAndWrite_refs: 1 } ) - console.log( - `=> ${DRY_RUN ? 'DRY RUN - would delete' : 'Deleted'} user ids (${ - DELETED_USER_COLLABORATOR_IDS.size - })` - ) + console.log(`Deleted user ids (${DELETED_USER_COLLABORATOR_IDS.size})`) if (DELETED_USER_COLLABORATOR_IDS.size) { console.log(Array.from(DELETED_USER_COLLABORATOR_IDS).join('\n')) } @@ -188,34 +136,79 @@ async function fixProjectsWithInvalidTokenAccessRefsIds( } } -async function main(DRY_RUN, PROJECTS_LIST) { - await fixProjectsWithInvalidTokenAccessRefsIds(DRY_RUN, PROJECTS_LIST) +// Copied from services/web/app/src/Features/Collaborators/CollaboratorsHandler.js +async function removeUserFromAllProjects(userId) { + const { readAndWrite, readOnly, tokenReadAndWrite, tokenReadOnly } = + await dangerouslyGetAllProjectsUserIsMemberOf(userId, { _id: 1 }) + const allProjects = readAndWrite + .concat(readOnly) + .concat(tokenReadAndWrite) + .concat(tokenReadOnly) + logger.info( + { + userId, + readAndWriteCount: readAndWrite.length, + readOnlyCount: readOnly.length, + tokenReadAndWriteCount: tokenReadAndWrite.length, + tokenReadOnlyCount: tokenReadOnly.length, + }, + 'removing user from projects' + ) + for (const project of allProjects) { + await removeUserFromProject(project._id, userId) + } + logger.info( + { + userId, + allProjectsCount: allProjects.length, + }, + 'removed user from all projects' + ) } -export default main - -if (fileURLToPath(import.meta.url) === process.argv[1]) { - if (argv.help || argv._.length > 1) { - console.error(`Usage: node scripts/remove_deleted_users_from_token_access_refs.mjs [OPTS] - Finds or removes deleted user ids from token access fields - "tokenAccessReadOnly_refs" and "tokenAccessReadAndWrite_refs" in the "projects" collection. - - If no projects are specified, all projects will be processed. - - Options: - - --dry-run finds projects and deleted users but does not do any updates - --projects list of projects ids to be fixed (comma separated) - `) - process.exit(1) - } - +// Copied from services/web/app/src/Features/Collaborators/CollaboratorsHandler.js +async function removeUserFromProject(projectId, userId) { try { - await main(DRY_RUN, PROJECTS_LIST) - console.error('Done') - process.exit(0) - } catch (error) { - console.error(error) - process.exit(1) + await db.projects.updateOne( + { _id: projectId }, + { + $pull: { + collaberator_refs: userId, + readOnly_refs: userId, + reviewer_refs: userId, + pendingEditor_refs: userId, + pendingReviewer_refs: userId, + tokenAccessReadOnly_refs: userId, + tokenAccessReadAndWrite_refs: userId, + archived: userId, + trashed: userId, + }, + } + ) + } catch (err) { + throw OError.tag(err, 'problem removing user from project collaborators', { + projectId, + userId, + }) } } + +// Copied from services/web/app/src/Features/Collaborators/CollaboratorsGetter.js +// This function returns all the projects that a user is a member of, regardless of +// the current state of the project, so it includes those projects where token access +// has been disabled. +async function dangerouslyGetAllProjectsUserIsMemberOf(userId, fields) { + const readAndWrite = await db.projects + .find({ collaberator_refs: userId }, fields) + .toArray() + const readOnly = await db.projects + .find({ readOnly_refs: userId }, fields) + .toArray() + const tokenReadAndWrite = await db.projects + .find({ tokenAccessReadAndWrite_refs: userId }, fields) + .toArray() + const tokenReadOnly = await db.projects + .find({ tokenAccessReadOnly_refs: userId }, fields) + .toArray() + return { readAndWrite, readOnly, tokenReadAndWrite, tokenReadOnly } +} diff --git a/tools/migrations/tsconfig.json b/tools/migrations/tsconfig.json new file mode 100644 index 0000000000..5a53f3a3e6 --- /dev/null +++ b/tools/migrations/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.backend.json", + "include": [ + "**/*.js", + "**/*.cjs", + "**/*.mjs", + "**/*.ts" + ] +}