diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index 64e75ffc86..2d7138b858 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -52,7 +52,7 @@ services: user: root mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index b7f7aa2653..a8ac718497 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/chat environment: MOCHA_GREP: ${MOCHA_GREP} @@ -50,7 +51,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index 690a4fd39e..4d256f1e41 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -52,7 +52,7 @@ services: user: root mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 59ecadc38a..7a5c5ac6a3 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/contacts environment: MOCHA_GREP: ${MOCHA_GREP} @@ -50,7 +51,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/contacts/test/unit/js/ContactsManagerTests.js b/services/contacts/test/unit/js/ContactsManagerTests.js index eeaae4cabb..b4506aa515 100644 --- a/services/contacts/test/unit/js/ContactsManagerTests.js +++ b/services/contacts/test/unit/js/ContactsManagerTests.js @@ -5,6 +5,7 @@ import { expect } from 'chai' import { ObjectId } from 'mongodb' import { cleanupTestDatabase } from '../../../app/js/mongodb.js' import * as ContactManager from '../../../app/js/ContactManager.js' +import '../../acceptance/js/MongoHelper.js' describe('ContactManager', function () { beforeEach(cleanupTestDatabase) diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index 60244b1480..8a9bd9824b 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -57,7 +57,7 @@ services: user: root mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index aa1a282764..4510aab937 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/docstore environment: MOCHA_GREP: ${MOCHA_GREP} @@ -55,7 +56,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index 143a0e5e7c..825c1335c4 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -71,7 +71,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 92227a6a8c..e8c35494c8 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/document-updater environment: MOCHA_GREP: ${MOCHA_GREP} @@ -69,7 +70,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index 8c1cb792c6..7bb0a1c295 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -95,7 +95,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 8b27e8a17d..8b9935eced 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -16,6 +16,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/history-v1 environment: MOCHA_GREP: ${MOCHA_GREP} @@ -99,7 +100,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/history-v1/test/setup.js b/services/history-v1/test/setup.js index 4d168f17a6..42bfdadeba 100644 --- a/services/history-v1/test/setup.js +++ b/services/history-v1/test/setup.js @@ -20,7 +20,8 @@ async function setupPostgresDatabase() { async function setupMongoDatabase() { this.timeout(60_000) await promisify(exec)( - `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce` + // Run saas migrations for backup indexes + `cd ../../tools/migrations && npm run migrations -- migrate -t saas` ) } diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index ebaa8b1409..622c6aafe7 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -55,7 +55,7 @@ services: user: root mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index cfaf888467..d89b93d0f1 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json working_dir: /overleaf/services/notifications environment: @@ -52,7 +53,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index d85afdc752..d369884e68 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -71,7 +71,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index 9bea08c8d2..05866fbcfa 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -10,6 +10,7 @@ services: - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + - ../../tools/migrations:/overleaf/tools/migrations working_dir: /overleaf/services/project-history environment: MOCHA_GREP: ${MOCHA_GREP} @@ -69,7 +70,7 @@ services: mongo: image: mongo:8.0.11 - command: --replSet overleaf + command: --replSet overleaf --notablescan volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js - ../../bin/shared/mongodb-docker-entrypoint-wait.sh:/mongodb-docker-entrypoint-wait.sh diff --git a/tools/migrations/20230512100122_ensure_history_migration.mjs b/tools/migrations/20230512100122_ensure_history_migration.mjs index 93164293c7..5bce6380e0 100644 --- a/tools/migrations/20230512100122_ensure_history_migration.mjs +++ b/tools/migrations/20230512100122_ensure_history_migration.mjs @@ -6,6 +6,10 @@ const tags = ['server-ce', 'server-pro'] const migrate = async client => { const { db } = client + const totalProjects = await db.projects.estimatedDocumentCount() + if (totalProjects === 0) return + + // Does not use an index. const count = await db.projects.countDocuments({ 'overleaf.history.display': { $ne: true }, }) diff --git a/tools/migrations/202407131109055_admin_reversed_hostname.mjs b/tools/migrations/202407131109055_admin_reversed_hostname.mjs deleted file mode 100644 index ad1adaa9b9..0000000000 --- a/tools/migrations/202407131109055_admin_reversed_hostname.mjs +++ /dev/null @@ -1,66 +0,0 @@ -const tags = ['server-ce', 'server-pro'] - -const migrate = async client => { - const { db } = client - - const adminsWithEmails = await db.users - .find( - { - isAdmin: true, - emails: { $exists: true }, - }, - { _id: 1, emails: 1 } - ) - .toArray() - - for (const { _id, emails } of adminsWithEmails) { - let shouldUpdateEmails = false - for (const emailObj of emails) { - if (!emailObj.reversedHostname) { - shouldUpdateEmails = true - emailObj.reversedHostname = emailObj.email - .split('@')[1] - .split('') - .reverse() - .join('') - } - } - if (shouldUpdateEmails) { - await db.users.updateOne({ _id }, { $set: { emails } }) - } - } - - const adminsNoEmails = await db.users - .find( - { - isAdmin: true, - emails: { $exists: false }, - }, - { _id: 1, email: 1 } - ) - .toArray() - - for (const { _id, email } of adminsNoEmails) { - const reversedHostname = email.split('@')[1].split('').reverse().join('') - await db.users.updateOne( - { _id }, - { - emails: [ - { - email, - reversedHostname, - createdAt: new Date(), - }, - ], - } - ) - } -} - -const rollback = async () => {} - -export default { - tags, - migrate, - rollback, -} diff --git a/tools/migrations/20240713110905_emails_reversed_hostname.mjs b/tools/migrations/20240713110905_emails_reversed_hostname.mjs new file mode 100644 index 0000000000..482af9690d --- /dev/null +++ b/tools/migrations/20240713110905_emails_reversed_hostname.mjs @@ -0,0 +1,40 @@ +const tags = ['server-ce', 'server-pro'] + +function fixEmails(user) { + let emails = user.emails + if (!emails || emails.length === 0) { + emails = [{ email: user.email, createdAt: new Date() }] + } + for (let i = 0; i < emails.length; i++) { + const reversedHostname = emails[i].email + .split('@')[1] + .split('') + .reverse() + .join('') + if (emails[i].reversedHostname !== reversedHostname) { + emails = emails.slice() + emails[i].reversedHostname = reversedHostname + } + } + return emails +} + +const migrate = async client => { + const { db } = client + + const cursor = db.users.find({}, { projection: { email: 1, emails: 1 } }) + for await (const user of cursor) { + const emails = fixEmails(user) + if (user.emails !== emails) { + await db.users.updateOne({ _id: user._id }, { $set: { emails } }) + } + } +} + +const rollback = async () => {} + +export default { + tags, + migrate, + rollback, +}