From 8f0913fafef10f9609b58d6ce3dce8be7cef886e Mon Sep 17 00:00:00 2001 From: Eric Mc Sween <5454374+emcsween@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:15:20 -0400 Subject: [PATCH] Merge pull request #27936 from overleaf/em-unit-tests-mongo Make Mongo available to unit tests in all services GitOrigin-RevId: b65bbb69883d5bba31d09802b89f35bdc523fe4d --- libraries/mongo-utils/index.js | 5 ++ libraries/mongo-utils/test-utils.js | 55 ++++++++++++++ libraries/redis-wrapper/index.js | 30 ++++++++ package-lock.json | 7 ++ services/chat/Makefile | 1 - services/chat/app/js/mongodb.js | 7 ++ services/chat/docker-compose.ci.yml | 7 ++ services/chat/docker-compose.yml | 6 ++ services/chat/package.json | 3 +- services/clsi/Makefile | 1 - services/clsi/docker-compose.ci.yml | 1 + services/clsi/docker-compose.yml | 1 + services/clsi/package.json | 2 +- services/contacts/Makefile | 1 - services/contacts/app/js/mongodb.js | 7 ++ services/contacts/docker-compose.ci.yml | 7 ++ services/contacts/docker-compose.yml | 6 ++ services/contacts/package.json | 3 +- .../test/unit/js/ContactsManagerTests.js | 75 +++++-------------- services/docstore/Makefile | 1 - services/docstore/app/js/mongodb.js | 8 ++ services/docstore/docker-compose.ci.yml | 7 ++ services/docstore/docker-compose.yml | 6 ++ services/docstore/package.json | 3 +- services/document-updater/Makefile | 1 - .../document-updater/app/js/RedisManager.js | 10 ++- services/document-updater/app/js/mongodb.js | 8 ++ .../document-updater/docker-compose.ci.yml | 25 +++++-- services/document-updater/docker-compose.yml | 24 ++++-- services/document-updater/package.json | 3 +- services/filestore/Makefile | 1 - services/filestore/docker-compose.ci.yml | 1 + services/filestore/docker-compose.yml | 1 + services/filestore/package.json | 2 +- services/history-v1/Makefile | 1 - services/history-v1/docker-compose.ci.yml | 25 +++++-- services/history-v1/docker-compose.yml | 24 ++++-- services/history-v1/package.json | 2 +- services/history-v1/storage/lib/mongodb.js | 8 ++ services/history-v1/storage/lib/redis.js | 7 ++ services/notifications/Makefile | 1 - services/notifications/app/js/mongodb.js | 8 ++ services/notifications/docker-compose.ci.yml | 7 ++ services/notifications/docker-compose.yml | 6 ++ services/notifications/package.json | 3 +- services/project-history/Makefile | 1 - .../project-history/app/js/RedisManager.js | 5 ++ services/project-history/app/js/mongodb.js | 8 +- .../project-history/docker-compose.ci.yml | 25 +++++-- services/project-history/docker-compose.yml | 24 ++++-- services/project-history/package.json | 3 +- services/real-time/Makefile | 1 - services/real-time/docker-compose.ci.yml | 20 +++-- services/real-time/docker-compose.yml | 20 +++-- services/real-time/package.json | 2 +- .../app/src/infrastructure/RedisWrapper.js | 13 +--- .../web/app/src/infrastructure/mongodb.js | 26 +------ 57 files changed, 404 insertions(+), 162 deletions(-) create mode 100644 libraries/mongo-utils/test-utils.js diff --git a/libraries/mongo-utils/index.js b/libraries/mongo-utils/index.js index e69de29bb2..0b676bed9d 100644 --- a/libraries/mongo-utils/index.js +++ b/libraries/mongo-utils/index.js @@ -0,0 +1,5 @@ +// @ts-check + +const { cleanupTestDatabase, dropTestDatabase } = require('./test-utils') + +module.exports = { cleanupTestDatabase, dropTestDatabase } diff --git a/libraries/mongo-utils/test-utils.js b/libraries/mongo-utils/test-utils.js new file mode 100644 index 0000000000..39ec1f0f90 --- /dev/null +++ b/libraries/mongo-utils/test-utils.js @@ -0,0 +1,55 @@ +// @ts-check + +/** + * @import { MongoClient } from 'mongodb' + * @import { MongoClient as LegacyMongoClient } from 'mongodb-legacy' + */ + +/** + * Delete all data from the test Mongo database + * + * This doesn't drop the collections, so indexes are preserved. + * + * @param {MongoClient | LegacyMongoClient} mongoClient + */ +async function cleanupTestDatabase(mongoClient) { + ensureTestDatabase(mongoClient) + const db = mongoClient.db() + const allCollections = await db.collections() + const collections = allCollections.filter( + coll => coll.collectionName !== 'migrations' + ) + await Promise.all(collections.map(coll => coll.deleteMany({}))) +} + +/** + * Drop the test Monto database + * + * This drops the whole database, including indexes. + * + * @param {MongoClient | LegacyMongoClient } mongoClient + */ +async function dropTestDatabase(mongoClient) { + ensureTestDatabase(mongoClient) + await mongoClient.db().dropDatabase() +} +/** + * Ensure that the given client is connected to a test database. + * + * This should be called before performing destructive operations on the test + * database. + * + * @param {MongoClient | LegacyMongoClient } mongoClient + */ +function ensureTestDatabase(mongoClient) { + const dbName = mongoClient.db().databaseName + const env = process.env.NODE_ENV + + if (dbName !== 'test-overleaf' || env !== 'test') { + throw new Error( + `Refusing to clear database '${dbName}' in environment '${env}'` + ) + } +} + +module.exports = { cleanupTestDatabase, dropTestDatabase } diff --git a/libraries/redis-wrapper/index.js b/libraries/redis-wrapper/index.js index 2e16014490..ce61f1eea1 100644 --- a/libraries/redis-wrapper/index.js +++ b/libraries/redis-wrapper/index.js @@ -1,3 +1,5 @@ +// @ts-check + const crypto = require('node:crypto') const os = require('node:os') const { promisify } = require('node:util') @@ -168,6 +170,34 @@ async function runWithTimeout({ runner, timeout, context }) { ]) } +/** + * Delete all data from the test Redis instance + * + * @param {Redis} rclient + */ +async function cleanupTestRedis(rclient) { + ensureTestRedis(rclient) + await rclient.flushall() +} + +/** + * Checks that the Redis client points to a test database + * + * In tests, the Redis instance is on a host called redis_test + * + * @param {Redis} rclient + */ +function ensureTestRedis(rclient) { + const host = rclient.options.host + const env = process.env.NODE_ENV + if (host !== 'redis_test' || env !== 'test') { + throw new Error( + `Refusing to clear Redis instance '${host}' in environment '${env}'` + ) + } +} + module.exports = { createClient, + cleanupTestRedis, } diff --git a/package-lock.json b/package-lock.json index 583aad502a..8929c472fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42339,6 +42339,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", @@ -42491,6 +42492,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", @@ -42556,6 +42558,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/promise-utils": "*", @@ -42627,6 +42630,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/promise-utils": "*", "@overleaf/ranges-tracker": "*", @@ -44089,6 +44093,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", @@ -44113,6 +44118,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/promise-utils": "*", "@overleaf/redis-wrapper": "*", @@ -44341,6 +44347,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/settings": "*", "async": "^3.2.5", diff --git a/services/chat/Makefile b/services/chat/Makefile index 3a566274ca..8b06470cc1 100644 --- a/services/chat/Makefile +++ b/services/chat/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/chat/app/js/mongodb.js b/services/chat/app/js/mongodb.js index 2ed6e01b37..977410937c 100644 --- a/services/chat/app/js/mongodb.js +++ b/services/chat/app/js/mongodb.js @@ -1,5 +1,8 @@ +// @ts-check + import Metrics from '@overleaf/metrics' import Settings from '@overleaf/settings' +import MongoUtils from '@overleaf/mongo-utils' import { MongoClient } from 'mongodb' export { ObjectId } from 'mongodb' @@ -16,3 +19,7 @@ export const db = { } Metrics.mongodb.monitor(mongoClient) + +export async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index 5ea5d35566..ca087c4b6f 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -6,10 +6,17 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + depends_on: + mongo: + condition: service_started test_acceptance: diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index f63d616aef..32de3bf570 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -9,14 +9,20 @@ services: - .:/overleaf/services/chat - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/chat environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started test_acceptance: image: node:22.18.0 diff --git a/services/chat/package.json b/services/chat/package.json index f3d37eb6d3..e74f09a5d3 100644 --- a/services/chat/package.json +++ b/services/chat/package.json @@ -10,7 +10,7 @@ "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "lint": "eslint --max-warnings 0 --format unix .", "format": "prettier --list-different $PWD/'**/*.*js'", "format:fix": "prettier --write $PWD/'**/*.*js'", @@ -20,6 +20,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", diff --git a/services/clsi/Makefile b/services/clsi/Makefile index 9f4e9eee1b..20273fc5d0 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -87,7 +87,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 6a528f3339..081ac2bc32 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -7,6 +7,7 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 2c65dce2a1..8c5ffa59f9 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -16,6 +16,7 @@ services: environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" command: npm run --silent test:unit diff --git a/services/clsi/package.json b/services/clsi/package.json index fe31c430bd..4b1db50b0f 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", diff --git a/services/contacts/Makefile b/services/contacts/Makefile index 21baec5a46..e32afc73ad 100644 --- a/services/contacts/Makefile +++ b/services/contacts/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/contacts/app/js/mongodb.js b/services/contacts/app/js/mongodb.js index d3d161a12f..95a5e51278 100644 --- a/services/contacts/app/js/mongodb.js +++ b/services/contacts/app/js/mongodb.js @@ -1,4 +1,7 @@ +// @ts-check + import Metrics from '@overleaf/metrics' +import MongoUtils from '@overleaf/mongo-utils' import Settings from '@overleaf/settings' import { MongoClient } from 'mongodb' @@ -15,3 +18,7 @@ export const db = { } Metrics.mongodb.monitor(mongoClient) + +export async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index 5ea5d35566..ca087c4b6f 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -6,10 +6,17 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + depends_on: + mongo: + condition: service_started test_acceptance: diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 7c1eb552ab..f68c5d2bde 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -9,14 +9,20 @@ services: - .:/overleaf/services/contacts - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/contacts environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started test_acceptance: image: node:22.18.0 diff --git a/services/contacts/package.json b/services/contacts/package.json index 596272a62e..81e6285aed 100644 --- a/services/contacts/package.json +++ b/services/contacts/package.json @@ -8,7 +8,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", @@ -20,6 +20,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", diff --git a/services/contacts/test/unit/js/ContactsManagerTests.js b/services/contacts/test/unit/js/ContactsManagerTests.js index f5f5ae59a7..eeaae4cabb 100644 --- a/services/contacts/test/unit/js/ContactsManagerTests.js +++ b/services/contacts/test/unit/js/ContactsManagerTests.js @@ -1,21 +1,16 @@ +// @ts-check + import sinon from 'sinon' import { expect } from 'chai' -import esmock from 'esmock' import { ObjectId } from 'mongodb' +import { cleanupTestDatabase } from '../../../app/js/mongodb.js' +import * as ContactManager from '../../../app/js/ContactManager.js' describe('ContactManager', function () { + beforeEach(cleanupTestDatabase) + beforeEach(async function () { this.clock = sinon.useFakeTimers(new Date()) - - this.db = { contacts: {} } - - this.ContactManager = await esmock('../../../app/js/ContactManager', { - '../../../app/js/mongodb': { - db: this.db, - ObjectId, - }, - }) - this.user_id = new ObjectId().toString() this.contact_id = new ObjectId().toString() }) @@ -25,42 +20,24 @@ describe('ContactManager', function () { }) describe('touchContact', function () { - beforeEach(function () { - this.db.contacts.updateOne = sinon.stub().resolves() - }) - describe('with a valid user_id', function () { it('should increment the contact count and timestamp', async function () { - await expect( - this.ContactManager.touchContact(this.user_id, 'mock_contact') - ).not.to.be.rejected - - expect(this.db.contacts.updateOne).to.be.calledWith( - { - user_id: sinon.match(o => o.toString() === this.user_id), + const now = new Date() + await ContactManager.touchContact(this.user_id, this.contact_id) + const contacts = await ContactManager.getContacts(this.user_id) + expect(contacts).to.deep.equal({ + [this.contact_id]: { + n: 1, + ts: now, }, - { - $inc: { - 'contacts.mock_contact.n': 1, - }, - $set: { - 'contacts.mock_contact.ts': new Date(), - }, - }, - { - upsert: true, - } - ) + }) }) }) describe('with an invalid user id', function () { it('should be rejected', async function () { await expect( - this.ContactManager.touchContact( - 'not-valid-object-id', - this.contact_id - ) + ContactManager.touchContact('not-valid-object-id', this.contact_id) ).to.be.rejectedWith( 'input must be a 24 character hex string, 12 byte Uint8Array, or an integer' ) @@ -69,29 +46,17 @@ describe('ContactManager', function () { }) describe('getContacts', function () { - beforeEach(function () { - this.user = { - contacts: ['mock', 'contacts'], - } - this.db.contacts.findOne = sinon.stub().resolves(this.user) - }) - describe('with a valid user_id', function () { - it("should find the user's contacts", async function () { - await expect( - this.ContactManager.getContacts(this.user_id) - ).to.eventually.deep.equal(this.user.contacts) - - expect(this.db.contacts.findOne).to.be.calledWith({ - user_id: sinon.match(o => o.toString() === this.user_id), - }) + it('should find an empty contact list', async function () { + const contacts = await ContactManager.getContacts(this.user_id) + expect(contacts).to.be.undefined }) }) describe('with an invalid user id', function () { it('should be rejected', async function () { - await expect(this.ContactManager.getContacts('not-valid-object-id')).to - .be.rejected + await expect(ContactManager.getContacts('not-valid-object-id')).to.be + .rejected }) }) }) diff --git a/services/docstore/Makefile b/services/docstore/Makefile index 82d9faa8e2..e962102f17 100644 --- a/services/docstore/Makefile +++ b/services/docstore/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/docstore/app/js/mongodb.js b/services/docstore/app/js/mongodb.js index 796cd6c78a..92e13c6670 100644 --- a/services/docstore/app/js/mongodb.js +++ b/services/docstore/app/js/mongodb.js @@ -1,5 +1,8 @@ +// @ts-check + const Metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') +const MongoUtils = require('@overleaf/mongo-utils') const { MongoClient, ObjectId } = require('mongodb-legacy') const mongoClient = new MongoClient(Settings.mongo.url, Settings.mongo.options) @@ -11,8 +14,13 @@ const db = { Metrics.mongodb.monitor(mongoClient) +async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} + module.exports = { db, mongoClient, ObjectId, + cleanupTestDatabase, } diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index 60eec75712..369cff74f6 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -6,10 +6,17 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + depends_on: + mongo: + condition: service_started test_acceptance: diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index 81481bcfeb..ce09176420 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -9,14 +9,20 @@ services: - .:/overleaf/services/docstore - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/docstore environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started test_acceptance: image: node:22.18.0 diff --git a/services/docstore/package.json b/services/docstore/package.json index bf5857fd49..787d898879 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", @@ -20,6 +20,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/promise-utils": "*", diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index 384c56d07c..d16c201e36 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/document-updater/app/js/RedisManager.js b/services/document-updater/app/js/RedisManager.js index 7f86036427..b14bd76f2b 100644 --- a/services/document-updater/app/js/RedisManager.js +++ b/services/document-updater/app/js/RedisManager.js @@ -1,7 +1,5 @@ const Settings = require('@overleaf/settings') -const rclient = require('@overleaf/redis-wrapper').createClient( - Settings.redis.documentupdater -) +const RedisWrapper = require('@overleaf/redis-wrapper') const logger = require('@overleaf/logger') const OError = require('@overleaf/o-error') const { promisifyAll } = require('@overleaf/promise-utils') @@ -11,6 +9,8 @@ const crypto = require('node:crypto') const async = require('async') const { docIsTooLarge } = require('./Limits') +const rclient = RedisWrapper.createClient(Settings.redis.documentupdater) + // Sometimes Redis calls take an unexpectedly long time. We have to be // quick with Redis calls because we're holding a lock that expires // after 30 seconds. We can't let any errors in the rest of the stack @@ -777,6 +777,10 @@ const RedisManager = { // binary in node < v5 return crypto.createHash('sha1').update(docLines, 'utf8').digest('hex') }, + + async cleanupTestRedis() { + await RedisWrapper.cleanupTestRedis(rclient) + }, } module.exports = RedisManager diff --git a/services/document-updater/app/js/mongodb.js b/services/document-updater/app/js/mongodb.js index 6e38993bc4..74f87cab18 100644 --- a/services/document-updater/app/js/mongodb.js +++ b/services/document-updater/app/js/mongodb.js @@ -1,4 +1,7 @@ +// @ts-check + const Metrics = require('@overleaf/metrics') +const MongoUtils = require('@overleaf/mongo-utils') const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb-legacy') @@ -20,9 +23,14 @@ async function healthCheck() { Metrics.mongodb.monitor(mongoClient) +async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} + module.exports = { db, ObjectId, mongoClient, healthCheck: require('node:util').callbackify(healthCheck), + cleanupTestDatabase, } diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index 52c767e92c..73874aaf3f 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -6,10 +6,23 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: @@ -17,10 +30,10 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -31,7 +44,7 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy user: node entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- @@ -45,7 +58,7 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index c8d0f4ab22..816a344f35 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -9,14 +9,26 @@ services: - .:/overleaf/services/document-updater - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/document-updater environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: image: node:22.18.0 @@ -28,10 +40,10 @@ services: working_dir: /overleaf/services/document-updater environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -42,12 +54,12 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:acceptance - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] diff --git a/services/document-updater/package.json b/services/document-updater/package.json index 7d892689e9..5a32057496 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "benchmark:apply": "node benchmarks/apply", @@ -20,6 +20,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/promise-utils": "*", "@overleaf/ranges-tracker": "*", diff --git a/services/filestore/Makefile b/services/filestore/Makefile index 0521fbadd8..6f135223ae 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -87,7 +87,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index 7b90eac777..ca1e86c17d 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -11,6 +11,7 @@ services: user: node command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index c71dd1d759..0eef4aecbc 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -19,6 +19,7 @@ services: environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" command: npm run --silent test:unit diff --git a/services/filestore/package.json b/services/filestore/package.json index 4b9043aed7..985cc9cc95 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -14,7 +14,7 @@ "format": "prettier --list-different $PWD/'**/*.*js'", "format:fix": "prettier --write $PWD/'**/*.*js'", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "lint:fix": "eslint --fix .", "types:check": "tsc --noEmit" }, diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile index 2fd35dd59c..a4ea5e329d 100644 --- a/services/history-v1/Makefile +++ b/services/history-v1/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index 98d4cdd0da..3d1314663c 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -9,10 +9,23 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: @@ -20,10 +33,10 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres AWS_S3_ENDPOINT: https://minio:9000 @@ -44,7 +57,7 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy postgres: condition: service_healthy @@ -68,7 +81,7 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 865ed8c67e..a4d79d39d2 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -15,14 +15,26 @@ services: - .:/overleaf/services/history-v1 - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/history-v1 environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: build: @@ -38,10 +50,10 @@ services: working_dir: /overleaf/services/history-v1 environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres AWS_S3_ENDPOINT: https://minio:9000 @@ -61,7 +73,7 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy postgres: condition: service_healthy @@ -76,7 +88,7 @@ services: entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:acceptance - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] diff --git a/services/history-v1/package.json b/services/history-v1/package.json index 4796cafd03..1156d4b546 100644 --- a/services/history-v1/package.json +++ b/services/history-v1/package.json @@ -66,7 +66,7 @@ "format:fix": "prettier --write $PWD/'**/*.*js'", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "nodemon": "node --watch app.js", "migrate": "knex migrate:latest", diff --git a/services/history-v1/storage/lib/mongodb.js b/services/history-v1/storage/lib/mongodb.js index e887bc25a5..210dd87354 100644 --- a/services/history-v1/storage/lib/mongodb.js +++ b/services/history-v1/storage/lib/mongodb.js @@ -1,4 +1,7 @@ +// @ts-check + const Metrics = require('@overleaf/metrics') +const MongoUtils = require('@overleaf/mongo-utils') const config = require('config') const { MongoClient } = require('mongodb') @@ -18,6 +21,10 @@ const backedUpBlobs = db.collection('projectHistoryBackedUpBlobs') Metrics.mongodb.monitor(client) +async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(client) +} + module.exports = { client, db, @@ -27,4 +34,5 @@ module.exports = { projects, shardedBlobs, backedUpBlobs, + cleanupTestDatabase, } diff --git a/services/history-v1/storage/lib/redis.js b/services/history-v1/storage/lib/redis.js index 9b00cc0a26..dfc4f25c58 100644 --- a/services/history-v1/storage/lib/redis.js +++ b/services/history-v1/storage/lib/redis.js @@ -1,3 +1,5 @@ +// @ts-check + const config = require('config') const redis = require('@overleaf/redis-wrapper') @@ -11,9 +13,14 @@ async function disconnect() { await Promise.all([rclientHistory.disconnect(), rclientLock.disconnect()]) } +async function cleanupTestRedis() { + await redis.cleanupTestRedis(rclientHistory) +} + module.exports = { rclientHistory, rclientLock, redis, disconnect, + cleanupTestRedis, } diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 8840a38cc9..4b43ed1f57 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/notifications/app/js/mongodb.js b/services/notifications/app/js/mongodb.js index 764998fb74..e0dd89fa08 100644 --- a/services/notifications/app/js/mongodb.js +++ b/services/notifications/app/js/mongodb.js @@ -1,4 +1,7 @@ +// @ts-check + const Metrics = require('@overleaf/metrics') +const MongoUtils = require('@overleaf/mongo-utils') const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb-legacy') @@ -11,8 +14,13 @@ const db = { Metrics.mongodb.monitor(mongoClient) +async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} + module.exports = { db, mongoClient, ObjectId, + cleanupTestDatabase, } diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index 5ea5d35566..ca087c4b6f 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -6,10 +6,17 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + depends_on: + mongo: + condition: service_started test_acceptance: diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index 046846a479..583aa6a1b1 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -9,14 +9,20 @@ services: - .:/overleaf/services/notifications - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/notifications environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started test_acceptance: image: node:22.18.0 diff --git a/services/notifications/package.json b/services/notifications/package.json index a591e897c9..5e2cbc48f0 100644 --- a/services/notifications/package.json +++ b/services/notifications/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", @@ -21,6 +21,7 @@ "dependencies": { "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/settings": "*", "async": "^3.2.5", "body-parser": "^1.20.3", diff --git a/services/project-history/Makefile b/services/project-history/Makefile index f4c4ebaf33..5393d0451a 100644 --- a/services/project-history/Makefile +++ b/services/project-history/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/project-history/app/js/RedisManager.js b/services/project-history/app/js/RedisManager.js index 2f79a10a91..8847f9dfc7 100644 --- a/services/project-history/app/js/RedisManager.js +++ b/services/project-history/app/js/RedisManager.js @@ -365,6 +365,10 @@ async function clearCachedHistoryId(projectId) { await rclient.del(key) } +async function cleanupTestRedis() { + await redis.cleanupTestRedis(rclient) +} + // EXPORTS const countUnprocessedUpdatesCb = callbackify(countUnprocessedUpdates) @@ -442,4 +446,5 @@ export const promises = { getCachedHistoryId, setCachedHistoryId, clearCachedHistoryId, + cleanupTestRedis, } diff --git a/services/project-history/app/js/mongodb.js b/services/project-history/app/js/mongodb.js index d639903ce2..6995362625 100644 --- a/services/project-history/app/js/mongodb.js +++ b/services/project-history/app/js/mongodb.js @@ -1,10 +1,11 @@ import Metrics from '@overleaf/metrics' +import MongoUtils from '@overleaf/mongo-utils' import Settings from '@overleaf/settings' import mongodb from 'mongodb-legacy' const { MongoClient, ObjectId } = mongodb /** - * @import { ProjectHistoryFailure } from './mongo-types.ts' + * @import { ProjectHistoryFailure } from './mongo-types' */ export { ObjectId } @@ -17,6 +18,10 @@ const mongoDb = mongoClient.db() Metrics.mongodb.monitor(mongoClient) +async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} + export const db = { deletedProjects: mongoDb.collection('deletedProjects'), projects: mongoDb.collection('projects'), @@ -24,4 +29,5 @@ export const db = { projectHistoryFailures: mongoDb.collection('projectHistoryFailures'), projectHistoryLabels: mongoDb.collection('projectHistoryLabels'), projectHistorySyncState: mongoDb.collection('projectHistorySyncState'), + cleanupTestDatabase, } diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index 52c767e92c..73874aaf3f 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -6,10 +6,23 @@ services: test_unit: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node + volumes: + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: @@ -17,10 +30,10 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -31,7 +44,7 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy user: node entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- @@ -45,7 +58,7 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index 10c8f5d955..c92c43f9e4 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -9,14 +9,26 @@ services: - .:/overleaf/services/project-history - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it working_dir: /overleaf/services/project-history environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:unit user: node + depends_on: + mongo: + condition: service_started + redis_test: + condition: service_healthy test_acceptance: image: node:22.18.0 @@ -28,10 +40,10 @@ services: working_dir: /overleaf/services/project-history environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -42,12 +54,12 @@ services: depends_on: mongo: condition: service_started - redis: + redis_test: condition: service_healthy entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=60 -- command: npm run --silent test:acceptance - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] diff --git a/services/project-history/package.json b/services/project-history/package.json index 4160f36f6f..304d23d957 100644 --- a/services/project-history/package.json +++ b/services/project-history/package.json @@ -10,7 +10,7 @@ "start": "node app.js", "nodemon": "node --watch app.js", "test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec --exit $@ test/unit/js", "lint": "eslint --max-warnings 0 --format unix .", "format": "prettier --list-different $PWD/'**/*.*js'", "format:fix": "prettier --write $PWD/'**/*.*js'", @@ -21,6 +21,7 @@ "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", "@overleaf/o-error": "*", "@overleaf/promise-utils": "*", "@overleaf/redis-wrapper": "*", diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 206b300806..ffa21ed9fc 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -86,7 +86,6 @@ test: format lint typecheck shellcheck test_unit test_acceptance test_unit: ifneq (,$(wildcard test/unit)) $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit - $(MAKE) test_unit_clean endif test_clean: test_unit_clean diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index a2de0ad009..1107b62bbf 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -8,8 +8,16 @@ services: user: node command: npm run test:unit:_run environment: + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test + depends_on: + redis_test: + condition: service_healthy test_acceptance: @@ -17,17 +25,17 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" depends_on: - redis: + redis_test: condition: service_healthy user: node command: npm run test:acceptance @@ -40,7 +48,7 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index f4b701c3c1..c368a904c8 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -13,10 +13,18 @@ services: environment: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ${LOG_LEVEL:-} + MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test command: npm run --silent test:unit user: node + depends_on: + redis_test: + condition: service_healthy test_acceptance: image: node:22.18.0 @@ -27,10 +35,10 @@ services: working_dir: /overleaf/services/real-time environment: ELASTIC_SEARCH_DSN: es:9200 - REDIS_HOST: redis - HISTORY_REDIS_HOST: redis - QUEUES_REDIS_HOST: redis - ANALYTICS_QUEUES_REDIS_HOST: redis + REDIS_HOST: redis_test + HISTORY_REDIS_HOST: redis_test + QUEUES_REDIS_HOST: redis_test + ANALYTICS_QUEUES_REDIS_HOST: redis_test MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -39,11 +47,11 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" user: node depends_on: - redis: + redis_test: condition: service_healthy command: npm run --silent test:acceptance - redis: + redis_test: image: redis:7.4.3 healthcheck: test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] diff --git a/services/real-time/package.json b/services/real-time/package.json index a52e0dfcf9..adb4ce974b 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -7,7 +7,7 @@ "start": "node app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", diff --git a/services/web/app/src/infrastructure/RedisWrapper.js b/services/web/app/src/infrastructure/RedisWrapper.js index e08d3b8c4f..4c27328e9b 100644 --- a/services/web/app/src/infrastructure/RedisWrapper.js +++ b/services/web/app/src/infrastructure/RedisWrapper.js @@ -20,18 +20,7 @@ function client(feature) { async function cleanupTestRedis() { const rclient = client() - ensureTestRedis(rclient) - await rclient.flushall() -} - -function ensureTestRedis(rclient) { - const host = rclient.options.host - const env = process.env.NODE_ENV - if (host !== 'redis_test' || env !== 'test') { - throw new Error( - `Refusing to clear Redis instance '${host}' in environment '${env}'` - ) - } + await redis.cleanupTestRedis(rclient) } module.exports = { client, cleanupTestRedis } diff --git a/services/web/app/src/infrastructure/mongodb.js b/services/web/app/src/infrastructure/mongodb.js index 24103b2d82..b0bce8a846 100644 --- a/services/web/app/src/infrastructure/mongodb.js +++ b/services/web/app/src/infrastructure/mongodb.js @@ -1,6 +1,7 @@ const mongodb = require('mongodb-legacy') const OError = require('@overleaf/o-error') const Settings = require('@overleaf/settings') +const MongoUtils = require('@overleaf/mongo-utils') const Mongoose = require('./Mongoose') const { addConnectionDrainer } = require('./GracefulShutdown') @@ -93,32 +94,11 @@ async function getCollectionNames() { } async function cleanupTestDatabase() { - ensureTestDatabase() - const collectionNames = await getCollectionNames() - const collections = [] - for (const name of collectionNames) { - if (name in db && name !== 'migrations') { - collections.push(db[name]) - } - } - await Promise.all(collections.map(coll => coll.deleteMany({}))) + await MongoUtils.cleanupTestDatabase(mongoClient) } async function dropTestDatabase() { - ensureTestDatabase() - await mongoClient.db().dropDatabase() -} - -function ensureTestDatabase() { - const internalDb = mongoClient.db() - const dbName = internalDb.databaseName - const env = process.env.NODE_ENV - - if (dbName !== 'test-overleaf' || env !== 'test') { - throw new OError( - `Refusing to clear database '${dbName}' in environment '${env}'` - ) - } + await MongoUtils.dropTestDatabase(mongoClient) } /**