diff --git a/package-lock.json b/package-lock.json index b7753ba14a..0431c8be69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35782,7 +35782,7 @@ "diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz", "express": "4.17.1", "lodash": "^4.17.21", - "mongodb": "^3.6.6", + "mongodb": "^4.11.0", "request": "^2.88.2", "requestretry": "^4.1.2" }, @@ -35816,6 +35816,40 @@ "node": ">= 0.8" } }, + "services/document-updater/node_modules/bson": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz", + "integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "services/document-updater/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "services/document-updater/node_modules/bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -35927,6 +35961,23 @@ "node": ">=4" } }, + "services/document-updater/node_modules/mongodb": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz", + "integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==", + "dependencies": { + "bson": "^4.7.0", + "mongodb-connection-string-url": "^2.5.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">=12.9.0" + }, + "optionalDependencies": { + "@aws-sdk/credential-providers": "^3.186.0", + "saslprep": "^1.0.3" + } + }, "services/document-updater/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -46853,7 +46904,7 @@ "express": "4.17.1", "lodash": "^4.17.21", "mocha": "^8.4.0", - "mongodb": "^3.6.6", + "mongodb": "^4.11.0", "request": "^2.88.2", "requestretry": "^4.1.2", "sandboxed-module": "^2.0.4", @@ -46878,6 +46929,23 @@ "type-is": "~1.6.17" } }, + "bson": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz", + "integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==", + "requires": { + "buffer": "^5.6.0" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -46968,6 +47036,18 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, + "mongodb": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz", + "integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==", + "requires": { + "@aws-sdk/credential-providers": "^3.186.0", + "bson": "^4.7.0", + "mongodb-connection-string-url": "^2.5.4", + "saslprep": "^1.0.3", + "socks": "^2.7.1" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", diff --git a/services/document-updater/app.js b/services/document-updater/app.js index e5a1cda19b..25f5a22781 100644 --- a/services/document-updater/app.js +++ b/services/document-updater/app.js @@ -225,8 +225,8 @@ const host = Settings.internal.documentupdater.host || 'localhost' if (!module.parent) { // Called directly - mongodb - .waitForDb() + mongodb.mongoClient + .connect() .then(() => { app.listen(port, host, function (err) { if (err) { diff --git a/services/document-updater/app/js/mongodb.js b/services/document-updater/app/js/mongodb.js index b265e99426..3509c80fe5 100644 --- a/services/document-updater/app/js/mongodb.js +++ b/services/document-updater/app/js/mongodb.js @@ -1,37 +1,25 @@ const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb') -const clientPromise = MongoClient.connect( - Settings.mongo.url, - Settings.mongo.options -) +const mongoClient = new MongoClient(Settings.mongo.url) +const mongoDb = mongoClient.db() + +const db = { + docs: mongoDb.collection('docs'), + docSnapshots: mongoDb.collection('docSnapshots'), + projects: mongoDb.collection('projects'), +} async function healthCheck() { - const internalDb = (await clientPromise).db() - const res = await internalDb.command({ ping: 1 }) + const res = await mongoDb.command({ ping: 1 }) if (!res.ok) { throw new Error('failed mongo ping') } } -let setupDbPromise -async function waitForDb() { - if (!setupDbPromise) { - setupDbPromise = setupDb() - } - await setupDbPromise -} - -const db = {} -async function setupDb() { - const internalDb = (await clientPromise).db() - - db.docSnapshots = internalDb.collection('docSnapshots') -} - module.exports = { db, ObjectId, + mongoClient, healthCheck: require('util').callbackify(healthCheck), - waitForDb, } diff --git a/services/document-updater/config/settings.defaults.js b/services/document-updater/config/settings.defaults.js index aef2565a56..6b8a2ae89b 100755 --- a/services/document-updater/config/settings.defaults.js +++ b/services/document-updater/config/settings.defaults.js @@ -166,10 +166,6 @@ module.exports = { dispatcherCount: parseInt(process.env.DISPATCHER_COUNT || 10, 10), mongo: { - options: { - useUnifiedTopology: - (process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true', - }, url: process.env.MONGO_CONNECTION_STRING || `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`, diff --git a/services/document-updater/package.json b/services/document-updater/package.json index 04680a54ac..9f6fe6b777 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -28,7 +28,7 @@ "diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz", "express": "4.17.1", "lodash": "^4.17.21", - "mongodb": "^3.6.6", + "mongodb": "^4.11.0", "request": "^2.88.2", "requestretry": "^4.1.2" }, diff --git a/services/document-updater/scripts/fix_docs_with_missing_project.js b/services/document-updater/scripts/fix_docs_with_missing_project.js index 1830c41057..0a9a05eaaa 100644 --- a/services/document-updater/scripts/fix_docs_with_missing_project.js +++ b/services/document-updater/scripts/fix_docs_with_missing_project.js @@ -4,25 +4,14 @@ const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.documentupdater ) const keys = Settings.redis.documentupdater.key_schema -const ProjectFlusher = require('app/js/ProjectFlusher') -const DocumentManager = require('app/js/DocumentManager') +const ProjectFlusher = require('../app/js/ProjectFlusher') +const DocumentManager = require('../app/js/DocumentManager') +const { mongoClient, db, ObjectId } = require('../app/js/mongodb') const util = require('util') const flushAndDeleteDocWithLock = util.promisify( DocumentManager.flushAndDeleteDocWithLock ) -const { MongoClient, ObjectId } = require('mongodb') - -const clientPromise = MongoClient.connect( - Settings.mongo.url, - Settings.mongo.options -) - -const db = {} -clientPromise.then(client => { - db.docs = client.db().collection('docs') -}) - async function fixDocsWithMissingProjectIds(dockeys, options) { const docIds = ProjectFlusher._extractIds(dockeys) for (const docId of docIds) { @@ -86,15 +75,13 @@ async function findAndProcessDocs(options) { } while (cursor !== '0') } -clientPromise.then(client => { - findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' }) - .then(result => { - rclient.quit() - client.close() - console.log('DONE') - }) - .catch(function (error) { - console.error(error) - process.exit(1) - }) -}) +findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' }) + .then(result => { + rclient.quit() + mongoClient.close() + console.log('DONE') + }) + .catch(function (error) { + console.error(error) + process.exit(1) + }) diff --git a/services/document-updater/scripts/remove_deleted_docs.js b/services/document-updater/scripts/remove_deleted_docs.js index 5e0393dd93..78965ac31b 100644 --- a/services/document-updater/scripts/remove_deleted_docs.js +++ b/services/document-updater/scripts/remove_deleted_docs.js @@ -4,28 +4,17 @@ const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.documentupdater ) const keys = Settings.redis.documentupdater.key_schema -const ProjectFlusher = require('app/js/ProjectFlusher') -const RedisManager = require('app/js/RedisManager') +const ProjectFlusher = require('../app/js/ProjectFlusher') +const RedisManager = require('../app/js/RedisManager') +const { mongoClient, db, ObjectId } = require('../app/js/mongodb') const util = require('util') const getDoc = util.promisify((projectId, docId, cb) => RedisManager.getDoc(projectId, docId, (err, ...args) => cb(err, args)) ) const removeDocFromMemory = util.promisify(RedisManager.removeDocFromMemory) -const { MongoClient, ObjectId } = require('mongodb') - -const clientPromise = MongoClient.connect( - Settings.mongo.url, - Settings.mongo.options -) const summary = { totalDocs: 0, deletedDocs: 0, skippedDocs: 0 } -const db = {} -clientPromise.then(client => { - db.docs = client.db().collection('docs') - db.projects = client.db().collection('projects') -}) - async function removeDeletedDocs(dockeys, options) { const docIds = ProjectFlusher._extractIds(dockeys) for (const docId of docIds) { @@ -159,15 +148,13 @@ async function findAndProcessDocs(options) { } while (cursor !== '0') } -clientPromise.then(client => { - findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' }) - .then(result => { - rclient.quit() - client.close() - console.log('DONE') - }) - .catch(function (error) { - console.error(error) - process.exit(1) - }) -}) +findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' }) + .then(result => { + rclient.quit() + mongoClient.close() + console.log('DONE') + }) + .catch(function (error) { + console.error(error) + process.exit(1) + }) diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js index 62b7f763a1..36517cd834 100644 --- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js +++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js @@ -9,7 +9,6 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const app = require('../../../../app') -const { waitForDb } = require('../../../../app/js/mongodb') require('@overleaf/logger').logger.level('fatal') module.exports = { @@ -27,20 +26,18 @@ module.exports = { } this.initing = true this.callbacks.push(callback) - waitForDb().then(() => { - return app.listen(3003, 'localhost', error => { - if (error != null) { - throw error + app.listen(3003, 'localhost', error => { + if (error != null) { + throw error + } + this.running = true + return (() => { + const result = [] + for (callback of Array.from(this.callbacks)) { + result.push(callback()) } - this.running = true - return (() => { - const result = [] - for (callback of Array.from(this.callbacks)) { - result.push(callback()) - } - return result - })() - }) + return result + })() }) }, }