From 1f566c355674e41ee831f0131545cf580e8dcfd9 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Thu, 15 Dec 2022 07:53:03 -0500 Subject: [PATCH] Merge pull request #10911 from overleaf/em-upgrade-mongo-docstore Upgrade the Mongo driver in docstore GitOrigin-RevId: 35e5de558e7900fcb455f1044703912077273d47 --- package-lock.json | 119 +++++++++++++++--- services/docstore/app.js | 6 +- services/docstore/app/js/DocManager.js | 4 +- services/docstore/app/js/HealthChecker.js | 2 +- services/docstore/app/js/RangeManager.js | 2 +- services/docstore/app/js/mongodb.js | 32 +---- services/docstore/config/settings.defaults.js | 7 +- services/docstore/package.json | 6 +- .../test/acceptance/js/helpers/DocstoreApp.js | 26 ++-- services/docstore/test/setup.js | 2 +- .../test/unit/js/RangeManagerTests.js | 1 - 11 files changed, 129 insertions(+), 78 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a649d5e05..9481cadc4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36209,11 +36209,11 @@ "bunyan": "^1.8.15", "celebrate": "^13.0.4", "express": "^4.17.1", - "mongodb": "^3.6.0", + "lodash": "^4.17.21", + "mongodb": "^4.11.0", "p-map": "^4.0.0", "request": "^2.88.2", - "streamifier": "^0.1.1", - "underscore": "~1.12.1" + "streamifier": "^0.1.1" }, "devDependencies": { "@google-cloud/storage": "^5.1.2", @@ -36258,6 +36258,29 @@ "node": ">=10" } }, + "services/docstore/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "services/docstore/node_modules/celebrate": { "version": "13.0.4", "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz", @@ -36274,6 +36297,14 @@ "integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==", "dev": true }, + "services/docstore/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "services/docstore/node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -36335,6 +36366,35 @@ "node": ">=10.0.0" } }, + "services/docstore/node_modules/mongodb": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz", + "integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==", + "dependencies": { + "bson": "^4.7.0", + "denque": "^2.1.0", + "mongodb-connection-string-url": "^2.5.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">=12.9.0" + }, + "optionalDependencies": { + "@aws-sdk/credential-providers": "^3.186.0", + "saslprep": "^1.0.3" + } + }, + "services/docstore/node_modules/mongodb/node_modules/bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "services/docstore/node_modules/sinon": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz", @@ -36366,11 +36426,6 @@ "node": ">=8" } }, - "services/docstore/node_modules/underscore": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", - "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==" - }, "services/document-updater": { "name": "@overleaf/document-updater", "dependencies": { @@ -46560,15 +46615,15 @@ "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "express": "^4.17.1", + "lodash": "^4.17.21", "mocha": "^8.4.0", - "mongodb": "^3.6.0", + "mongodb": "^4.11.0", "p-map": "^4.0.0", "request": "^2.88.2", "sandboxed-module": "~2.0.4", "sinon": "~9.0.2", "sinon-chai": "^3.7.0", - "streamifier": "^0.1.1", - "underscore": "~1.12.1" + "streamifier": "^0.1.1" }, "dependencies": { "@google-cloud/storage": { @@ -46601,6 +46656,15 @@ "xdg-basedir": "^4.0.0" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "celebrate": { "version": "13.0.4", "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz", @@ -46617,6 +46681,11 @@ "integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==", "dev": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -46660,6 +46729,29 @@ "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", "dev": true }, + "mongodb": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz", + "integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==", + "requires": { + "@aws-sdk/credential-providers": "^3.186.0", + "bson": "^4.7.0", + "denque": "^2.1.0", + "mongodb-connection-string-url": "^2.5.4", + "saslprep": "^1.0.3", + "socks": "^2.7.1" + }, + "dependencies": { + "bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "requires": { + "buffer": "^5.6.0" + } + } + } + }, "sinon": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz", @@ -46683,11 +46775,6 @@ "requires": { "has-flag": "^4.0.0" } - }, - "underscore": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", - "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==" } } }, diff --git a/services/docstore/app.js b/services/docstore/app.js index b09b6d362b..b9aa78908a 100644 --- a/services/docstore/app.js +++ b/services/docstore/app.js @@ -15,7 +15,7 @@ const { Joi, errors: handleValidationErrors, } = require('celebrate') -const mongodb = require('./app/js/mongodb') +const { mongoClient } = require('./app/js/mongodb') const Errors = require('./app/js/Errors') const HttpController = require('./app/js/HttpController') @@ -101,8 +101,8 @@ const { host } = Settings.internal.docstore if (!module.parent) { // Called directly - mongodb - .waitForDb() + mongoClient + .connect() .then(() => { const server = app.listen(port, host, function (err) { if (err) { diff --git a/services/docstore/app/js/DocManager.js b/services/docstore/app/js/DocManager.js index 1f5aeee846..931ec9e9a7 100644 --- a/services/docstore/app/js/DocManager.js +++ b/services/docstore/app/js/DocManager.js @@ -15,7 +15,7 @@ let DocManager const MongoManager = require('./MongoManager') const Errors = require('./Errors') const logger = require('@overleaf/logger') -const _ = require('underscore') +const _ = require('lodash') const DocArchive = require('./DocArchiveManager') const RangeManager = require('./RangeManager') const Settings = require('@overleaf/settings') @@ -157,7 +157,7 @@ module.exports = DocManager = { ) return callback(err) } - doc = _.extend(doc, archivedDoc) + Object.assign(doc, archivedDoc) callback(null, doc) }) } diff --git a/services/docstore/app/js/HealthChecker.js b/services/docstore/app/js/HealthChecker.js index 6af4339b4b..808fa37906 100644 --- a/services/docstore/app/js/HealthChecker.js +++ b/services/docstore/app/js/HealthChecker.js @@ -12,7 +12,7 @@ const { db, ObjectId } = require('./mongodb') const request = require('request') const async = require('async') -const _ = require('underscore') +const _ = require('lodash') const crypto = require('crypto') const settings = require('@overleaf/settings') const { port } = settings.internal.docstore diff --git a/services/docstore/app/js/RangeManager.js b/services/docstore/app/js/RangeManager.js index 78ef59abd3..d872fad25b 100644 --- a/services/docstore/app/js/RangeManager.js +++ b/services/docstore/app/js/RangeManager.js @@ -12,7 +12,7 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ let RangeManager -const _ = require('underscore') +const _ = require('lodash') const { ObjectId } = require('./mongodb') module.exports = RangeManager = { diff --git a/services/docstore/app/js/mongodb.js b/services/docstore/app/js/mongodb.js index b49eb7b80b..c9453ae9f1 100644 --- a/services/docstore/app/js/mongodb.js +++ b/services/docstore/app/js/mongodb.js @@ -1,36 +1,16 @@ const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb') -const clientPromise = MongoClient.connect( - Settings.mongo.url, - Settings.mongo.options -) +const mongoClient = new MongoClient(Settings.mongo.url) +const mongoDb = mongoClient.db() -let setupDbPromise -async function waitForDb() { - if (!setupDbPromise) { - setupDbPromise = setupDb() - } - await setupDbPromise -} - -const db = {} -async function setupDb() { - const internalDb = (await clientPromise).db() - - db.docs = internalDb.collection('docs') - db.docOps = internalDb.collection('docOps') -} -async function addCollection(name) { - await waitForDb() - const internalDb = (await clientPromise).db() - - db[name] = internalDb.collection(name) +const db = { + docs: mongoDb.collection('docs'), + docOps: mongoDb.collection('docOps'), } module.exports = { db, + mongoClient, ObjectId, - addCollection, - waitForDb, } diff --git a/services/docstore/config/settings.defaults.js b/services/docstore/config/settings.defaults.js index d69c9befd5..5b549c2cee 100644 --- a/services/docstore/config/settings.defaults.js +++ b/services/docstore/config/settings.defaults.js @@ -9,12 +9,7 @@ const Settings = { }, }, - mongo: { - options: { - useUnifiedTopology: - (process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true', - }, - }, + mongo: {}, docstore: { archiveOnSoftDelete: process.env.ARCHIVE_ON_SOFT_DELETE === 'true', diff --git a/services/docstore/package.json b/services/docstore/package.json index e61c5ed1a7..7083a60677 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -27,11 +27,11 @@ "bunyan": "^1.8.15", "celebrate": "^13.0.4", "express": "^4.17.1", - "mongodb": "^3.6.0", + "lodash": "^4.17.21", + "mongodb": "^4.11.0", "p-map": "^4.0.0", "request": "^2.88.2", - "streamifier": "^0.1.1", - "underscore": "~1.12.1" + "streamifier": "^0.1.1" }, "devDependencies": { "@google-cloud/storage": "^5.1.2", diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js index 0ce3e570b5..f6f11932b8 100644 --- a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js +++ b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js @@ -1,5 +1,4 @@ const app = require('../../../../app') -const { waitForDb } = require('../../../../app/js/mongodb') require('@overleaf/logger').logger.level('error') const settings = require('@overleaf/settings') @@ -8,9 +7,6 @@ module.exports = { initing: false, callbacks: [], ensureRunning(callback) { - if (callback == null) { - callback = function () {} - } if (this.running) { return callback() } else if (this.initing) { @@ -18,20 +14,14 @@ module.exports = { } this.initing = true this.callbacks.push(callback) - waitForDb().then(() => { - return app.listen(settings.internal.docstore.port, 'localhost', error => { - if (error != null) { - throw error - } - this.running = true - return (() => { - const result = [] - for (callback of Array.from(this.callbacks)) { - result.push(callback()) - } - return result - })() - }) + app.listen(settings.internal.docstore.port, 'localhost', error => { + if (error != null) { + throw error + } + this.running = true + for (callback of Array.from(this.callbacks)) { + callback() + } }) }, } diff --git a/services/docstore/test/setup.js b/services/docstore/test/setup.js index 9fef936003..9358e01c7e 100644 --- a/services/docstore/test/setup.js +++ b/services/docstore/test/setup.js @@ -30,7 +30,7 @@ SandboxedModule.configure({ requires: { '@overleaf/logger': stubs.logger, }, - globals: { Buffer, JSON, console, process }, + globals: { Buffer, JSON, Math, console, process }, }) exports.mochaHooks = { diff --git a/services/docstore/test/unit/js/RangeManagerTests.js b/services/docstore/test/unit/js/RangeManagerTests.js index 271d9daa69..db7b15783a 100644 --- a/services/docstore/test/unit/js/RangeManagerTests.js +++ b/services/docstore/test/unit/js/RangeManagerTests.js @@ -18,7 +18,6 @@ const modulePath = require('path').join( '../../../app/js/RangeManager' ) const { ObjectId } = require('mongodb') -const _ = require('underscore') describe('RangeManager', function () { beforeEach(function () {