diff --git a/libraries/metrics/index.js b/libraries/metrics/index.js index 460e5967de..43d715b37e 100644 --- a/libraries/metrics/index.js +++ b/libraries/metrics/index.js @@ -251,4 +251,5 @@ module.exports.http = require('./http') module.exports.open_sockets = require('./open_sockets') module.exports.event_loop = require('./event_loop') module.exports.memory = require('./memory') +module.exports.mongodb = require('./mongodb') module.exports.timeAsyncMethod = require('./timeAsyncMethod') diff --git a/libraries/metrics/mongodb.js b/libraries/metrics/mongodb.js new file mode 100644 index 0000000000..0894ddf785 --- /dev/null +++ b/libraries/metrics/mongodb.js @@ -0,0 +1,54 @@ +const { Gauge } = require('prom-client') + +function monitor(mongoClient) { + const labelNames = ['mongo_server'] + const poolSize = new Gauge({ + name: 'mongo_connection_pool_size', + help: 'number of connections in the connection pool', + labelNames, + // Use this one metric's collect() to set all metrics' values. + collect, + }) + const availableConnections = new Gauge({ + name: 'mongo_connection_pool_available', + help: 'number of connections that are not busy', + labelNames, + }) + const waitQueueSize = new Gauge({ + name: 'mongo_connection_pool_waiting', + help: 'number of operations waiting for an available connection', + labelNames, + }) + const maxPoolSize = new Gauge({ + name: 'mongo_connection_pool_max', + help: 'max size for the connection pool', + labelNames, + }) + + function collect() { + // Reset all gauges in case they contain values for servers that + // disappeared + poolSize.reset() + availableConnections.reset() + waitQueueSize.reset() + maxPoolSize.reset() + + const servers = mongoClient.topology?.s?.servers + if (servers != null) { + for (const [address, server] of servers) { + const pool = server.s?.pool + if (pool == null) { + continue + } + + const labels = { mongo_server: address } + poolSize.set(labels, pool.totalConnectionCount) + availableConnections.set(labels, pool.availableConnectionCount) + waitQueueSize.set(labels, pool.waitQueueSize) + maxPoolSize.set(labels, pool.options.maxPoolSize) + } + } + } +} + +module.exports = { monitor } diff --git a/libraries/metrics/test/unit/js/mongodb.js b/libraries/metrics/test/unit/js/mongodb.js new file mode 100644 index 0000000000..f74ece4d43 --- /dev/null +++ b/libraries/metrics/test/unit/js/mongodb.js @@ -0,0 +1,89 @@ +const Metrics = require('../../..') + +const { expect } = require('chai') +const prom = require('prom-client') + +describe('mongodb', function () { + beforeEach(function () { + prom.register.clear() + this.pool = { + totalConnectionCount: 8, + availableConnectionCount: 2, + waitQueueSize: 4, + options: { maxPoolSize: 10 }, + } + this.servers = new Map([['server1', { s: { pool: this.pool } }]]) + + this.mongoClient = { topology: { s: { servers: this.servers } } } + }) + + it('handles an unconnected client', async function () { + const mongoClient = {} + Metrics.mongodb.monitor(mongoClient) + const metrics = await getMetrics() + expect(metrics).to.deep.equal({}) + }) + + it('collects Mongo metrics', async function () { + Metrics.mongodb.monitor(this.mongoClient) + const metrics = await getMetrics() + expect(metrics).to.deep.equal({ + 'mongo_connection_pool_max:server1': 10, + 'mongo_connection_pool_size:server1': 8, + 'mongo_connection_pool_available:server1': 2, + 'mongo_connection_pool_waiting:server1': 4, + }) + }) + + it('handles topology changes', async function () { + Metrics.mongodb.monitor(this.mongoClient) + let metrics = await getMetrics() + expect(metrics).to.deep.equal({ + 'mongo_connection_pool_max:server1': 10, + 'mongo_connection_pool_size:server1': 8, + 'mongo_connection_pool_available:server1': 2, + 'mongo_connection_pool_waiting:server1': 4, + }) + + // Add a server + this.servers.set('server2', this.servers.get('server1')) + metrics = await getMetrics() + expect(metrics).to.deep.equal({ + 'mongo_connection_pool_max:server1': 10, + 'mongo_connection_pool_size:server1': 8, + 'mongo_connection_pool_available:server1': 2, + 'mongo_connection_pool_waiting:server1': 4, + 'mongo_connection_pool_max:server2': 10, + 'mongo_connection_pool_size:server2': 8, + 'mongo_connection_pool_available:server2': 2, + 'mongo_connection_pool_waiting:server2': 4, + }) + + // Delete a server + this.servers.delete('server1') + metrics = await getMetrics() + expect(metrics).to.deep.equal({ + 'mongo_connection_pool_max:server2': 10, + 'mongo_connection_pool_size:server2': 8, + 'mongo_connection_pool_available:server2': 2, + 'mongo_connection_pool_waiting:server2': 4, + }) + + // Delete another server + this.servers.delete('server2') + metrics = await getMetrics() + expect(metrics).to.deep.equal({}) + }) +}) + +async function getMetrics() { + const metrics = await prom.register.getMetricsAsJSON() + const result = {} + for (const metric of metrics) { + for (const value of metric.values) { + const key = `${metric.name}:${value.labels.mongo_server}` + result[key] = value.value + } + } + return result +} diff --git a/services/chat/app/js/mongodb.js b/services/chat/app/js/mongodb.js index 927ba0e4a2..2472fa3cf4 100644 --- a/services/chat/app/js/mongodb.js +++ b/services/chat/app/js/mongodb.js @@ -1,3 +1,4 @@ +import Metrics from '@overleaf/metrics' import Settings from '@overleaf/settings' import { MongoClient } from 'mongodb' @@ -10,3 +11,5 @@ export const db = { messages: mongoDb.collection('messages'), rooms: mongoDb.collection('rooms'), } + +Metrics.mongodb.monitor(mongoClient) diff --git a/services/contacts/app/js/mongodb.js b/services/contacts/app/js/mongodb.js index 93a3bfbd41..eade7160c9 100644 --- a/services/contacts/app/js/mongodb.js +++ b/services/contacts/app/js/mongodb.js @@ -1,12 +1,14 @@ +import Metrics from '@overleaf/metrics' import Settings from '@overleaf/settings' import { MongoClient } from 'mongodb' export { ObjectId } from 'mongodb' export const mongoClient = new MongoClient(Settings.mongo.url) - const mongoDb = mongoClient.db() export const db = { contacts: mongoDb.collection('contacts'), } + +Metrics.mongodb.monitor(mongoClient) diff --git a/services/docstore/app/js/mongodb.js b/services/docstore/app/js/mongodb.js index c9453ae9f1..8daf41fb98 100644 --- a/services/docstore/app/js/mongodb.js +++ b/services/docstore/app/js/mongodb.js @@ -1,3 +1,4 @@ +const Metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb') @@ -9,6 +10,8 @@ const db = { docOps: mongoDb.collection('docOps'), } +Metrics.mongodb.monitor(mongoClient) + module.exports = { db, mongoClient, diff --git a/services/document-updater/app/js/mongodb.js b/services/document-updater/app/js/mongodb.js index 3509c80fe5..52868d45c8 100644 --- a/services/document-updater/app/js/mongodb.js +++ b/services/document-updater/app/js/mongodb.js @@ -1,3 +1,4 @@ +const Metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb') @@ -17,6 +18,8 @@ async function healthCheck() { } } +Metrics.mongodb.monitor(mongoClient) + module.exports = { db, ObjectId, diff --git a/services/history-v1/storage/lib/mongodb.js b/services/history-v1/storage/lib/mongodb.js index 4e6d098d2d..53b1837a8f 100644 --- a/services/history-v1/storage/lib/mongodb.js +++ b/services/history-v1/storage/lib/mongodb.js @@ -1,3 +1,5 @@ +const Metrics = require('@overleaf/metrics') + const config = require('config') const { MongoClient } = require('mongodb') @@ -9,4 +11,6 @@ const blobs = db.collection('projectHistoryBlobs') const globalBlobs = db.collection('projectHistoryGlobalBlobs') const shardedBlobs = db.collection('projectHistoryShardedBlobs') +Metrics.mongodb.monitor(client) + module.exports = { client, db, chunks, blobs, globalBlobs, shardedBlobs } diff --git a/services/notifications/app/js/mongodb.js b/services/notifications/app/js/mongodb.js index 3119bf2478..ff20d299e4 100644 --- a/services/notifications/app/js/mongodb.js +++ b/services/notifications/app/js/mongodb.js @@ -1,3 +1,4 @@ +const Metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') const { MongoClient, ObjectId } = require('mongodb') @@ -8,6 +9,8 @@ const db = { notifications: mongoDb.collection('notifications'), } +Metrics.mongodb.monitor(mongoClient) + module.exports = { db, mongoClient, diff --git a/services/project-history/app/js/mongodb.js b/services/project-history/app/js/mongodb.js index eb7814bc59..845b7947f2 100644 --- a/services/project-history/app/js/mongodb.js +++ b/services/project-history/app/js/mongodb.js @@ -1,3 +1,4 @@ +import Metrics from '@overleaf/metrics' import Settings from '@overleaf/settings' import { MongoClient } from 'mongodb' @@ -6,6 +7,8 @@ export { ObjectId } from 'mongodb' export const mongoClient = new MongoClient(Settings.mongo.url) const mongoDb = mongoClient.db() +Metrics.mongodb.monitor(mongoClient) + export const db = { deletedProjects: mongoDb.collection('deletedProjects'), projects: mongoDb.collection('projects'), diff --git a/services/web/app/src/infrastructure/mongodb.js b/services/web/app/src/infrastructure/mongodb.js index 9d27f96b2b..5316060aed 100644 --- a/services/web/app/src/infrastructure/mongodb.js +++ b/services/web/app/src/infrastructure/mongodb.js @@ -1,9 +1,6 @@ const Metrics = require('@overleaf/metrics') const { ObjectId } = require('mongodb') const OError = require('@overleaf/o-error') -const { - addOptionalCleanupHandlerAfterDrainingConnections, -} = require('./GracefulShutdown') const { getNativeDb } = require('./Mongoose') if ( @@ -26,7 +23,7 @@ async function waitForDb() { const db = {} async function setupDb() { const internalDb = await getNativeDb() - collectStatsForDb(internalDb, 'shared') + Metrics.mongodb.monitor(internalDb) db.contacts = internalDb.collection('contacts') db.deletedFiles = internalDb.collection('deletedFiles') @@ -110,34 +107,6 @@ async function getCollectionInternal(name) { return internalDb.collection(name) } -function collectStatsForDb(internalDb, label) { - const collectOnce = () => { - for (const [name, server] of internalDb.s.topology.s.servers.entries()) { - const { availableConnectionCount, waitQueueSize } = server.s.pool - const { maxPoolSize } = server.s.pool.options - const opts = { status: `${label}:${name}` } - Metrics.gauge('mongo_connection_pool_max', maxPoolSize, 1, opts) - Metrics.gauge( - 'mongo_connection_pool_available', - availableConnectionCount, - 1, - opts - ) - Metrics.gauge('mongo_connection_pool_waiting', waitQueueSize, 1, opts) - } - } - collectOnce() // init metrics - const intervalHandle = setInterval(collectOnce, 60_000) - addOptionalCleanupHandlerAfterDrainingConnections( - `collect mongo connection pool metrics (${label})`, - () => { - clearInterval(intervalHandle) - // collect one more time. - collectOnce() - } - ) -} - module.exports = { db, ObjectId, @@ -145,5 +114,4 @@ module.exports = { getCollectionInternal, dropTestDatabase, waitForDb, - collectStatsForDb, }