From acaaa7c205c6e6a22f33be5f5a2a23a15c0c6c63 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 11 Oct 2024 16:13:44 +0100 Subject: [PATCH] Merge pull request #20679 from overleaf/bg-issue19022 Move flush_all_projects endpoint in document-updater into script GitOrigin-RevId: cb774d860b5928b7fece1a8e21b0b76aecae73ff --- package-lock.json | 2 ++ services/document-updater/app.js | 1 - .../document-updater/app/js/HttpController.js | 19 ----------- services/document-updater/package.json | 1 + .../document-updater/scripts/flush_all.js | 34 ++++++++++++++++--- .../js/HttpController/HttpControllerTests.js | 1 - 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 86c39f2e4e..b51246dcf7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38056,6 +38056,7 @@ "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688", "express": "^4.21.0", "lodash": "^4.17.21", + "minimist": "^1.2.8", "mongodb-legacy": "^6.0.1", "request": "^2.88.2", "requestretry": "^7.1.0" @@ -48796,6 +48797,7 @@ "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688", "express": "^4.21.0", "lodash": "^4.17.21", + "minimist": "^1.2.8", "mocha": "^10.2.0", "mongodb-legacy": "^6.0.1", "request": "^2.88.2", diff --git a/services/document-updater/app.js b/services/document-updater/app.js index 2b203afaf2..fab33150c3 100644 --- a/services/document-updater/app.js +++ b/services/document-updater/app.js @@ -187,7 +187,6 @@ app.delete( app.post('/project/:project_id/block', HttpController.blockProject) app.post('/project/:project_id/unblock', HttpController.unblockProject) -app.get('/flush_all_projects', HttpController.flushAllProjects) app.get('/flush_queued_projects', HttpController.flushQueuedProjects) app.get('/total', (req, res, next) => { diff --git a/services/document-updater/app/js/HttpController.js b/services/document-updater/app/js/HttpController.js index 28411061dc..a3c7b6cd44 100644 --- a/services/document-updater/app/js/HttpController.js +++ b/services/document-updater/app/js/HttpController.js @@ -6,7 +6,6 @@ const Errors = require('./Errors') const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') const Metrics = require('./Metrics') -const ProjectFlusher = require('./ProjectFlusher') const DeleteQueueManager = require('./DeleteQueueManager') const { getTotalSizeOfLines } = require('./Limits') const async = require('async') @@ -409,23 +408,6 @@ function resyncProjectHistory(req, res, next) { ) } -function flushAllProjects(req, res, next) { - res.setTimeout(5 * 60 * 1000) - const options = { - limit: req.query.limit || 1000, - concurrency: req.query.concurrency || 5, - dryRun: req.query.dryRun || false, - } - ProjectFlusher.flushAllProjects(options, (err, projectIds) => { - if (err) { - logger.err({ err }, 'error bulk flushing projects') - res.sendStatus(500) - } else { - res.send(projectIds) - } - }) -} - function flushQueuedProjects(req, res, next) { res.setTimeout(10 * 60 * 1000) const options = { @@ -490,7 +472,6 @@ module.exports = { deleteComment, updateProject, resyncProjectHistory, - flushAllProjects, flushQueuedProjects, blockProject, unblockProject, diff --git a/services/document-updater/package.json b/services/document-updater/package.json index faf23b740f..bf49f53b21 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -32,6 +32,7 @@ "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688", "express": "^4.21.0", "lodash": "^4.17.21", + "minimist": "^1.2.8", "mongodb-legacy": "^6.0.1", "request": "^2.88.2", "requestretry": "^7.1.0" diff --git a/services/document-updater/scripts/flush_all.js b/services/document-updater/scripts/flush_all.js index 88a180bfbb..68bd831362 100644 --- a/services/document-updater/scripts/flush_all.js +++ b/services/document-updater/scripts/flush_all.js @@ -1,12 +1,38 @@ const ProjectFlusher = require('../app/js/ProjectFlusher') +const minimist = require('minimist') async function main() { - console.log('Flushing all projects') - return await new Promise((resolve, reject) => { - const options = { + const argv = minimist(process.argv.slice(2), { + default: { limit: 100000, concurrency: 5, - } + 'dry-run': false, + }, + boolean: ['dry-run', 'help'], + alias: { h: 'help', n: 'dry-run', j: 'concurrency' }, + }) + + if (argv.help) { + console.log(` +Usage: node scripts/flush_all.js [options] + +Options: + --limit Number of projects to flush (default: 100000) + --concurrency, -j Number of concurrent flush operations (default: 5) + --dryRun, -n Perform a dry run without making any changes (default: false) + --help, -h Show this help message + `) + process.exit(0) + } + + const options = { + limit: argv.limit, + concurrency: argv.concurrency, + dryRun: argv['dry-run'], + } + console.log('Flushing all projects with options:', options) + + return await new Promise((resolve, reject) => { ProjectFlusher.flushAllProjects(options, err => { if (err) { reject(err) diff --git a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js index 024c5a5f63..c422b8c2c0 100644 --- a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js +++ b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js @@ -13,7 +13,6 @@ describe('HttpController', function () { }), './ProjectHistoryRedisManager': (this.ProjectHistoryRedisManager = {}), './ProjectManager': (this.ProjectManager = {}), - './ProjectFlusher': { flushAllProjects() {} }, './DeleteQueueManager': (this.DeleteQueueManager = {}), './RedisManager': (this.RedisManager = { DOC_OPS_TTL: 42,