diff --git a/services/document-updater/app/js/ProjectFlusher.js b/services/document-updater/app/js/ProjectFlusher.js index bcfaf4b581..d6b8c5b133 100644 --- a/services/document-updater/app/js/ProjectFlusher.js +++ b/services/document-updater/app/js/ProjectFlusher.js @@ -53,7 +53,7 @@ function _extractIds(keyList) { } async function flushAllProjects(options) { - logger.info({ options }, 'flushing all projects') + logger.debug({ options }, 'listing all projects with docs') const projectKeys = await _getKeys( docUpdaterKeys.docsInProject({ project_id: '*' }), options.limit @@ -62,13 +62,21 @@ async function flushAllProjects(options) { if (options.dryRun) { return projectIds } + const total = projectIds.length + logger.info({ total, options }, 'flushing all projects') + let flushed = 0 const results = await promiseMapSettledWithLimit( options.concurrency, projectIds, - projectId => - ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, { + async projectId => { + await ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, { background: true, }) + flushed++ + if (options.logProgress && flushed % options.logProgress === 0) { + logger.info({ flushed, total }, 'Flush all projects progress') + } + } ) const success = [] diff --git a/services/document-updater/scripts/flush_all.js b/services/document-updater/scripts/flush_all.js index 3e359df651..42fad7b9a3 100644 --- a/services/document-updater/scripts/flush_all.js +++ b/services/document-updater/scripts/flush_all.js @@ -4,9 +4,10 @@ const minimist = require('minimist') async function main() { const argv = minimist(process.argv.slice(2), { default: { - limit: 100000, + limit: 100_000, concurrency: 5, 'dry-run': false, + 'log-progress': 1_000, }, boolean: ['dry-run', 'help'], alias: { h: 'help', n: 'dry-run', j: 'concurrency' }, @@ -19,7 +20,8 @@ Usage: node scripts/flush_all.js [options] Options: --limit Number of projects to flush (default: 100000) --concurrency, -j Number of concurrent flush operations (default: 5) - --dryRun, -n Perform a dry run without making any changes (default: false) + --dry-run, -n Perform a dry run without making any changes (default: false) + --log-progress Log progress after flushing every Nth project (default: 1000) --help, -h Show this help message `) process.exit(0) @@ -29,6 +31,7 @@ Options: limit: argv.limit, concurrency: argv.concurrency, dryRun: argv['dry-run'], + logProgress: argv['log-progress'], } console.log('Flushing all projects with options:', options)