agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 09:09:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

[document-updater] flush_all: log progress after every 1k projects (#33757)

Browse Source 
GitOrigin-RevId: b5b68f6f53bece51234799fb626d0d6a2a5b590c
This commit is contained in:
Jakob Ackermann
2026-05-18 12:47:04 +02:00
committed by Copybot
parent 293d89a4cb
commit 1f8371e0a3
2 changed files with 16 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
services/document-updater/app/js/ProjectFlusher.js
View File

@@ -53,7 +53,7 @@ function _extractIds(keyList) {
} }
async function flushAllProjects(options) { async function flushAllProjects(options) {
logger.info({ options }, 'flushing all projects') logger.debug({ options }, 'listing all projects with docs')
const projectKeys = await _getKeys( const projectKeys = await _getKeys(
docUpdaterKeys.docsInProject({ project_id: '*' }), docUpdaterKeys.docsInProject({ project_id: '*' }),
options.limit options.limit
@@ -62,13 +62,21 @@ async function flushAllProjects(options) {
if (options.dryRun) { if (options.dryRun) {
return projectIds return projectIds
} }
const total = projectIds.length
logger.info({ total, options }, 'flushing all projects')
let flushed = 0
const results = await promiseMapSettledWithLimit( const results = await promiseMapSettledWithLimit(
options.concurrency, options.concurrency,
projectIds, projectIds,
projectId => async projectId => {
ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, { await ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, {
background: true, background: true,
}) })
flushed++
if (options.logProgress && flushed % options.logProgress === 0) {
logger.info({ flushed, total }, 'Flush all projects progress')
}
}
) )
const success = [] const success = []

7
services/document-updater/scripts/flush_all.js
View File

@@ -4,9 +4,10 @@ const minimist = require('minimist')
async function main() { async function main() {
const argv = minimist(process.argv.slice(2), { const argv = minimist(process.argv.slice(2), {
default: { default: {
limit: 100000, limit: 100_000,
concurrency: 5, concurrency: 5,
'dry-run': false, 'dry-run': false,
'log-progress': 1_000,
}, },
boolean: ['dry-run', 'help'], boolean: ['dry-run', 'help'],
alias: { h: 'help', n: 'dry-run', j: 'concurrency' }, alias: { h: 'help', n: 'dry-run', j: 'concurrency' },
@@ -19,7 +20,8 @@ Usage: node scripts/flush_all.js [options]
Options: Options:
--limit Number of projects to flush (default: 100000) --limit Number of projects to flush (default: 100000)
--concurrency, -j Number of concurrent flush operations (default: 5) --concurrency, -j Number of concurrent flush operations (default: 5)
--dryRun, -n Perform a dry run without making any changes (default: false) --dry-run, -n Perform a dry run without making any changes (default: false)
--log-progress Log progress after flushing every Nth project (default: 1000)
--help, -h Show this help message --help, -h Show this help message
`) `)
process.exit(0) process.exit(0)
@@ -29,6 +31,7 @@ Options:
limit: argv.limit, limit: argv.limit,
concurrency: argv.concurrency, concurrency: argv.concurrency,
dryRun: argv['dry-run'], dryRun: argv['dry-run'],
logProgress: argv['log-progress'],
} }
console.log('Flushing all projects with options:', options) console.log('Flushing all projects with options:', options)