[document-updater] flush_all: log progress after every 1k projects (#33757)

GitOrigin-RevId: b5b68f6f53bece51234799fb626d0d6a2a5b590c

services/document-updater/app/js/ProjectFlusher.js

@@ -53,7 +53,7 @@ function _extractIds(keyList) {
 }
 
 async function flushAllProjects(options) {
-  logger.info({ options }, 'flushing all projects')
+  logger.debug({ options }, 'listing all projects with docs')
   const projectKeys = await _getKeys(
     docUpdaterKeys.docsInProject({ project_id: '*' }),
     options.limit
@@ -62,13 +62,21 @@ async function flushAllProjects(options) {
   if (options.dryRun) {
     return projectIds
   }
+  const total = projectIds.length
+  logger.info({ total, options }, 'flushing all projects')
+  let flushed = 0
   const results = await promiseMapSettledWithLimit(
     options.concurrency,
     projectIds,
-    projectId =>
-      ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, {
+    async projectId => {
+      await ProjectManager.promises.flushAndDeleteProjectWithLocks(projectId, {
         background: true,
       })
+      flushed++
+      if (options.logProgress && flushed % options.logProgress === 0) {
+        logger.info({ flushed, total }, 'Flush all projects progress')
+      }
+    }
   )
 
   const success = []

services/document-updater/scripts/flush_all.js

@@ -4,9 +4,10 @@ const minimist = require('minimist')
 async function main() {
   const argv = minimist(process.argv.slice(2), {
     default: {
-      limit: 100000,
+      limit: 100_000,
       concurrency: 5,
       'dry-run': false,
+      'log-progress': 1_000,
     },
     boolean: ['dry-run', 'help'],
     alias: { h: 'help', n: 'dry-run', j: 'concurrency' },
@@ -19,7 +20,8 @@ Usage: node scripts/flush_all.js [options]
 Options:
   --limit            Number of projects to flush (default: 100000)
   --concurrency, -j  Number of concurrent flush operations (default: 5)
-  --dryRun, -n       Perform a dry run without making any changes (default: false)
+  --dry-run, -n      Perform a dry run without making any changes (default: false)
+  --log-progress     Log progress after flushing every Nth project (default: 1000)
   --help, -h         Show this help message
     `)
     process.exit(0)
@@ -29,6 +31,7 @@ Options:
     limit: argv.limit,
     concurrency: argv.concurrency,
     dryRun: argv['dry-run'],
+    logProgress: argv['log-progress'],
   }
   console.log('Flushing all projects with options:', options)