From f70ef538889affc6a157a46438e75a539feff2bd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Mar 2025 11:20:27 +0000 Subject: [PATCH] Merge pull request #24053 from overleaf/bg-backup-queue-tweaks-II additional backup queue tweaks GitOrigin-RevId: b54e0c5f4a23bb9962c5062bd1b80f9f124c3c87 --- .../storage/scripts/backup_worker.mjs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/services/history-v1/storage/scripts/backup_worker.mjs b/services/history-v1/storage/scripts/backup_worker.mjs index a61931350d..9a28059dc2 100644 --- a/services/history-v1/storage/scripts/backup_worker.mjs +++ b/services/history-v1/storage/scripts/backup_worker.mjs @@ -19,7 +19,9 @@ configureBackup({ concurrency: 50, batchConcurrency: 3, useSecondary: true }) const backupQueue = new Queue('backup', { redis: redisOptions, settings: { - stalledInterval: 0, // don't check for stalled jobs + lockDuration: 15 * 60 * 1000, // 15 minutes + lockRenewTime: 60 * 1000, // 1 minute + maxStalledCount: 0, // mark stalled jobs as failed }, }) @@ -46,6 +48,22 @@ backupQueue.on('error', error => { logger.error({ error }, 'queue error') }) +backupQueue.on('stalled', job => { + logger.error({ job }, 'job has stalled') +}) + +backupQueue.on('lock-extension-failed', (job, err) => { + logger.error({ job, err }, 'lock extension failed') +}) + +backupQueue.on('paused', () => { + logger.info('queue paused') +}) + +backupQueue.on('resumed', () => { + logger.info('queue resumed') +}) + // Process jobs backupQueue.process(CONCURRENCY, async job => { const { projectId, startDate, endDate } = job.data