agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 17:19:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #29775 from overleaf/bg-history-use-configurable-concurrency-in-backup-worker

Browse Source 
allow configurable concurrency for history-v1 backup worker

GitOrigin-RevId: 59c734b013f99e215cc84688142cb0fbe45b064b
This commit is contained in:
Brian Gough
2025-11-21 10:32:17 +00:00
committed by Copybot
parent e861e28296
commit e822ac0ee0
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
services/history-v1/storage/scripts/backup_worker.mjs
View File

@@ -8,7 +8,8 @@ import {
configureBackup,
} from './backup.mjs'
const CONCURRENCY = 15
const JOB_CONCURRENCY = parseInt(process.env.JOB_CONCURRENCY, 10) || 15
const UPLOAD_CONCURRENCY = parseInt(process.env.UPLOAD_CONCURRENCY, 10) || 50
const WARN_THRESHOLD = 2 * 60 * 60 * 1000 // warn if projects are older than this
const redisOptions = config.get('redis.queue')
const JOB_TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000] // milliseconds
@@ -17,7 +18,7 @@ const LAG_TIME_BUCKETS_HRS = [
] // hours
// Configure backup settings to match worker concurrency
configureBackup({ concurrency: 50, useSecondary: true })
configureBackup({ concurrency: UPLOAD_CONCURRENCY, useSecondary: true })
// Create a Bull queue named 'backup'
const backupQueue = new Queue('backup', {
@@ -69,7 +70,7 @@ backupQueue.on('resumed', () => {
})
// Process jobs
backupQueue.process(CONCURRENCY, async job => {
backupQueue.process(JOB_CONCURRENCY, async job => {
const { projectId, startDate, endDate } = job.data
if (projectId) {