diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs index 9835096695..7ecce86d6c 100644 --- a/services/history-v1/storage/scripts/backup.mjs +++ b/services/history-v1/storage/scripts/backup.mjs @@ -63,6 +63,7 @@ const RETRY_DELAY = 1000 let CONCURRENCY = 4 let BATCH_CONCURRENCY = 1 let BLOB_LIMITER = pLimit(CONCURRENCY) +let USE_SECONDARY = false /** * Configure backup settings @@ -74,6 +75,7 @@ export function configureBackup(options = {}) { CONCURRENCY = options.concurrency || 1 BATCH_CONCURRENCY = options.batchConcurrency || 1 BLOB_LIMITER = pLimit(CONCURRENCY) + USE_SECONDARY = options.useSecondary || false } let gracefulShutdownInitiated = false @@ -330,6 +332,11 @@ const optionDefinitions = [ type: String, description: 'End date for initialization (ISO format)', }, + { + name: 'use-secondary', + type: Boolean, + description: 'Use secondary read preference for backup status', + }, { name: 'compare', alias: 'C', @@ -387,6 +394,10 @@ function handleOptions() { process.exit(1) } + if (options['use-secondary']) { + USE_SECONDARY = true + } + if ( options.compare && !options.projectId && @@ -416,7 +427,9 @@ async function analyseBackupStatus(projectId) { await getBackupStatus(projectId) // TODO: when we have confidence that the latestChunkMetadata always matches // the values from the backupStatus we can skip loading it here - const latestChunkMetadata = await loadLatestRaw(historyId) + const latestChunkMetadata = await loadLatestRaw(historyId, { + readOnly: Boolean(USE_SECONDARY), + }) if ( currentEndVersion && currentEndVersion !== latestChunkMetadata.endVersion @@ -506,6 +519,9 @@ function makeChunkKey(projectId, startVersion) { } export async function backupProject(projectId, options) { + if (gracefulShutdownInitiated) { + return + } await ensureGlobalBlobsLoaded() // FIXME: flush the project first! // Let's assume the the flush happens externally and triggers this backup diff --git a/services/history-v1/storage/scripts/backup_worker.mjs b/services/history-v1/storage/scripts/backup_worker.mjs index 3e592e5ffb..a61931350d 100644 --- a/services/history-v1/storage/scripts/backup_worker.mjs +++ b/services/history-v1/storage/scripts/backup_worker.mjs @@ -13,11 +13,14 @@ const redisOptions = config.get('redis.queue') const TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000] // Configure backup settings to match worker concurrency -configureBackup({ concurrency: 5, batchConcurrency: 5 }) +configureBackup({ concurrency: 50, batchConcurrency: 3, useSecondary: true }) // Create a Bull queue named 'backup' const backupQueue = new Queue('backup', { redis: redisOptions, + settings: { + stalledInterval: 0, // don't check for stalled jobs + }, }) // Log queue events