From 581a3b7ce7146c16f77b2bd24a94bf0223f06bc5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 17 Nov 2025 14:38:09 +0000 Subject: [PATCH] Merge pull request #29700 from overleaf/bg-history-v1-log-stalled-uploads Add timeout warnings for long-running blob and chunk uploads GitOrigin-RevId: 5a57ebaaea9dfcaf23153f7e6f27190af541eb16 --- .../history-v1/storage/lib/backupBlob.mjs | 7 +++++ .../history-v1/storage/scripts/backup.mjs | 30 ++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/services/history-v1/storage/lib/backupBlob.mjs b/services/history-v1/storage/lib/backupBlob.mjs index 89625c6ed6..e5a4fc8289 100644 --- a/services/history-v1/storage/lib/backupBlob.mjs +++ b/services/history-v1/storage/lib/backupBlob.mjs @@ -74,6 +74,12 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) { let backupSource let contentEncoding let size + const timer = setTimeout(function () { + logger.warn( + { historyId, blob, path, size }, + 'blob upload still active after 1 minute' + ) + }, 60 * 1000) try { if (blob.getStringLength()) { backupSource = filePathCompressed @@ -109,6 +115,7 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) { } ) } finally { + clearTimeout(timer) if (backupSource === filePathCompressed) { try { await fs.promises.rm(filePathCompressed, { force: true }) diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs index f1019f5142..e044fc69b5 100644 --- a/services/history-v1/storage/scripts/backup.mjs +++ b/services/history-v1/storage/scripts/backup.mjs @@ -221,16 +221,26 @@ async function backupChunk( } const key = makeChunkKey(historyId, chunkToBackup.startVersion) logger.debug({ chunkRecord, historyId, projectId, key }, 'backing up chunk') - await chunkBackupPersistorForProject.sendStream( - chunksBucket, - makeChunkKey(historyId, chunkToBackup.startVersion), - Stream.Readable.from([chunkBuffer]), - { - contentType: 'application/json', - contentEncoding: 'gzip', - contentLength: chunkBuffer.byteLength, - } - ) + const timer = setTimeout(function () { + logger.warn( + { historyId, chunkRecord, size: chunkBuffer.byteLength }, + 'chunk upload still active after 1 minute' + ) + }, 60 * 1000) + try { + await chunkBackupPersistorForProject.sendStream( + chunksBucket, + makeChunkKey(historyId, chunkToBackup.startVersion), + Stream.Readable.from([chunkBuffer]), + { + contentType: 'application/json', + contentEncoding: 'gzip', + contentLength: chunkBuffer.byteLength, + } + ) + } finally { + clearTimeout(timer) + } } async function updateBackupStatus(