diff --git a/services/history-v1/storage/lib/backupBlob.mjs b/services/history-v1/storage/lib/backupBlob.mjs index 89625c6ed6..e5a4fc8289 100644 --- a/services/history-v1/storage/lib/backupBlob.mjs +++ b/services/history-v1/storage/lib/backupBlob.mjs @@ -74,6 +74,12 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) { let backupSource let contentEncoding let size + const timer = setTimeout(function () { + logger.warn( + { historyId, blob, path, size }, + 'blob upload still active after 1 minute' + ) + }, 60 * 1000) try { if (blob.getStringLength()) { backupSource = filePathCompressed @@ -109,6 +115,7 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) { } ) } finally { + clearTimeout(timer) if (backupSource === filePathCompressed) { try { await fs.promises.rm(filePathCompressed, { force: true }) diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs index f1019f5142..e044fc69b5 100644 --- a/services/history-v1/storage/scripts/backup.mjs +++ b/services/history-v1/storage/scripts/backup.mjs @@ -221,16 +221,26 @@ async function backupChunk( } const key = makeChunkKey(historyId, chunkToBackup.startVersion) logger.debug({ chunkRecord, historyId, projectId, key }, 'backing up chunk') - await chunkBackupPersistorForProject.sendStream( - chunksBucket, - makeChunkKey(historyId, chunkToBackup.startVersion), - Stream.Readable.from([chunkBuffer]), - { - contentType: 'application/json', - contentEncoding: 'gzip', - contentLength: chunkBuffer.byteLength, - } - ) + const timer = setTimeout(function () { + logger.warn( + { historyId, chunkRecord, size: chunkBuffer.byteLength }, + 'chunk upload still active after 1 minute' + ) + }, 60 * 1000) + try { + await chunkBackupPersistorForProject.sendStream( + chunksBucket, + makeChunkKey(historyId, chunkToBackup.startVersion), + Stream.Readable.from([chunkBuffer]), + { + contentType: 'application/json', + contentEncoding: 'gzip', + contentLength: chunkBuffer.byteLength, + } + ) + } finally { + clearTimeout(timer) + } } async function updateBackupStatus(