Merge pull request #29700 from overleaf/bg-history-v1-log-stalled-uploads

Add timeout warnings for long-running blob and chunk uploads

GitOrigin-RevId: 5a57ebaaea9dfcaf23153f7e6f27190af541eb16

services/history-v1/storage/lib/backupBlob.mjs

@@ -74,6 +74,12 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) {
   let backupSource
   let contentEncoding
   let size
+  const timer = setTimeout(function () {
+    logger.warn(
+      { historyId, blob, path, size },
+      'blob upload still active after 1 minute'
+    )
+  }, 60 * 1000)
   try {
     if (blob.getStringLength()) {
       backupSource = filePathCompressed
@@ -109,6 +115,7 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) {
       }
     )
   } finally {
+    clearTimeout(timer)
     if (backupSource === filePathCompressed) {
       try {
         await fs.promises.rm(filePathCompressed, { force: true })

services/history-v1/storage/scripts/backup.mjs

@@ -221,16 +221,26 @@ async function backupChunk(
   }
   const key = makeChunkKey(historyId, chunkToBackup.startVersion)
   logger.debug({ chunkRecord, historyId, projectId, key }, 'backing up chunk')
-  await chunkBackupPersistorForProject.sendStream(
-    chunksBucket,
-    makeChunkKey(historyId, chunkToBackup.startVersion),
-    Stream.Readable.from([chunkBuffer]),
-    {
-      contentType: 'application/json',
-      contentEncoding: 'gzip',
-      contentLength: chunkBuffer.byteLength,
-    }
-  )
+  const timer = setTimeout(function () {
+    logger.warn(
+      { historyId, chunkRecord, size: chunkBuffer.byteLength },
+      'chunk upload still active after 1 minute'
+    )
+  }, 60 * 1000)
+  try {
+    await chunkBackupPersistorForProject.sendStream(
+      chunksBucket,
+      makeChunkKey(historyId, chunkToBackup.startVersion),
+      Stream.Readable.from([chunkBuffer]),
+      {
+        contentType: 'application/json',
+        contentEncoding: 'gzip',
+        contentLength: chunkBuffer.byteLength,
+      }
+    )
+  } finally {
+    clearTimeout(timer)
+  }
 }
 
 async function updateBackupStatus(