From 7f2f7a769d4ee5b05b12ce61e15b5d4ac84b1f59 Mon Sep 17 00:00:00 2001
From: Brian Gough <briangough@users.noreply.github.com>
Date: Mon, 24 Feb 2025 14:51:08 +0000
Subject: [PATCH] Merge pull request #23834 from
 overleaf/bg-backup-graceful-shutdown

add graceful shutdown handling to backup script

GitOrigin-RevId: f0fb9a3867b26c7800edefab126fd05fbabdae02
---
 .../history-v1/storage/scripts/backup.mjs     | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs
index be548809c9..d704eff779 100644
--- a/services/history-v1/storage/scripts/backup.mjs
+++ b/services/history-v1/storage/scripts/backup.mjs
@@ -54,6 +54,16 @@ let CONCURRENCY = 4
 let BATCH_CONCURRENCY = 1
 let BLOB_LIMITER = pLimit(CONCURRENCY)
 
+let gracefulShutdownInitiated = false
+
+process.on('SIGINT', handleSignal)
+process.on('SIGTERM', handleSignal)
+
+function handleSignal() {
+  gracefulShutdownInitiated = true
+  console.warn('graceful shutdown initiated, draining queue')
+}
+
 async function retry(fn, times, delayMs) {
   let attempts = times
   while (attempts > 0) {
@@ -621,6 +631,9 @@ async function initializeProjects(options) {
   const limiter = pLimit(BATCH_CONCURRENCY)
 
   async function processBatch(batch) {
+    if (gracefulShutdownInitiated) {
+      throw new Error('graceful shutdown')
+    }
     const batchOperations = batch.map(project =>
       limiter(backupProject, project._id.toHexString(), options)
     )
@@ -651,6 +664,10 @@ async function initializeProjects(options) {
 async function backupPendingProjects(options) {
   const intervalMs = options.interval * 1000
   for await (const project of listPendingBackups(intervalMs)) {
+    if (gracefulShutdownInitiated) {
+      console.warn('graceful shutdown: stopping pending project backups')
+      break
+    }
     const projectId = project._id.toHexString()
     console.log(`Backing up pending project with ID: ${projectId}`)
     await backupProject(projectId, options)
@@ -838,6 +855,9 @@ async function compareAllProjects(options) {
   let totalProjects = 0
 
   async function processBatch(batch) {
+    if (gracefulShutdownInitiated) {
+      throw new Error('graceful shutdown')
+    }
     const batchOperations = batch.map(project =>
       limiter(async () => {
         const projectId = project._id.toHexString()
@@ -911,7 +931,9 @@ async function main() {
 
 main()
   .then(() => {
-    console.log('Completed')
+    console.log(
+      gracefulShutdownInitiated ? 'Exited - graceful shutdown' : 'Completed'
+    )
   })
   .catch(err => {
     console.error('Error backing up project:', err)