Merge pull request #35695 from overleaf/copilot/close-graceful-shutdown-issue-35694

Handle graceful shutdown in project and user deletion batch expiry

GitOrigin-RevId: 43182850ad8e55b480ea0073c393222366d7e00a

services/web/app/src/Features/Project/ProjectDeleter.mjs

@@ -120,8 +120,17 @@ async function expireDeletedProjectsAfterDuration() {
     'expiring batch of deleted projects'
   )
   try {
+    let processedCount = 0
     for (const projectId of projectIds) {
       await expireDeletedProject(projectId)
+      processedCount++
+      if (Settings.shuttingDown) {
+        logger.warn(
+          { projectCount: projectIds.length, processedCount },
+          'graceful shutdown in progress, stopping batch of deleted projects early'
+        )
+        return
+      }
     }
     logger.info(
       { projectCount: projectIds.length },

services/web/app/src/Features/User/UserDeleter.mjs

@@ -139,6 +139,13 @@ async function expireDeletedUsersAfterDuration() {
       await expireDeletedUser(deletedUserId)
       logger.info({ deletedUserId }, 'removing deleted user audit log entries')
       await UserAuditLogEntry.deleteMany({ userId: deletedUserId }).exec()
+      if (Settings.shuttingDown) {
+        logger.warn(
+          { userCount: deletedUsers.length, processedCount: i + 1 },
+          'graceful shutdown in progress, stopping batch of deleted users early'
+        )
+        return
+      }
     }
     logger.info(
       { deletedUsers: deletedUsers.length },

services/web/test/unit/src/Project/ProjectDeleter.test.mjs

@@ -18,6 +18,12 @@ describe('ProjectDeleter', function () {
     tk.freeze(Date.now())
     ctx.ip = '192.170.18.1'
     ctx.project = dummyProject()
+
+    ctx.settings = {
+      shuttingDown: false,
+      projectHardDeletionDelay: 1000 * 60 * 60 * 24 * 90, // 90 days
+    }
+    vi.doMock('@overleaf/settings', () => ({ default: ctx.settings }))
     ctx.user = {
       _id: '588f3ddae8ebc1bac07c9fa4',
       first_name: 'bjkdsjfk',
@@ -480,6 +486,76 @@ describe('ProjectDeleter', function () {
     })
   })
 
+  describe('expireDeletedProjectsAfterDuration when graceful shutdown is in progress', function () {
+    beforeEach(async function (ctx) {
+      ctx.settings.shuttingDown = true
+
+      ctx.DeletedProjectMock.expects('find')
+        .withArgs({
+          'deleterData.deletedAt': {
+            $lt: new Date(moment().subtract(90, 'days')),
+          },
+          project: {
+            $type: 'object',
+          },
+        })
+        .chain('exec')
+        .resolves(ctx.deletedProjects)
+
+      for (const deletedProject of ctx.deletedProjects) {
+        ctx.ProjectMock.expects('findById')
+          .withArgs(deletedProject.deleterData.deletedProjectId)
+          .atLeast(0)
+          .atMost(1)
+          .chain('exec')
+          .resolves(null)
+
+        ctx.DeletedProjectMock.expects('findOne')
+          .withArgs({
+            'deleterData.deletedProjectId': deletedProject.project._id,
+          })
+          .atLeast(0)
+          .atMost(1)
+          .chain('exec')
+          .resolves(deletedProject)
+
+        ctx.DeletedProjectMock.expects('updateOne')
+          .withArgs(
+            {
+              _id: deletedProject._id,
+            },
+            {
+              $set: {
+                'deleterData.deleterIpAddress': null,
+                project: null,
+              },
+            }
+          )
+          .atLeast(0)
+          .atMost(1)
+          .chain('exec')
+          .resolves()
+      }
+
+      await ctx.ProjectDeleter.promises.expireDeletedProjectsAfterDuration()
+    })
+
+    it('should stop processing after the first project deletion', function (ctx) {
+      expect(ctx.DocstoreManager.promises.destroyProject).to.have.been
+        .calledOnce
+    })
+
+    it('should log a warning about the early termination due to graceful shutdown', function (ctx) {
+      expect(ctx.logger.warn).toHaveBeenCalledWith(
+        expect.objectContaining({
+          projectCount: ctx.deletedProjects.length,
+          processedCount: 1,
+        }),
+        'graceful shutdown in progress, stopping batch of deleted projects early'
+      )
+    })
+  })
+
   describe('expireDeletedProject', function () {
     describe('on an inactive project', function () {
       beforeEach(async function (ctx) {

services/web/test/unit/src/User/UserDeleter.test.mjs

@@ -23,6 +23,13 @@ describe('UserDeleter', function () {
     ctx.UserMock = sinon.mock(User)
     ctx.DeletedUserMock = sinon.mock(DeletedUser)
 
+    ctx.settings = {
+      shuttingDown: false,
+      userHardDeletionDelay: 1000 * 60 * 60 * 24 * 90, // 90 days
+      overleaf: {}, // needed for Features.hasFeature('saas') to return true
+    }
+    vi.doMock('@overleaf/settings', () => ({ default: ctx.settings }))
+
     tk.freeze(Date.now())
 
     ctx.mockedUser = sinon.mock(
@@ -683,6 +690,65 @@ describe('UserDeleter', function () {
     })
   })
 
+  describe('expireDeletedUsersAfterDuration when graceful shutdown is in progress', function () {
+    const userId1 = new ObjectId()
+    const userId2 = new ObjectId()
+
+    beforeEach(async function (ctx) {
+      ctx.settings.shuttingDown = true
+
+      ctx.deletedUsers = [
+        {
+          user: { _id: userId1 },
+          deleterData: { deletedUserId: userId1 },
+          save: sinon.stub().resolves(),
+        },
+        {
+          user: { _id: userId2 },
+          deleterData: { deletedUserId: userId2 },
+          save: sinon.stub().resolves(),
+        },
+      ]
+
+      ctx.DeletedUserMock.expects('find')
+        .withArgs({
+          'deleterData.deletedAt': {
+            $lt: new Date(moment().subtract(90, 'days')),
+          },
+          user: {
+            $type: 'object',
+          },
+        })
+        .chain('exec')
+        .resolves(ctx.deletedUsers)
+
+      // Only set up findOne for the first user since shutdown stops after it
+      ctx.DeletedUserMock.expects('findOne')
+        .withArgs({
+          'deleterData.deletedUserId': userId1,
+        })
+        .chain('exec')
+        .resolves(ctx.deletedUsers[0])
+
+      await ctx.UserDeleter.promises.expireDeletedUsersAfterDuration()
+    })
+
+    it('should stop processing after the first user deletion', function (ctx) {
+      expect(ctx.deletedUsers[0].save).to.have.been.called
+      expect(ctx.deletedUsers[1].save).not.to.have.been.called
+    })
+
+    it('should log a warning about the early termination due to graceful shutdown', function (ctx) {
+      expect(ctx.logger.warn).toHaveBeenCalledWith(
+        expect.objectContaining({
+          userCount: ctx.deletedUsers.length,
+          processedCount: 1,
+        }),
+        'graceful shutdown in progress, stopping batch of deleted users early'
+      )
+    })
+  })
+
   describe('expireDeletedUser', function () {
     beforeEach(function (ctx) {
       ctx.mockedDeletedUser = sinon.mock(