Merge pull request #34992 from overleaf/em-dropbox-rate-limits

[third-party-datastore] Handle Dropbox rate limits and per-project file-count limits during sync

GitOrigin-RevId: 0a3c4a06b6444a70394c35cb404c566ef7e25cef

services/web/test/unit/src/Cooldown/CooldownManager.sequential.test.mjs

@@ -1,50 +0,0 @@
-import { beforeEach, describe, expect, it } from 'vitest'
-import mongodb from 'mongodb-legacy'
-import CooldownManager from '../../../../app/src/Features/Cooldown/CooldownManager.mjs'
-import RedisWrapper from '../../../../app/src/infrastructure/RedisWrapper.mjs'
-
-const { ObjectId } = mongodb
-
-describe('CooldownManager', function () {
-  beforeEach(RedisWrapper.cleanupTestRedis)
-
-  beforeEach(function (ctx) {
-    ctx.project1Id = new ObjectId().toString()
-    ctx.project2Id = new ObjectId().toString()
-  })
-
-  describe('_buildKey', function () {
-    it('should build a properly formatted redis key', function () {
-      expect(CooldownManager._buildKey('ABC')).to.equal('Cooldown:{ABC}')
-    })
-  })
-
-  describe('isProjectOnCooldown', function () {
-    describe('when no project is on cooldown', function () {
-      it('returns false for project 1', async function (ctx) {
-        const result = await CooldownManager.isProjectOnCooldown(ctx.project1Id)
-        expect(result).to.be.false
-      })
-
-      it('returns false for project 2', async function (ctx) {
-        const result = await CooldownManager.isProjectOnCooldown(ctx.project2Id)
-        expect(result).to.be.false
-      })
-    })
-    describe('when project 1 is on cooldown', function () {
-      beforeEach(async function (ctx) {
-        await CooldownManager.putProjectOnCooldown(ctx.project1Id)
-      })
-
-      it('returns true for project 1', async function (ctx) {
-        const result = await CooldownManager.isProjectOnCooldown(ctx.project1Id)
-        expect(result).to.be.true
-      })
-
-      it('returns false for project 2', async function (ctx) {
-        const result = await CooldownManager.isProjectOnCooldown(ctx.project2Id)
-        expect(result).to.be.false
-      })
-    })
-  })
-})

services/web/test/unit/src/Editor/EditorHttpController.test.mjs

@@ -519,7 +519,7 @@ describe('EditorHttpController', function () {
 
       it('handle too many files', async function (ctx) {
         ctx.EditorController.promises.addDoc.rejects(
-          new Error('project_has_too_many_files')
+          new Errors.TooManyFilesError('project_has_too_many_files')
         )
         await new Promise(resolve => {
           ctx.res.callback = () => {
@@ -580,7 +580,7 @@ describe('EditorHttpController', function () {
       it('handle too many files', async function (ctx) {
         await new Promise(resolve => {
           ctx.EditorController.promises.addFolder.rejects(
-            new Error('project_has_too_many_files')
+            new Errors.TooManyFilesError('project_has_too_many_files')
           )
           ctx.res.callback = () => {
             expect(ctx.res.body).to.equal('"project_has_too_many_files_limit"')

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandler.test.mjs

@@ -73,7 +73,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
     }
 
     ctx.Settings = { maxEntitiesPerProject: 100 }
-    ctx.CooldownManager = {}
     ctx.LockManager = {
       promises: {
         runWithLock: sinon.spy((namespace, id, runner) => runner()),
@@ -201,10 +200,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
       default: ctx.Settings,
     }))
 
-    vi.doMock('../../../../app/src/Features/Cooldown/CooldownManager', () => ({
-      default: ctx.CooldownManager,
-    }))
-
     vi.doMock('../../../../app/src/models/Folder', () => ({
       Folder: ctx.FolderModel,
     }))

services/web/test/unit/src/ThirdPartyDataStore/TpdsController.test.mjs

@@ -59,6 +59,10 @@ describe('TpdsController', function () {
       },
     }
 
+    vi.doMock('../../../../app/src/Features/Errors/Errors.js', () => ({
+      default: Errors,
+    }))
+
     vi.doMock(
       '../../../../app/src/Features/ThirdPartyDataStore/TpdsUpdateHandler',
       () => ({
@@ -249,37 +253,16 @@ describe('TpdsController', function () {
       })
     })
 
-    it('should return a 400 error when the project is too big', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.TpdsUpdateHandler.promises.newUpdate.rejects({
-          message: 'project_has_too_many_files',
-        })
-        const res = {
-          sendStatus: status => {
-            expect(status).to.equal(400)
-            ctx.NotificationsBuilder.promises.tpdsFileLimit.should.have.been.calledWith(
-              ctx.user_id
-            )
-            resolve()
-          },
-        }
-        ctx.TpdsController.mergeUpdate(ctx.req, res)
-      })
-    })
-
-    it('should return a 429 error when the update receiver fails due to too many requests error', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.TpdsUpdateHandler.promises.newUpdate.rejects(
-          new Errors.TooManyRequestsError('project on cooldown')
-        )
-        const res = {
-          sendStatus: status => {
-            expect(status).to.equal(429)
-            resolve()
-          },
-        }
-        ctx.TpdsController.mergeUpdate(ctx.req, res)
-      })
+    it('should notify and rethrow when the project is too big', async function (ctx) {
+      const err = new Errors.TooManyFilesError('project has too many files')
+      ctx.TpdsUpdateHandler.promises.newUpdate.rejects(err)
+      const res = { json: sinon.stub() }
+      const next = sinon.stub()
+      await ctx.TpdsController.mergeUpdate(ctx.req, res, next)
+      expect(next).to.have.been.calledWith(err)
+      ctx.NotificationsBuilder.promises.tpdsFileLimit.should.have.been.calledWith(
+        ctx.user_id
+      )
     })
   })
 

services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateHandler.test.mjs

@@ -1,17 +1,12 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import sinon from 'sinon'
 import mongodb from 'mongodb-legacy'
-import Errors from '../../../../app/src/Features/Errors/Errors.js'
 
 const ObjectId = mongodb.ObjectId
 
 const MODULE_PATH =
   '../../../../app/src/Features/ThirdPartyDataStore/TpdsUpdateHandler.mjs'
 
-vi.mock('../../../../app/src/Features/Errors/Errors.js', () =>
-  vi.importActual('../../../../app/src/Features/Errors/Errors.js')
-)
-
 describe('TpdsUpdateHandler', function () {
   beforeEach(async function (ctx) {
     ctx.projectName = 'My recipes'
@@ -44,9 +39,6 @@ describe('TpdsUpdateHandler', function () {
       parentFolder_id: new ObjectId(),
     }
 
-    ctx.CooldownManager = {
-      isProjectOnCooldown: sinon.stub().resolves(false),
-    }
     ctx.FileTypeManager = {
       shouldIgnore: sinon.stub().returns(false),
     }
@@ -105,10 +97,6 @@ describe('TpdsUpdateHandler', function () {
       },
     }
 
-    vi.doMock('../../../../app/src/Features/Cooldown/CooldownManager', () => ({
-      default: ctx.CooldownManager,
-    }))
-
     vi.doMock('../../../../app/src/Features/Uploads/FileTypeManager', () => ({
       default: ctx.FileTypeManager,
     }))
@@ -255,24 +243,6 @@ describe('TpdsUpdateHandler', function () {
       expectUpdateNotProcessed()
       expectDropboxNotUnlinked()
     })
-
-    describe('update to a project on cooldown', async function () {
-      setupMatchingProjects(['active1'])
-      setupProjectOnCooldown()
-      beforeEach(async function (ctx) {
-        await expect(
-          ctx.TpdsUpdateHandler.promises.newUpdate(
-            ctx.userId,
-            '', // projectId
-            ctx.projectName,
-            ctx.path,
-            ctx.update,
-            ctx.source
-          )
-        ).to.be.rejectedWith(Errors.TooManyRequestsError)
-      })
-      expectUpdateNotProcessed()
-    })
   })
 
   describe('getting a file delete', function () {
@@ -453,22 +423,6 @@ describe('TpdsUpdateHandler', function () {
       expectFolderUpdateNotProcessed()
       expectDropboxUnlinked()
     })
-
-    describe('update to a project on cooldown', async function () {
-      setupMatchingProjects(['active1'])
-      setupProjectOnCooldown()
-      beforeEach(async function (ctx) {
-        await expect(
-          ctx.TpdsUpdateHandler.promises.createFolder(
-            ctx.userId,
-            ctx.projectId,
-            ctx.projectName,
-            ctx.path
-          )
-        ).to.be.rejectedWith(Errors.TooManyRequestsError)
-      })
-      expectFolderUpdateNotProcessed()
-    })
   })
 })
 
@@ -483,14 +437,6 @@ function setupMatchingProjects(projectKeys) {
   })
 }
 
-function setupProjectOnCooldown() {
-  beforeEach(function (ctx) {
-    ctx.CooldownManager.isProjectOnCooldown
-      .withArgs(ctx.projects.active1._id)
-      .resolves(true)
-  })
-}
-
 /* Test helpers */
 
 function receiveUpdate() {

services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs

@@ -13,6 +13,7 @@ import ArchiveErrors from '../../../../app/src/Features/Uploads/ArchiveErrors.mj
 import {
   FileTooLargeError,
   DocumentConversionError,
+  TooManyFilesError,
 } from '../../../../app/src/Features/Errors/Errors.js'
 
 vi.mock('../../../../app/src/Features/Errors/Errors.js', () =>
@@ -422,7 +423,7 @@ describe('ProjectUploadController', function () {
       beforeEach(function (ctx) {
         ctx.FileSystemImportManager.addEntity = sinon
           .stub()
-          .callsArgWith(6, new Error('project_has_too_many_files'))
+          .callsArgWith(6, new TooManyFilesError('project_has_too_many_files'))
         ctx.ProjectUploadController.uploadFile(ctx.req, ctx.res)
       })