Merge pull request #28628 from overleaf/bg-cache-history-size-on-project

Implement project size checks on file uploads

GitOrigin-RevId: 4dffe237e6992b859f07964cffa82ff1c13e91c9

services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs

@@ -15,18 +15,60 @@ class MockV1HistoryApi extends AbstractMockApi {
     this.blobs = {}
   }
 
+  computeBlobStats(historyId, blobHashes) {
+    let textBlobBytes = 0
+    let binaryBlobBytes = 0
+    let nTextBlobs = 0
+    let nBinaryBlobs = 0
+    if (!blobHashes) {
+      blobHashes = this.blobs[historyId]
+        ? Object.keys(this.blobs[historyId])
+        : []
+    }
+    for (const hash of blobHashes) {
+      const buf = this.blobs[historyId][hash]
+      if (buf) {
+        const size = buf.byteLength
+
+        // Check if the blob content is valid UTF-8
+        let isText = false
+        try {
+          const decoder = new TextDecoder('utf-8', { fatal: true })
+          decoder.decode(buf)
+          isText = true
+        } catch (e) {
+          // Not valid UTF-8, treat as binary
+          isText = false
+        }
+
+        if (isText) {
+          textBlobBytes += size
+          nTextBlobs++
+        } else {
+          binaryBlobBytes += size
+          nBinaryBlobs++
+        }
+      }
+    }
+
+    const totalBytes = textBlobBytes + binaryBlobBytes
+
+    return {
+      projectId: historyId,
+      textBlobBytes,
+      binaryBlobBytes,
+      totalBytes,
+      nTextBlobs,
+      nBinaryBlobs,
+    }
+  }
+
   applyRoutes() {
     this.app.post('/api/projects/blob-stats', (req, res, next) => {
       res.json(
+        // Calculate actual sizes from uploaded blobs
         req.body.projectIds.map(projectId => {
-          return {
-            projectId,
-            textBlobBytes: 7331,
-            binaryBlobBytes: 1337,
-            totalBytes: 7331 + 1337,
-            nTextBlobs: 13,
-            nBinaryBlobs: 42,
-          }
+          return this.computeBlobStats(projectId)
         })
       )
     })
@@ -34,51 +76,9 @@ class MockV1HistoryApi extends AbstractMockApi {
     this.app.post('/api/projects/:historyId/blob-stats', (req, res, next) => {
       const { historyId } = req.params
       const { blobHashes } = req.body
-
-      let textBlobBytes = 0
-      let binaryBlobBytes = 0
-      let nTextBlobs = 0
-      let nBinaryBlobs = 0
-
       // Calculate actual sizes from uploaded blobs
-      if (blobHashes && this.blobs[historyId]) {
-        for (const hash of blobHashes) {
-          const buf = this.blobs[historyId][hash]
-          if (buf) {
-            const size = buf.byteLength
-
-            // Check if the blob content is valid UTF-8
-            let isText = false
-            try {
-              const decoder = new TextDecoder('utf-8', { fatal: true })
-              decoder.decode(buf)
-              isText = true
-            } catch (e) {
-              // Not valid UTF-8, treat as binary
-              isText = false
-            }
-
-            if (isText) {
-              textBlobBytes += size
-              nTextBlobs++
-            } else {
-              binaryBlobBytes += size
-              nBinaryBlobs++
-            }
-          }
-        }
-      }
-
-      const totalBytes = textBlobBytes + binaryBlobBytes
-
-      res.json({
-        projectId: historyId,
-        textBlobBytes,
-        binaryBlobBytes,
-        totalBytes,
-        nTextBlobs,
-        nBinaryBlobs,
-      })
+      const result = this.computeBlobStats(historyId, blobHashes)
+      res.json(result)
     })
 
     this.app.get(

services/web/test/unit/src/FileStore/FileStoreHandlerTests.js

@@ -71,6 +71,12 @@ describe('FileStoreHandler', function () {
       hasFeature: sinon.stub(),
     }
 
+    this.Modules = {
+      hooks: {
+        fire: sinon.stub().callsArgWith(2, null),
+      },
+    }
+
     this.handler = SandboxedModule.require(MODULE_PATH, {
       requires: {
         '@overleaf/settings': this.settings,
@@ -79,6 +85,7 @@ describe('FileStoreHandler', function () {
         '../Project/ProjectDetailsHandler': this.ProjectDetailsHandler,
         './FileHashManager': this.FileHashManager,
         '../../infrastructure/Features': this.Features,
+        '../../infrastructure/Modules': this.Modules,
         // FIXME: need to stub File object here
         '../../models/File': {
           File: this.FileModel,
@@ -131,6 +138,31 @@ describe('FileStoreHandler', function () {
         .should.equal(true)
     })
 
+    it('should call the preUploadFile hook', async function () {
+      this.fs.createReadStream.returns({
+        pipe() {},
+        on(type, cb) {
+          if (type === 'open') {
+            cb()
+          }
+        },
+      })
+      await this.handler.promises.uploadFileFromDisk(
+        this.projectId,
+        this.fileArgs,
+        this.fsPath
+      )
+      this.Modules.hooks.fire
+        .calledWith('preUploadFile', {
+          projectId: this.projectId,
+          historyId: this.historyId,
+          fileArgs: this.fileArgs,
+          fsPath: this.fsPath,
+          size: this.fileSize,
+        })
+        .should.equal(true)
+    })
+
     it('should upload the file to the history store as a blob', async function () {
       this.fs.createReadStream.returns({
         pipe() {},
@@ -169,6 +201,29 @@ describe('FileStoreHandler', function () {
       expect(this.request).to.not.have.been.called
     })
 
+    it('should call the postUploadFile hook', async function () {
+      this.fs.createReadStream.returns({
+        pipe() {},
+        on(type, cb) {
+          if (type === 'open') {
+            cb()
+          }
+        },
+      })
+      await this.handler.promises.uploadFileFromDisk(
+        this.projectId,
+        this.fileArgs,
+        this.fsPath
+      )
+      this.Modules.hooks.fire
+        .calledWith('postUploadFile', {
+          projectId: this.projectId,
+          fileRef: sinon.match.instanceOf(this.FileModel),
+          size: this.fileSize,
+        })
+        .should.equal(true)
+    })
+
     it('should resolve with the url and fileRef', async function () {
       const { fileRef } = await this.handler.promises.uploadFileFromDisk(
         this.projectId,