[web] remove deledFiles collection (#25750)

* [history-v1] remove processing of deleted files when back-filling hashes

* [web] remove deledFiles collection

GitOrigin-RevId: 7c080e564f7d7acb33ebe7ebe012f415a847d0df

services/web/test/acceptance/src/BackFillDeletedFilesTests.mjs

@@ -1,176 +0,0 @@
-import { exec } from 'node:child_process'
-import { promisify } from 'node:util'
-import { expect } from 'chai'
-import logger from '@overleaf/logger'
-import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
-import UserHelper from './helpers/User.mjs'
-
-const User = UserHelper.promises
-
-async function getDeletedFiles(projectId) {
-  return (await db.projects.findOne({ _id: projectId })).deletedFiles
-}
-
-async function setDeletedFiles(projectId, deletedFiles) {
-  await db.projects.updateOne({ _id: projectId }, { $set: { deletedFiles } })
-}
-
-async function unsetDeletedFiles(projectId) {
-  await db.projects.updateOne(
-    { _id: projectId },
-    { $unset: { deletedFiles: 1 } }
-  )
-}
-
-describe('BackFillDeletedFiles', function () {
-  let user, projectId1, projectId2, projectId3, projectId4, projectId5
-
-  beforeEach('create projects', async function () {
-    user = new User()
-    await user.login()
-
-    projectId1 = new ObjectId(await user.createProject('project1'))
-    projectId2 = new ObjectId(await user.createProject('project2'))
-    projectId3 = new ObjectId(await user.createProject('project3'))
-    projectId4 = new ObjectId(await user.createProject('project4'))
-    projectId5 = new ObjectId(await user.createProject('project5'))
-  })
-
-  let fileId1, fileId2, fileId3, fileId4
-  beforeEach('create files', function () {
-    // take a short cut and just allocate file ids
-    fileId1 = new ObjectId()
-    fileId2 = new ObjectId()
-    fileId3 = new ObjectId()
-    fileId4 = new ObjectId()
-  })
-  const otherFileDetails = {
-    name: 'universe.jpg',
-    linkedFileData: null,
-    hash: 'ed19e7d6779b47d8c63f6fa5a21954dcfb6cac00',
-    deletedAt: new Date(),
-    __v: 0,
-  }
-  let deletedFiles1, deletedFiles2, deletedFiles3
-  beforeEach('set deletedFiles details', async function () {
-    deletedFiles1 = [
-      { _id: fileId1, ...otherFileDetails },
-      { _id: fileId2, ...otherFileDetails },
-    ]
-    deletedFiles2 = [{ _id: fileId3, ...otherFileDetails }]
-    await setDeletedFiles(projectId1, deletedFiles1)
-    await setDeletedFiles(projectId2, deletedFiles2)
-
-    // a project without deletedFiles entries
-    await setDeletedFiles(projectId3, [])
-    // a project without deletedFiles array
-    await unsetDeletedFiles(projectId4)
-    // duplicate entry
-    deletedFiles3 = [
-      { _id: fileId4, ...otherFileDetails },
-      { _id: fileId4, ...otherFileDetails },
-    ]
-    await setDeletedFiles(projectId5, deletedFiles3)
-  })
-
-  async function runScript(args = []) {
-    let result
-    try {
-      result = await promisify(exec)(
-        ['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1', 'VERBOSE_LOGGING=true']
-          .concat(['node', 'scripts/back_fill_deleted_files.mjs'])
-          .concat(args)
-          .join(' ')
-      )
-    } catch (error) {
-      // dump details like exit code, stdErr and stdOut
-      logger.error({ error }, 'script failed')
-      throw error
-    }
-    const { stdout: stdOut } = result
-
-    expect(stdOut).to.match(
-      new RegExp(`Running update on batch with ids .+${projectId1}`)
-    )
-    expect(stdOut).to.match(
-      new RegExp(`Running update on batch with ids .+${projectId2}`)
-    )
-    expect(stdOut).to.not.match(
-      new RegExp(`Running update on batch with ids .+${projectId3}`)
-    )
-    expect(stdOut).to.not.match(
-      new RegExp(`Running update on batch with ids .+${projectId4}`)
-    )
-    expect(stdOut).to.match(
-      new RegExp(`Running update on batch with ids .+${projectId5}`)
-    )
-  }
-
-  function checkAreFilesBackFilled() {
-    it('should back fill file and set projectId', async function () {
-      const docs = await db.deletedFiles
-        .find({}, { sort: { _id: 1 } })
-        .toArray()
-      expect(docs).to.deep.equal([
-        { _id: fileId1, projectId: projectId1, ...otherFileDetails },
-        { _id: fileId2, projectId: projectId1, ...otherFileDetails },
-        { _id: fileId3, projectId: projectId2, ...otherFileDetails },
-        { _id: fileId4, projectId: projectId5, ...otherFileDetails },
-      ])
-    })
-  }
-
-  describe('back fill only', function () {
-    beforeEach('run script', runScript)
-
-    checkAreFilesBackFilled()
-
-    it('should leave the deletedFiles as is', async function () {
-      expect(await getDeletedFiles(projectId1)).to.deep.equal(deletedFiles1)
-      expect(await getDeletedFiles(projectId2)).to.deep.equal(deletedFiles2)
-      expect(await getDeletedFiles(projectId5)).to.deep.equal(deletedFiles3)
-    })
-  })
-
-  describe('back fill and cleanup', function () {
-    beforeEach('run script with cleanup flag', async function () {
-      await runScript(['--perform-cleanup'])
-    })
-
-    checkAreFilesBackFilled()
-
-    it('should cleanup the deletedFiles', async function () {
-      expect(await getDeletedFiles(projectId1)).to.deep.equal([])
-      expect(await getDeletedFiles(projectId2)).to.deep.equal([])
-      expect(await getDeletedFiles(projectId5)).to.deep.equal([])
-    })
-  })
-
-  describe('fix partial inserts and cleanup', function () {
-    beforeEach('simulate one missing insert', async function () {
-      await setDeletedFiles(projectId1, [deletedFiles1[0]])
-    })
-    beforeEach('run script with cleanup flag', async function () {
-      await runScript(['--perform-cleanup'])
-    })
-    beforeEach('add case for one missing file', async function () {
-      await setDeletedFiles(projectId1, deletedFiles1)
-    })
-    beforeEach('add cases for no more files to insert', async function () {
-      await setDeletedFiles(projectId2, deletedFiles2)
-      await setDeletedFiles(projectId5, deletedFiles3)
-    })
-
-    beforeEach('fixing partial insert and cleanup', async function () {
-      await runScript(['--fix-partial-inserts', '--perform-cleanup'])
-    })
-
-    checkAreFilesBackFilled()
-
-    it('should cleanup the deletedFiles', async function () {
-      expect(await getDeletedFiles(projectId1)).to.deep.equal([])
-      expect(await getDeletedFiles(projectId2)).to.deep.equal([])
-      expect(await getDeletedFiles(projectId5)).to.deep.equal([])
-    })
-  })
-})

services/web/test/acceptance/src/DeletionTests.mjs

@@ -252,81 +252,6 @@ describe('Deleting a project', function () {
     })
   })
 
-  describe('when the project has deleted files', function () {
-    beforeEach('get rootFolder id', function (done) {
-      this.user.getProject(this.projectId, (error, project) => {
-        if (error) return done(error)
-        this.rootFolder = project.rootFolder[0]._id
-        done()
-      })
-    })
-
-    let allFileIds
-    beforeEach('reset allFileIds', function () {
-      allFileIds = []
-    })
-    function createAndDeleteFile(name) {
-      let fileId
-      beforeEach(`create file ${name}`, function (done) {
-        this.user.uploadExampleFileInProject(
-          this.projectId,
-          this.rootFolder,
-          name,
-          (error, theFileId) => {
-            fileId = theFileId
-            allFileIds.push(theFileId)
-            done(error)
-          }
-        )
-      })
-      beforeEach(`delete file ${name}`, function (done) {
-        this.user.deleteItemInProject(this.projectId, 'file', fileId, done)
-      })
-    }
-    for (const name of ['a.png', 'another.png']) {
-      createAndDeleteFile(name)
-    }
-
-    it('should have two deleteFiles entries', async function () {
-      const files = await db.deletedFiles
-        .find({}, { sort: { _id: 1 } })
-        .toArray()
-      expect(files).to.have.length(2)
-      expect(files.map(file => file._id.toString())).to.deep.equal(allFileIds)
-    })
-
-    describe('When the deleted project is expired', function () {
-      beforeEach('soft delete the project', function (done) {
-        this.user.deleteProject(this.projectId, done)
-      })
-      beforeEach('hard delete the project', function (done) {
-        request.post(
-          `/internal/project/${this.projectId}/expire-deleted-project`,
-          {
-            auth: {
-              user: settings.apis.web.user,
-              pass: settings.apis.web.pass,
-              sendImmediately: true,
-            },
-          },
-          (error, res) => {
-            expect(error).not.to.exist
-            expect(res.statusCode).to.equal(200)
-
-            done()
-          }
-        )
-      })
-
-      it('should cleanup the deleteFiles', async function () {
-        const files = await db.deletedFiles
-          .find({}, { sort: { _id: 1 } })
-          .toArray()
-        expect(files).to.deep.equal([])
-      })
-    })
-  })
-
   describe('When the project has docs', function () {
     beforeEach(function (done) {
       this.user.getProject(this.projectId, (error, project) => {

services/web/test/unit/src/Project/ProjectDeleterTests.js

@@ -99,10 +99,6 @@ describe('ProjectDeleter', function () {
     }
 
     this.db = {
-      deletedFiles: {
-        indexExists: sinon.stub().resolves(false),
-        deleteMany: sinon.stub(),
-      },
       projects: {
         insertOne: sinon.stub().resolves(),
       },

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js

@@ -4,7 +4,6 @@ const tk = require('timekeeper')
 const Errors = require('../../../../app/src/Features/Errors/Errors')
 const { ObjectId } = require('mongodb-legacy')
 const SandboxedModule = require('sandboxed-module')
-const { DeletedFile } = require('../helpers/models/DeletedFile')
 const { Project } = require('../helpers/models/Project')
 
 const MODULE_PATH =
@@ -77,7 +76,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
     }
 
     this.FolderModel = sinon.stub()
-    this.DeletedFileMock = sinon.mock(DeletedFile)
     this.ProjectMock = sinon.mock(Project)
     this.ProjectEntityHandler = {
       getAllEntitiesFromProject: sinon.stub(),
@@ -197,7 +195,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
         '../Cooldown/CooldownManager': this.CooldownManager,
         '../../models/Folder': { Folder: this.FolderModel },
         '../../infrastructure/LockManager': this.LockManager,
-        '../../models/DeletedFile': { DeletedFile },
         '../../models/Project': { Project },
         './ProjectEntityHandler': this.ProjectEntityHandler,
         './ProjectLocator': this.ProjectLocator,
@@ -208,7 +205,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
   })
 
   afterEach(function () {
-    this.DeletedFileMock.restore()
     this.ProjectMock.restore()
     tk.reset()
   })
@@ -374,17 +370,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
         linkedFileData: { some: 'data' },
         hash: 'some-hash',
       }
-      // Add a deleted file record
-      this.DeletedFileMock.expects('create')
-        .withArgs({
-          projectId: this.project._id,
-          _id: this.file._id,
-          name: this.file.name,
-          linkedFileData: this.file.linkedFileData,
-          hash: this.file.hash,
-          deletedAt: sinon.match.date,
-        })
-        .resolves()
       // Update the file in place
       this.ProjectMock.expects('findOneAndUpdate')
         .withArgs(
@@ -421,7 +406,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
     })
 
     it('updates the database', function () {
-      this.DeletedFileMock.verify()
       this.ProjectMock.verify()
     })
   })
@@ -1059,29 +1043,6 @@ describe('ProjectEntityMongoUpdateHandler', function () {
     })
   })
 
-  describe('_insertDeletedFileReference', function () {
-    beforeEach(async function () {
-      this.DeletedFileMock.expects('create')
-        .withArgs({
-          projectId: this.project._id,
-          _id: this.file._id,
-          name: this.file.name,
-          linkedFileData: this.file.linkedFileData,
-          hash: this.file.hash,
-          deletedAt: sinon.match.date,
-        })
-        .resolves()
-      await this.subject.promises._insertDeletedFileReference(
-        this.project._id,
-        this.file
-      )
-    })
-
-    it('should update the database', function () {
-      this.DeletedFileMock.verify()
-    })
-  })
-
   describe('createNewFolderStructure', function () {
     beforeEach(function () {
       this.mockRootFolder = 'MOCK_ROOT_FOLDER'

services/web/test/unit/src/Project/ProjectEntityUpdateHandlerTests.js

@@ -133,7 +133,6 @@ describe('ProjectEntityUpdateHandler', function () {
         addFolder: sinon.stub(),
         _confirmFolder: sinon.stub(),
         _putElement: sinon.stub(),
-        _insertDeletedFileReference: sinon.stub(),
         replaceFileWithNew: sinon.stub(),
         mkdirp: sinon.stub(),
         moveEntity: sinon.stub(),
@@ -2572,7 +2571,6 @@ describe('ProjectEntityUpdateHandler', function () {
       this.ProjectEntityUpdateHandler.promises.unsetRootDoc = sinon
         .stub()
         .resolves()
-      this.ProjectEntityMongoUpdateHandler.promises._insertDeletedFileReference.resolves()
     })
 
     describe('a file', function () {
@@ -2592,12 +2590,6 @@ describe('ProjectEntityUpdateHandler', function () {
           )
       })
 
-      it('should insert the file into the deletedFiles collection', function () {
-        this.ProjectEntityMongoUpdateHandler.promises._insertDeletedFileReference
-          .calledWith(this.project._id, this.entity)
-          .should.equal(true)
-      })
-
       it('should not delete the file from FileStoreHandler', function () {
         this.FileStoreHandler.promises.deleteFile
           .calledWith(projectId, this.entityId)
@@ -2696,7 +2688,6 @@ describe('ProjectEntityUpdateHandler', function () {
         }
 
         this.ProjectEntityUpdateHandler._cleanUpDoc = sinon.stub().resolves()
-        this.ProjectEntityUpdateHandler._cleanUpFile = sinon.stub().resolves()
         const path = '/folder'
         this.newProject = 'new-project'
         this.subtreeListing =
@@ -2711,17 +2702,6 @@ describe('ProjectEntityUpdateHandler', function () {
           )
       })
 
-      it('should clean up all sub files', function () {
-        this.ProjectEntityUpdateHandler._cleanUpFile.should.have.been.calledWith(
-          this.project,
-          this.file1
-        )
-        this.ProjectEntityUpdateHandler._cleanUpFile.should.have.been.calledWith(
-          this.project,
-          this.file2
-        )
-      })
-
       it('should clean up all sub docs', function () {
         this.ProjectEntityUpdateHandler._cleanUpDoc
           .calledWith(

services/web/test/unit/src/helpers/models/DeletedFile.js

@@ -1,3 +0,0 @@
-const mockModel = require('../MockModel')
-
-module.exports = mockModel('DeletedFile')