Merge pull request #3701 from overleaf/jpa-script-back-fill-deleted-files-stage-2

[scripts] back_fill_deleted_files: optionally fix partial inserts

GitOrigin-RevId: 7d48d311db1af0b38222725ca86cdba8951b16b3

services/web/scripts/back_fill_deleted_files.js

@@ -5,7 +5,8 @@ const { promiseMapWithLimit, promisify } = require('../app/src/util/promises')
 const { db } = require('../app/src/infrastructure/mongodb')
 const sleep = promisify(setTimeout)
 
-const PERFORM_CLEANUP = process.argv.pop() === '--perform-cleanup'
+const PERFORM_CLEANUP = process.argv.includes('--perform-cleanup')
+const FIX_PARTIAL_INSERTS = process.argv.includes('--fix-partial-inserts')
 const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt(
   process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
   10
@@ -50,7 +51,12 @@ async function backFillFiles(project) {
   project.deletedFiles.forEach(file => {
     file.projectId = projectId
   })
-  await db.deletedFiles.insertMany(project.deletedFiles)
+
+  if (FIX_PARTIAL_INSERTS) {
+    await fixPartialInserts(project)
+  } else {
+    await db.deletedFiles.insertMany(project.deletedFiles)
+  }
 }
 
 function filterDuplicatesInPlace(project) {
@@ -63,6 +69,28 @@ function filterDuplicatesInPlace(project) {
   })
 }
 
+async function fixPartialInserts(project) {
+  const seenFileIds = new Set(
+    (
+      await db.deletedFiles
+        .find(
+          { _id: { $in: project.deletedFiles.map(file => file._id) } },
+          { projection: { _id: 1 } }
+        )
+        .toArray()
+    ).map(file => file._id.toString())
+  )
+  project.deletedFiles = project.deletedFiles.filter(file => {
+    const id = file._id.toString()
+    if (seenFileIds.has(id)) return false
+    seenFileIds.add(id)
+    return true
+  })
+  if (project.deletedFiles.length > 0) {
+    await db.deletedFiles.insertMany(project.deletedFiles)
+  }
+}
+
 async function cleanupProject(project) {
   await db.projects.updateOne(
     { _id: project._id },