Merge pull request #11280 from overleaf/bg-issue11277

Move history migration logic to web module

GitOrigin-RevId: 6f1ba33519277b9ba13ecb2a13ae2c43ee06f675

services/web/scripts/history/HistoryUpgradeHelper.js

@@ -1,335 +0,0 @@
-const { ReadPreference, ObjectId } = require('mongodb')
-const { db } = require('../../app/src/infrastructure/mongodb')
-const Settings = require('@overleaf/settings')
-
-const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
-const HistoryManager = require('../../app/src/Features/History/HistoryManager')
-const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
-const ProjectEntityHandler = require('../../app/src/Features/Project/ProjectEntityHandler')
-const ProjectEntityUpdateHandler = require('../../app/src/Features/Project/ProjectEntityUpdateHandler')
-
-// Timestamp of when 'Enable history for SL in background' release
-const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
-const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
-  ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
-)
-const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
-  OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
-
-async function determineProjectHistoryType(project) {
-  if (project.overleaf && project.overleaf.history) {
-    if (project.overleaf.history.upgradeFailed) {
-      return 'UpgradeFailed'
-    }
-    if (project.overleaf.history.conversionFailed) {
-      return 'ConversionFailed'
-    }
-  }
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.id
-  ) {
-    if (project.overleaf.history.display) {
-      // v2: full project history, do nothing
-      return 'V2'
-    } else {
-      if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
-        // IF project initialised after full project history enabled for all projects
-        //    THEN project history should contain all information we need, without intervention
-        return 'V1WithoutConversion'
-      } else {
-        // ELSE SL history may predate full project history
-        //    THEN delete full project history and convert their SL history to full project history
-        // --
-        // TODO: how to verify this, can get rough start date of SL history, but not full project history
-        const preserveHistory = await shouldPreserveHistory(project)
-        const anyDocHistory = await anyDocHistoryExists(project)
-        const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-        if (preserveHistory) {
-          if (anyDocHistory || anyDocHistoryIndex) {
-            // if SL history exists that we need to preserve, then we must convert
-            return 'V1WithConversion'
-          } else {
-            // otherwise just upgrade without conversion
-            return 'V1WithoutConversion'
-          }
-        } else {
-          // if preserveHistory false, then max 7 days of SL history
-          // but v1 already record to both histories, so safe to upgrade
-          return 'V1WithoutConversion'
-        }
-      }
-    }
-  } else {
-    const preserveHistory = await shouldPreserveHistory(project)
-    const anyDocHistory = await anyDocHistoryExists(project)
-    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-    if (anyDocHistory || anyDocHistoryIndex) {
-      // IF there is SL history ->
-      if (preserveHistory) {
-        // that needs to be preserved:
-        //    THEN initialise full project history and convert SL history to full project history
-        return 'NoneWithConversion'
-      } else {
-        return 'NoneWithTemporaryHistory'
-      }
-    } else {
-      // ELSE there is not any SL history ->
-      //    THEN initialise full project history and sync with current content
-      return 'NoneWithoutConversion'
-    }
-  }
-}
-
-async function upgradeProject(project) {
-  const historyType = await determineProjectHistoryType(project)
-  if (historyType === 'V2') {
-    return { historyType, upgraded: true }
-  }
-  const upgradeFn = getUpgradeFunctionForType(historyType)
-  if (!upgradeFn) {
-    return { error: 'unsupported history type' }
-  }
-  const result = await upgradeFn(project)
-  result.historyType = historyType
-  return result
-}
-
-// Do upgrades/conversion:
-
-function getUpgradeFunctionForType(historyType) {
-  return UpgradeFunctionMapping[historyType]
-}
-
-const UpgradeFunctionMapping = {
-  NoneWithoutConversion: doUpgradeForNoneWithoutConversion,
-  UpgradeFailed: doUpgradeForNoneWithoutConversion,
-  ConversionFailed: doUpgradeForNoneWithConversion,
-  V1WithoutConversion: doUpgradeForV1WithoutConversion,
-  V1WithConversion: doUpgradeForV1WithConversion,
-  NoneWithConversion: doUpgradeForNoneWithConversion,
-  NoneWithTemporaryHistory: doUpgradeForNoneWithConversion,
-}
-
-async function doUpgradeForV1WithoutConversion(project) {
-  await db.projects.updateOne(
-    { _id: project._id },
-    {
-      $set: {
-        'overleaf.history.display': true,
-        'overleaf.history.upgradedAt': new Date(),
-        'overleaf.history.upgradeReason': `v1-without-sl-history`,
-      },
-    }
-  )
-  return { upgraded: true }
-}
-
-async function doUpgradeForV1WithConversion(project) {
-  const result = {}
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  try {
-    // We treat these essentially as None projects, the V1 history is irrelevant,
-    // so we will delete it, and do a conversion as if we're a None project
-    await ProjectHistoryController.deleteProjectHistory(projectIdString)
-    await ProjectHistoryController.migrateProjectHistory(projectIdString)
-  } catch (err) {
-    // if migrateProjectHistory fails, it cleans up by deleting
-    // the history and unsetting the history id
-    // therefore a failed project will still look like a 'None with conversion' project
-    result.error = err
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.conversionFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: projectId },
-    {
-      $set: {
-        'overleaf.history.upgradeReason': `v1-with-conversion`,
-      },
-      $unset: {
-        'overleaf.history.upgradeFailed': true,
-        'overleaf.history.conversionFailed': true,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-async function doUpgradeForNoneWithoutConversion(project) {
-  const result = {}
-  const projectId = project._id
-  try {
-    // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
-    // However sends a force resync project to project history instead
-    // of a resync request to doc-updater
-    let historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId)
-    if (historyId == null) {
-      historyId = await HistoryManager.promises.initializeProject(projectId)
-      if (historyId != null) {
-        await ProjectHistoryHandler.promises.setHistoryId(projectId, historyId)
-      }
-    }
-    await HistoryManager.promises.resyncProject(projectId, {
-      force: true,
-      origin: { kind: 'history-migration' },
-    })
-    await HistoryManager.promises.flushProject(projectId)
-  } catch (err) {
-    result.error = err
-    await db.projects.updateOne(
-      { _id: project._id },
-      {
-        $set: {
-          'overleaf.history.upgradeFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: project._id },
-    {
-      $set: {
-        'overleaf.history.display': true,
-        'overleaf.history.upgradedAt': new Date(),
-        'overleaf.history.upgradeReason': `none-without-conversion`,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-async function doUpgradeForNoneWithConversion(project) {
-  const result = {}
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  try {
-    await ProjectHistoryController.migrateProjectHistory(projectIdString)
-  } catch (err) {
-    // if migrateProjectHistory fails, it cleans up by deleting
-    // the history and unsetting the history id
-    // therefore a failed project will still look like a 'None with conversion' project
-    result.error = err
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.conversionFailed': true,
-        },
-      }
-    )
-    return result
-  }
-  await db.projects.updateOne(
-    { _id: projectId },
-    {
-      $set: {
-        'overleaf.history.upgradeReason': `none-with-conversion`,
-      },
-      $unset: {
-        'overleaf.history.upgradeFailed': true,
-        'overleaf.history.conversionFailed': true,
-      },
-    }
-  )
-  result.upgraded = true
-  return result
-}
-
-// Util
-
-function projectCreatedAfterFullProjectHistoryEnabled(project) {
-  return (
-    project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
-  )
-}
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: ReadPreference.SECONDARY }
-  )
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
-}
-
-async function convertLargeDocsToFile(projectId, userId) {
-  const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
-  let convertedDocCount = 0
-  for (const doc of Object.values(docs)) {
-    const sizeBound = JSON.stringify(doc.lines)
-    if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) {
-      await ProjectEntityUpdateHandler.promises.convertDocToFile(
-        projectId,
-        doc._id,
-        userId,
-        null
-      )
-      convertedDocCount++
-    }
-  }
-  return convertedDocCount
-}
-
-// check whether the total size of the document in characters exceeds the
-// maxDocLength.
-//
-// Copied from document-updater:
-// https://github.com/overleaf/internal/blob/74adfbebda5f3c2c37d9937f0db5c4106ecde492/services/document-updater/app/js/Limits.js#L18
-function docIsTooLarge(estimatedSize, lines, maxDocLength) {
-  if (estimatedSize <= maxDocLength) {
-    return false // definitely under the limit, no need to calculate the total size
-  }
-  // calculate the total size, bailing out early if the size limit is reached
-  let size = 0
-  for (const line of lines) {
-    size += line.length + 1 // include the newline
-    if (size > maxDocLength) return true
-  }
-  // since we didn't hit the limit in the loop, the document is within the allowed length
-  return false
-}
-
-module.exports = {
-  determineProjectHistoryType,
-  getUpgradeFunctionForType,
-  upgradeProject,
-  convertLargeDocsToFile,
-}

services/web/scripts/history/count_project_history_categories.js

@@ -11,7 +11,10 @@ process.env.MONGO_SOCKET_TIMEOUT =
 
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
-const { determineProjectHistoryType } = require('./HistoryUpgradeHelper')
+const {
+  determineProjectHistoryType,
+  countProjects,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 const COUNT = {
   V2: 0,
@@ -22,6 +25,8 @@ const COUNT = {
   NoneWithTemporaryHistory: 0,
   UpgradeFailed: 0,
   ConversionFailed: 0,
+  MigratedProjects: 0,
+  TotalProjects: 0,
 }
 
 async function processBatch(_, projects) {
@@ -60,6 +65,10 @@ async function main() {
     projection,
     options
   )
+  COUNT.MigratedProjects = await countProjects({
+    'overleaf.history.display': true,
+  })
+  COUNT.TotalProjects = await countProjects()
   console.log('Final')
   console.log(COUNT)
 }

services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js

@@ -29,12 +29,15 @@ const USER_ID = process.env.USER_ID
 const CONVERT_LARGE_DOCS_TO_FILE =
   process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true'
 
-const { ObjectId, ReadPreference } = require('mongodb')
+const { ObjectId } = require('mongodb')
 const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
-const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
-const HistoryUpgradeHelper = require('./HistoryUpgradeHelper')
+const {
+  anyDocHistoryExists,
+  anyDocHistoryIndexExists,
+  doUpgradeForNoneWithConversion,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 console.log({
   DRY_RUN,
@@ -111,99 +114,49 @@ async function processProject(project) {
       }
     }
   }
-  const anyDocHistory = await anyDocHistoryExists(project)
-  if (anyDocHistory) {
-    return await doUpgradeForNoneWithConversion(project)
-  }
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (anyDocHistoryIndex) {
-    return await doUpgradeForNoneWithConversion(project)
-  }
-}
-
-async function doUpgradeForNoneWithConversion(project) {
   if (RESULT.failed >= MAX_FAILURES) {
     return
   }
   if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
     return
   }
-  RESULT.attempted += 1
-  const projectId = project._id
-  // migrateProjectHistory expects project id as a string
-  const projectIdString = project._id.toString()
-  if (!DRY_RUN) {
-    try {
-      if (CONVERT_LARGE_DOCS_TO_FILE) {
-        const convertedDocCount =
-          await HistoryUpgradeHelper.convertLargeDocsToFile(projectId, USER_ID)
-        console.log(
-          `converted ${convertedDocCount} large docs to binary files for project ${projectId}`
-        )
-      }
-      await ProjectHistoryController.migrateProjectHistory(projectIdString, {
+  const anyDocHistoryOrIndex =
+    (await anyDocHistoryExists(project)) ||
+    (await anyDocHistoryIndexExists(project))
+  if (anyDocHistoryOrIndex) {
+    RESULT.attempted += 1
+    if (DRY_RUN) {
+      return
+    }
+    const result = await doUpgradeForNoneWithConversion(project, {
+      migrationOptions: {
         archiveOnFailure: ARCHIVE_ON_FAILURE,
         fixInvalidCharacters: FIX_INVALID_CHARACTERS,
         forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE,
         importZipFilePath: IMPORT_ZIP_FILE_PATH,
         cutoffDate: CUTOFF_DATE,
-      })
-    } catch (err) {
-      // if migrateProjectHistory fails, it cleans up by deleting
-      // the history and unsetting the history id
-      // therefore a failed project will still look like a 'None with conversion' project
-      RESULT.failed += 1
-      console.error(`project ${projectId} FAILED with error: `, err)
-      // We set a failed flag so future runs of the script don't automatically retry
-      await db.projects.updateOne(
-        { _id: projectId },
-        {
-          $set: {
-            'overleaf.history.conversionFailed': true,
-          },
-        }
+      },
+      convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE,
+      userId: USER_ID,
+      reason: `${SCRIPT_VERSION}`,
+    })
+    if (result.convertedDocCount) {
+      console.log(
+        `project ${project._id} converted ${result.convertedDocCount} docs to filestore`
       )
-      return
     }
-    await db.projects.updateOne(
-      { _id: projectId },
-      {
-        $set: {
-          'overleaf.history.upgradeReason': `none-with-conversion/${SCRIPT_VERSION}`,
-        },
-        $unset: {
-          'overleaf.history.upgradeFailed': true,
-          'overleaf.history.conversionFailed': true,
-        },
+    if (result.error) {
+      console.error(`project ${project._id} FAILED with error: `, result.error)
+      RESULT.failed += 1
+    } else if (result.upgraded) {
+      if (VERBOSE_LOGGING) {
+        console.log(
+          `project ${project._id} converted and upgraded to full project history`
+        )
       }
-    )
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${projectId} converted and upgraded to full project history`
-    )
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      RESULT.projectsUpgraded += 1
     }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
+  }
 }
 
 async function main() {

services/web/scripts/history/upgrade_project.js

@@ -1,6 +1,8 @@
 const { ReadPreference, ObjectId } = require('mongodb')
 const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
-const { upgradeProject } = require('./HistoryUpgradeHelper')
+const {
+  upgradeProject,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 async function processProject(project) {
   const result = await upgradeProject(project)