Merge pull request #18906 from overleaf/em-migrate-existing-histories-2

History ranges migration script - second attempt

GitOrigin-RevId: 60a2c04e2a72e76a58e9e179fefc4186a96fde32

services/web/app/src/Features/Docstore/DocstoreManager.js

@@ -83,6 +83,10 @@ function getAllDeletedDocs(projectId, callback) {
   })
 }
 
+/**
+ * @param {string} projectId
+ * @param {Callback} callback
+ */
 function getAllRanges(projectId, callback) {
   const url = `${settings.apis.docstore.url}/project/${projectId}/ranges`
   request.get(

services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js

@@ -9,46 +9,6 @@ const { promisify } = require('util')
 const { promisifyMultiResult } = require('@overleaf/promise-utils')
 const ProjectGetter = require('../Project/ProjectGetter')
 
-module.exports = {
-  flushProjectToMongo,
-  flushMultipleProjectsToMongo,
-  flushProjectToMongoAndDelete,
-  flushDocToMongo,
-  deleteDoc,
-  getDocument,
-  setDocument,
-  getProjectDocsIfMatch,
-  clearProjectState,
-  acceptChanges,
-  resolveThread,
-  reopenThread,
-  deleteThread,
-  resyncProjectHistory,
-  updateProjectStructure,
-  promises: {
-    flushProjectToMongo: promisify(flushProjectToMongo),
-    flushMultipleProjectsToMongo: promisify(flushMultipleProjectsToMongo),
-    flushProjectToMongoAndDelete: promisify(flushProjectToMongoAndDelete),
-    flushDocToMongo: promisify(flushDocToMongo),
-    deleteDoc: promisify(deleteDoc),
-    getDocument: promisifyMultiResult(getDocument, [
-      'lines',
-      'version',
-      'ranges',
-      'ops',
-    ]),
-    setDocument: promisify(setDocument),
-    getProjectDocsIfMatch: promisify(getProjectDocsIfMatch),
-    clearProjectState: promisify(clearProjectState),
-    acceptChanges: promisify(acceptChanges),
-    resolveThread: promisify(resolveThread),
-    reopenThread: promisify(reopenThread),
-    deleteThread: promisify(deleteThread),
-    resyncProjectHistory: promisify(resyncProjectHistory),
-    updateProjectStructure: promisify(updateProjectStructure),
-  },
-}
-
 function flushProjectToMongo(projectId, callback) {
   _makeRequest(
     {
@@ -267,12 +227,17 @@ function resyncProjectHistory(
   projectHistoryId,
   docs,
   files,
+  opts,
   callback
 ) {
+  const body = { docs, files, projectHistoryId }
+  if (opts.historyRangesMigration) {
+    body.historyRangesMigration = opts.historyRangesMigration
+  }
   _makeRequest(
     {
       path: `/project/${projectId}/history/resync`,
-      json: { docs, files, projectHistoryId },
+      json: body,
       method: 'POST',
       timeout: 6 * 60 * 1000, // allow 6 minutes for resync
     },
@@ -479,3 +444,43 @@ function _getUpdates(
 
   return { deletes, adds, renames }
 }
+
+module.exports = {
+  flushProjectToMongo,
+  flushMultipleProjectsToMongo,
+  flushProjectToMongoAndDelete,
+  flushDocToMongo,
+  deleteDoc,
+  getDocument,
+  setDocument,
+  getProjectDocsIfMatch,
+  clearProjectState,
+  acceptChanges,
+  resolveThread,
+  reopenThread,
+  deleteThread,
+  resyncProjectHistory,
+  updateProjectStructure,
+  promises: {
+    flushProjectToMongo: promisify(flushProjectToMongo),
+    flushMultipleProjectsToMongo: promisify(flushMultipleProjectsToMongo),
+    flushProjectToMongoAndDelete: promisify(flushProjectToMongoAndDelete),
+    flushDocToMongo: promisify(flushDocToMongo),
+    deleteDoc: promisify(deleteDoc),
+    getDocument: promisifyMultiResult(getDocument, [
+      'lines',
+      'version',
+      'ranges',
+      'ops',
+    ]),
+    setDocument: promisify(setDocument),
+    getProjectDocsIfMatch: promisify(getProjectDocsIfMatch),
+    clearProjectState: promisify(clearProjectState),
+    acceptChanges: promisify(acceptChanges),
+    resolveThread: promisify(resolveThread),
+    reopenThread: promisify(reopenThread),
+    deleteThread: promisify(deleteThread),
+    resyncProjectHistory: promisify(resyncProjectHistory),
+    updateProjectStructure: promisify(updateProjectStructure),
+  },
+}

services/web/app/src/Features/History/HistoryController.js

@@ -68,15 +68,24 @@ module.exports = HistoryController = {
     // increase timeout to 6 minutes
     res.setTimeout(6 * 60 * 1000)
     const projectId = req.params.Project_id
-    ProjectEntityUpdateHandler.resyncProjectHistory(projectId, function (err) {
-      if (err instanceof Errors.ProjectHistoryDisabledError) {
-        return res.sendStatus(404)
+    const opts = {}
+    const historyRangesMigration = req.body.historyRangesMigration
+    if (historyRangesMigration) {
+      opts.historyRangesMigration = historyRangesMigration
+    }
+    ProjectEntityUpdateHandler.resyncProjectHistory(
+      projectId,
+      opts,
+      function (err) {
+        if (err instanceof Errors.ProjectHistoryDisabledError) {
+          return res.sendStatus(404)
+        }
+        if (err) {
+          return next(err)
+        }
+        res.sendStatus(204)
       }
-      if (err) {
-        return next(err)
-      }
-      res.sendStatus(204)
-    })
+    )
   },
 
   restoreFileFromV2(req, res, next) {

services/web/app/src/Features/History/HistoryManager.js

@@ -51,6 +51,9 @@ async function resyncProject(projectId, options = {}) {
   if (options.origin) {
     body.origin = options.origin
   }
+  if (options.historyRangesMigration) {
+    body.historyRangesMigration = options.historyRangesMigration
+  }
   try {
     await fetchNothing(
       `${settings.apis.project_history.url}/project/${projectId}/resync`,

services/web/app/src/Features/History/HistoryRangesSupportMigration.js

@@ -0,0 +1,22 @@
+// @ts-check
+
+const { callbackify } = require('util')
+const HistoryManager = require('../History/HistoryManager')
+
+/**
+ * Migrate a single project
+ *
+ * @param {string} projectId
+ * @param {"forwards" | "backwards"} direction
+ */
+async function migrateProject(projectId, direction = 'forwards') {
+  await HistoryManager.promises.flushProject(projectId)
+  await HistoryManager.promises.resyncProject(projectId, {
+    historyRangesMigration: direction,
+  })
+}
+
+module.exports = {
+  migrateProject: callbackify(migrateProject),
+  promises: { migrateProject },
+}

services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js

@@ -15,6 +15,7 @@ const { Project } = require('../../models/Project')
 const ProjectEntityHandler = require('./ProjectEntityHandler')
 const ProjectGetter = require('./ProjectGetter')
 const ProjectLocator = require('./ProjectLocator')
+const ProjectOptionsHandler = require('./ProjectOptionsHandler')
 const ProjectUpdateHandler = require('./ProjectUpdateHandler')
 const ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandler')
 const SafePath = require('./SafePath')
@@ -154,6 +155,8 @@ function getDocContext(projectId, docId, callback) {
 }
 
 const ProjectEntityUpdateHandler = {
+  LOCK_NAMESPACE,
+
   updateDocLines(
     projectId,
     docId,
@@ -1394,7 +1397,7 @@ const ProjectEntityUpdateHandler = {
   // This doesn't directly update project structure but we need to take the lock
   // to prevent anything else being queued before the resync update
   resyncProjectHistory: wrapWithLock(
-    (projectId, callback) =>
+    (projectId, opts, callback) =>
       ProjectGetter.getProject(
         projectId,
         { rootFolder: true, overleaf: true },
@@ -1449,7 +1452,21 @@ const ProjectEntityUpdateHandler = {
                 projectHistoryId,
                 docs,
                 files,
-                callback
+                opts,
+                err => {
+                  if (err) {
+                    return callback(err)
+                  }
+                  if (opts.historyRangesMigration) {
+                    ProjectOptionsHandler.setHistoryRangesSupport(
+                      projectId,
+                      opts.historyRangesMigration === 'forwards',
+                      callback
+                    )
+                  } else {
+                    callback()
+                  }
+                }
               )
             }
           )

services/web/app/src/Features/Project/ProjectHistoryHandler.js

@@ -46,7 +46,10 @@ const ProjectHistoryHandler = {
 
     await ProjectHistoryHandler.setHistoryId(projectId, historyId)
 
-    await ProjectEntityUpdateHandler.promises.resyncProjectHistory(projectId)
+    await ProjectEntityUpdateHandler.promises.resyncProjectHistory(
+      projectId,
+      {}
+    )
 
     await HistoryManager.promises.flushProject(projectId)
   },

services/web/app/src/Features/Project/ProjectOptionsHandler.js

@@ -64,9 +64,11 @@ const ProjectOptionsHandler = {
     return Project.updateOne(conditions, update, {})
   },
 
-  async enableHistoryRangesSupport(projectId) {
+  async setHistoryRangesSupport(projectId, enabled) {
     const conditions = { _id: new ObjectId(projectId) }
-    const update = { $set: { 'overleaf.history.rangesSupportEnabled': true } }
+    const update = {
+      $set: { 'overleaf.history.rangesSupportEnabled': enabled },
+    }
     // NOTE: Updating the Mongoose model with the same query doesn't work. Maybe
     // because rangesSupportEnabled is not part of the schema?
     return db.projects.updateOne(conditions, update)
@@ -83,8 +85,8 @@ module.exports = {
   unsetBrandVariationId: callbackify(
     ProjectOptionsHandler.unsetBrandVariationId
   ),
-  enableHistoryRangesSupport: callbackify(
-    ProjectOptionsHandler.enableHistoryRangesSupport
+  setHistoryRangesSupport: callbackify(
+    ProjectOptionsHandler.setHistoryRangesSupport
   ),
   promises: ProjectOptionsHandler,
 }

services/web/app/src/infrastructure/LockManager.js

@@ -1,7 +1,6 @@
 const settings = require('@overleaf/settings')
 const RedisWrapper = require('./RedisWrapper')
 const rclient = RedisWrapper.client('lock')
-const { callbackify, promisify } = require('util')
 
 const RedisWebLocker = require('@overleaf/redis-wrapper/RedisWebLocker')
 
@@ -34,16 +33,4 @@ LockManager.withTimeout = function (timeout) {
   return createLockManager(lockManagerSettingsWithTimeout)
 }
 
-// need to bind the promisified function when it is part of a class
-// see https://nodejs.org/dist/latest-v16.x/docs/api/util.html#utilpromisifyoriginal
-const promisifiedRunWithLock = promisify(LockManager.runWithLock).bind(
-  LockManager
-)
-LockManager.promises = {
-  runWithLock(namespace, id, runner) {
-    const cbRunner = callbackify(runner)
-    return promisifiedRunWithLock(namespace, id, cbRunner)
-  },
-}
-
 module.exports = LockManager

services/web/scripts/history/migrate_ranges_support.js

@@ -0,0 +1,41 @@
+const HistoryRangesSupportMigration = require('../../app/src/Features/History/HistoryRangesSupportMigration')
+const { waitForDb } = require('../../app/src/infrastructure/mongodb')
+const minimist = require('minimist')
+
+async function main() {
+  await waitForDb()
+  const { projectId, direction } = parseArgs()
+  await HistoryRangesSupportMigration.promises.migrateProject(
+    projectId,
+    direction
+  )
+}
+
+function usage() {
+  console.log('Usage: migrate_ranges_support.js PROJECT_ID [--backwards]')
+}
+
+function parseArgs() {
+  const args = minimist(process.argv.slice(2), {
+    boolean: ['backwards'],
+  })
+
+  if (args._.length !== 1) {
+    usage()
+    process.exit(1)
+  }
+
+  return {
+    direction: args.backwards ? 'backwards' : 'forwards',
+    projectId: args._[0],
+  }
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })

services/web/test/unit/src/History/HistoryControllerTests.js

@@ -233,7 +233,7 @@ describe('HistoryController', function () {
     describe('for a project without project-history enabled', function () {
       beforeEach(function () {
         this.project_id = 'mock-project-id'
-        this.req = { params: { Project_id: this.project_id } }
+        this.req = { params: { Project_id: this.project_id }, body: {} }
         this.res = { setTimeout: sinon.stub(), sendStatus: sinon.stub() }
         this.next = sinon.stub()
 
@@ -257,7 +257,7 @@ describe('HistoryController', function () {
     describe('for a project with project-history enabled', function () {
       beforeEach(function () {
         this.project_id = 'mock-project-id'
-        this.req = { params: { Project_id: this.project_id } }
+        this.req = { params: { Project_id: this.project_id }, body: {} }
         this.res = { setTimeout: sinon.stub(), sendStatus: sinon.stub() }
         this.next = sinon.stub()
 

services/web/test/unit/src/Project/ProjectEntityUpdateHandlerTests.js

@@ -149,6 +149,9 @@ describe('ProjectEntityUpdateHandler', function () {
     this.EditorRealTimeController = {
       emitToRoom: sinon.stub(),
     }
+    this.ProjectOptionsHandler = {
+      setHistoryRangesSupport: sinon.stub().yields(),
+    }
     this.ProjectEntityUpdateHandler = SandboxedModule.require(MODULE_PATH, {
       requires: {
         '@overleaf/settings': { validRootDocExtensions: ['tex'] },
@@ -167,6 +170,7 @@ describe('ProjectEntityUpdateHandler', function () {
         './ProjectEntityHandler': this.ProjectEntityHandler,
         './ProjectEntityMongoUpdateHandler':
           this.ProjectEntityMongoUpdateHandler,
+        './ProjectOptionsHandler': this.ProjectOptionsHandler,
         '../ThirdPartyDataStore/TpdsUpdateSender': this.TpdsUpdateSender,
         '../Editor/EditorRealTimeController': this.EditorRealTimeController,
         '../../infrastructure/FileWriter': this.FileWriter,
@@ -1962,6 +1966,7 @@ describe('ProjectEntityUpdateHandler', function () {
 
         this.ProjectEntityUpdateHandler.resyncProjectHistory(
           projectId,
+          {},
           this.callback
         )
       })
@@ -1987,6 +1992,7 @@ describe('ProjectEntityUpdateHandler', function () {
 
         this.ProjectEntityUpdateHandler.resyncProjectHistory(
           projectId,
+          {},
           this.callback
         )
       })
@@ -2025,6 +2031,7 @@ describe('ProjectEntityUpdateHandler', function () {
         })
         this.ProjectEntityUpdateHandler.resyncProjectHistory(
           projectId,
+          {},
           this.callback
         )
       })
@@ -2113,7 +2120,11 @@ describe('ProjectEntityUpdateHandler', function () {
           files: this.files,
           folders: [],
         })
-        this.ProjectEntityUpdateHandler.resyncProjectHistory(projectId, done)
+        this.ProjectEntityUpdateHandler.resyncProjectHistory(
+          projectId,
+          {},
+          done
+        )
       })
 
       it('renames the duplicate files', function () {
@@ -2214,7 +2225,11 @@ describe('ProjectEntityUpdateHandler', function () {
           files: this.files,
           folders: [],
         })
-        this.ProjectEntityUpdateHandler.resyncProjectHistory(projectId, done)
+        this.ProjectEntityUpdateHandler.resyncProjectHistory(
+          projectId,
+          {},
+          done
+        )
       })
 
       it('renames the files', function () {
@@ -2301,7 +2316,11 @@ describe('ProjectEntityUpdateHandler', function () {
           files,
           folders,
         })
-        this.ProjectEntityUpdateHandler.resyncProjectHistory(projectId, done)
+        this.ProjectEntityUpdateHandler.resyncProjectHistory(
+          projectId,
+          {},
+          done
+        )
       })
 
       it('renames the folder', function () {
@@ -2352,7 +2371,11 @@ describe('ProjectEntityUpdateHandler', function () {
           files,
           folders,
         })
-        this.ProjectEntityUpdateHandler.resyncProjectHistory(projectId, done)
+        this.ProjectEntityUpdateHandler.resyncProjectHistory(
+          projectId,
+          {},
+          done
+        )
       })
 
       it('renames the doc', function () {
@@ -2385,6 +2408,7 @@ describe('ProjectEntityUpdateHandler', function () {
         this.ProjectEntityHandler.getAllEntitiesFromProject.throws()
         this.ProjectEntityUpdateHandler.resyncProjectHistory(
           projectId,
+          {},
           this.callback
         )
       })

services/web/test/unit/src/Project/ProjectOptionsHandlerTests.js

@@ -191,7 +191,7 @@ describe('ProjectOptionsHandler', function () {
 
   describe('setting the rangesSupportEnabled', function () {
     it('should perform and update on mongo', async function () {
-      await this.handler.promises.enableHistoryRangesSupport(projectId)
+      await this.handler.promises.setHistoryRangesSupport(projectId, true)
       sinon.assert.calledWith(
         this.db.projects.updateOne,
         { _id: new ObjectId(projectId) },
@@ -205,8 +205,8 @@ describe('ProjectOptionsHandler', function () {
       })
 
       it('should be rejected', async function () {
-        expect(this.handler.promises.enableHistoryRangesSupport(projectId)).to
-          .be.rejected
+        expect(this.handler.promises.setHistoryRangesSupport(projectId, true))
+          .to.be.rejected
       })
     })
   })