[document-updater] add track-changes support to setDoc (#35820)

GitOrigin-RevId: 5d39dd1945e710ec3f7068830fb2476e59f0f2a5

services/document-updater/app.js

@@ -220,6 +220,8 @@ app.use((error, req, res, next) => {
     return res.status(413).send('request entity too large')
   } else if (error instanceof Errors.DocumentValidationError) {
     return res.sendStatus(422)
+  } else if (error instanceof Errors.OTTypeMismatchError) {
+    return res.sendStatus(422)
   } else {
     logger.error({ err: error, req }, 'request errored')
     return res.status(500).send('Oops, something went wrong')

services/document-updater/app/js/DocumentManager.js

@@ -11,6 +11,7 @@ const RangesManager = require('./RangesManager')
 const { extractOriginOrSource } = require('./Utils')
 const { getTotalSizeOfLines } = require('./Limits')
 const Settings = require('@overleaf/settings')
+const RangesTracker = require('@overleaf/ranges-tracker')
 const { StringFileData } = require('overleaf-editor-core')
 const {
   diffAsTextOperation,
@@ -134,14 +135,20 @@ const DocumentManager = {
     }
   },
 
-  async appendToDoc(projectId, docId, linesToAppend, originOrSource, userId) {
+  async appendToDoc(
+    projectId,
+    docId,
+    linesToAppend,
+    originOrSource,
+    userId,
+    trackChanges
+  ) {
     let { lines: currentLines, type } = await DocumentManager.getDoc(
       projectId,
       docId
     )
     if (type === 'history-ot') {
       const file = StringFileData.fromRaw(currentLines)
-      // TODO(24596): tc support for history-ot
       currentLines = file.getLines()
     }
     const currentLineSize = getTotalSizeOfLines(currentLines)
@@ -161,7 +168,8 @@ const DocumentManager = {
       originOrSource,
       userId,
       false,
-      false
+      false,
+      trackChanges
     )
   },
 
@@ -172,7 +180,8 @@ const DocumentManager = {
     originOrSource,
     userId,
     undoing,
-    external
+    external,
+    trackChanges
   ) {
     if (newLines == null) {
       throw new Error('No lines were provided to setDoc')
@@ -197,7 +206,11 @@ const DocumentManager = {
     let op
     if (type === 'history-ot') {
       const file = StringFileData.fromRaw(oldLines)
-      const operation = diffAsTextOperation(file, newLines.join('\n'))
+      const operation = diffAsTextOperation(
+        file,
+        newLines.join('\n'),
+        trackChanges ? { tracking: { userId, ts: new Date() } } : {}
+      )
       if (operation.isNoop()) {
         op = []
       } else {
@@ -222,6 +235,9 @@ const DocumentManager = {
         user_id: userId,
       },
     }
+    if (type !== 'history-ot' && trackChanges) {
+      update.meta.tc = RangesTracker.generateIdSeed()
+    }
     if (external) {
       update.meta.type = 'external'
     }
@@ -666,7 +682,8 @@ const DocumentManager = {
     source,
     userId,
     undoing,
-    external
+    external,
+    trackChanges
   ) {
     const UpdateManager = require('./UpdateManager')
     return await UpdateManager.promises.lockUpdatesAndDo(
@@ -677,11 +694,19 @@ const DocumentManager = {
       source,
       userId,
       undoing,
-      external
+      external,
+      trackChanges
     )
   },
 
-  async appendToDocWithLock(projectId, docId, lines, source, userId) {
+  async appendToDocWithLock(
+    projectId,
+    docId,
+    lines,
+    source,
+    userId,
+    trackChanges
+  ) {
     const UpdateManager = require('./UpdateManager')
     return await UpdateManager.promises.lockUpdatesAndDo(
       DocumentManager.appendToDoc,
@@ -689,7 +714,8 @@ const DocumentManager = {
       docId,
       lines,
       source,
-      userId
+      userId,
+      trackChanges
     )
   },
 

services/document-updater/app/js/HttpController.js

@@ -199,7 +199,10 @@ async function clearProjectState(req, res) {
 async function setDoc(req, res) {
   const docId = req.params.doc_id
   const projectId = req.params.project_id
-  const { lines, source, user_id: userId, undoing } = req.body
+  const { lines, source, user_id: userId, undoing, trackChanges } = req.body
+  if (trackChanges && !userId) {
+    return res.status(400).send('track changes requires a user id')
+  }
   const lineSize = getTotalSizeOfLines(lines)
 
   if (lineSize > Settings.max_doc_length) {
@@ -222,7 +225,8 @@ async function setDoc(req, res) {
     source,
     userId,
     undoing,
-    true
+    true,
+    trackChanges
   )
   timer.done()
   logger.debug({ projectId, docId }, 'set doc via http')
@@ -236,7 +240,10 @@ async function setDoc(req, res) {
 async function appendToDoc(req, res) {
   const docId = req.params.doc_id
   const projectId = req.params.project_id
-  const { lines, source, user_id: userId } = req.body
+  const { lines, source, user_id: userId, trackChanges } = req.body
+  if (trackChanges && !userId) {
+    return res.status(400).send('track changes requires a user id')
+  }
   const timer = new Metrics.Timer('http.appendToDoc')
 
   let result
@@ -246,7 +253,8 @@ async function appendToDoc(req, res) {
       docId,
       lines,
       source,
-      userId
+      userId,
+      trackChanges
     )
   } catch (error) {
     if (error instanceof Errors.FileTooLargeError) {

services/document-updater/test/acceptance/js/SettingADocumentTests.js

@@ -756,6 +756,312 @@ describe('Setting a document', function () {
         })
       })
     }
+
+    describe('with track changes enabled', function () {
+      async function getDocRaw(docId) {
+        return JSON.parse(
+          await docUpdaterRedis.get(Keys.docLines({ doc_id: docId }))
+        )
+      }
+
+      function expectRecentTimestamp(tracking) {
+        expect(new Date(tracking.ts).getTime()).to.be.closeTo(
+          Date.now(),
+          30_000
+        )
+        return tracking.ts
+      }
+
+      describe('when adding content', function () {
+        beforeEach(async function () {
+          this.body = await DocUpdaterClient.setDocLines(
+            this.project_id,
+            this.doc_id,
+            ['one', 'INSERT', 'two', 'three'],
+            this.source,
+            userId,
+            false,
+            true
+          )
+        })
+
+        it('should record the insertion as a tracked insert', async function () {
+          const data = await getDocRaw(this.doc_id)
+          const insertTs = expectRecentTimestamp(
+            data.trackedChanges[0].tracking
+          )
+          expect(data).to.deep.equal({
+            content: 'one\nINSERT\none and a half\ntwo\nthree',
+            trackedChanges: [
+              {
+                range: { pos: 4, length: 'INSERT\n'.length },
+                tracking: { ts: insertTs, type: 'insert', userId },
+              },
+              {
+                range: { pos: 'INSERT\n'.length + 4, length: 15 },
+                tracking: { ts, type: 'delete', userId },
+              },
+            ],
+          })
+        })
+      })
+
+      describe('when removing content', function () {
+        beforeEach(async function () {
+          this.body = await DocUpdaterClient.setDocLines(
+            this.project_id,
+            this.doc_id,
+            ['one', 'two'],
+            this.source,
+            userId,
+            false,
+            true
+          )
+        })
+
+        it('should record the removal as a tracked delete, keeping the content', async function () {
+          const data = await getDocRaw(this.doc_id)
+          const deleteTs = expectRecentTimestamp(
+            data.trackedChanges[1].tracking
+          )
+          expect(data).to.deep.equal({
+            content: 'one\none and a half\ntwo\nthree',
+            trackedChanges: [
+              {
+                range: { pos: 4, length: 15 },
+                tracking: { ts, type: 'delete', userId },
+              },
+              {
+                range: { pos: 22, length: '\nthree'.length },
+                tracking: { ts: deleteTs, type: 'delete', userId },
+              },
+            ],
+          })
+        })
+
+        it('should hide the removed content from the doc lines', async function () {
+          const doc = await DocUpdaterClient.getDoc(
+            this.project_id,
+            this.doc_id
+          )
+          expect(doc.lines).to.deep.equal(['one', 'two'])
+        })
+      })
+
+      describe('when appending content', function () {
+        beforeEach(async function () {
+          this.body = await DocUpdaterClient.appendToDoc(
+            this.project_id,
+            this.doc_id,
+            ['four'],
+            this.source,
+            userId,
+            true
+          )
+        })
+
+        it('should record the appended content as a tracked insert', async function () {
+          const data = await getDocRaw(this.doc_id)
+          const insertTs = expectRecentTimestamp(
+            data.trackedChanges[1].tracking
+          )
+          expect(data).to.deep.equal({
+            content: 'one\none and a half\ntwo\nthree\nfour',
+            trackedChanges: [
+              {
+                range: { pos: 4, length: 15 },
+                tracking: { ts, type: 'delete', userId },
+              },
+              {
+                range: { pos: 28, length: '\nfour'.length },
+                tracking: { ts: insertTs, type: 'insert', userId },
+              },
+            ],
+          })
+        })
+
+        it('should include the appended content in the doc lines', async function () {
+          const doc = await DocUpdaterClient.getDoc(
+            this.project_id,
+            this.doc_id
+          )
+          expect(doc.lines).to.deep.equal(['one', 'two', 'three', 'four'])
+        })
+      })
+    })
+  })
+
+  describe('with track changes enabled (sharejs-text-ot)', function () {
+    const lines = ['one', 'one and a half', 'two', 'three']
+    const idSeed = '587357bd35e64f6157'
+    const userId = DocUpdaterClient.randomId()
+
+    beforeEach(async function () {
+      this.project_id = DocUpdaterClient.randomId()
+      this.doc_id = DocUpdaterClient.randomId()
+      this.update = {
+        doc: this.doc_id,
+        op: [
+          {
+            d: 'one and a half\n',
+            p: 4,
+          },
+        ],
+        meta: {
+          tc: idSeed,
+          user_id: userId,
+        },
+        v: this.version,
+      }
+      MockWebApi.insertDoc(this.project_id, this.doc_id, {
+        lines,
+        version: this.version,
+      })
+      await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
+      await DocUpdaterClient.sendUpdate(
+        this.project_id,
+        this.doc_id,
+        this.update
+      )
+      await DocUpdaterClient.waitForPendingUpdates(this.doc_id)
+    })
+
+    afterEach(function () {
+      MockProjectHistoryApi.flushProject.resetHistory()
+      MockWebApi.setDocument.resetHistory()
+    })
+
+    function expectPreservedTrackedDelete(changes) {
+      const deleteChange = changes.find(
+        change => change.op.d === 'one and a half\n'
+      )
+      expect(deleteChange.op).to.deep.equal({ d: 'one and a half\n', p: 4 })
+      expect(deleteChange.id).to.equal(idSeed + '000001')
+      expect(deleteChange.metadata.user_id).to.equal(userId)
+    }
+
+    function expectNewTrackedChange(change, op) {
+      expect(change.op).to.deep.equal(op)
+      expect(change.id).to.match(/^[0-9a-f]{24}$/)
+      expect(change.id.startsWith(idSeed)).to.equal(false)
+      expect(change.metadata.user_id).to.equal(userId)
+      expect(new Date(change.metadata.ts).getTime()).to.be.closeTo(
+        Date.now(),
+        30_000
+      )
+    }
+
+    describe('when adding content', function () {
+      beforeEach(async function () {
+        this.body = await DocUpdaterClient.setDocLines(
+          this.project_id,
+          this.doc_id,
+          ['one', 'two', 'INSERT', 'three'],
+          this.source,
+          userId,
+          false,
+          true
+        )
+      })
+
+      it('should record the insertion as a tracked insert', async function () {
+        const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
+        expect(doc.lines).to.deep.equal(['one', 'two', 'INSERT', 'three'])
+        expect(doc.ranges.changes).to.have.length(2)
+        expectPreservedTrackedDelete(doc.ranges.changes)
+        const insertChange = doc.ranges.changes.find(
+          change => change.op.i != null
+        )
+        expectNewTrackedChange(insertChange, { i: 'INSERT\n', p: 8 })
+      })
+    })
+
+    describe('when removing content', function () {
+      beforeEach(async function () {
+        this.body = await DocUpdaterClient.setDocLines(
+          this.project_id,
+          this.doc_id,
+          ['one', 'two'],
+          this.source,
+          userId,
+          false,
+          true
+        )
+      })
+
+      it('should record the removal as a tracked delete', async function () {
+        const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
+        expect(doc.lines).to.deep.equal(['one', 'two'])
+        expect(doc.ranges.changes).to.have.length(2)
+        expectPreservedTrackedDelete(doc.ranges.changes)
+        const newDeleteChange = doc.ranges.changes.find(
+          change => change.op.d === '\nthree'
+        )
+        expectNewTrackedChange(newDeleteChange, { d: '\nthree', p: 7 })
+      })
+    })
+
+    describe('when appending content', function () {
+      beforeEach(async function () {
+        this.body = await DocUpdaterClient.appendToDoc(
+          this.project_id,
+          this.doc_id,
+          ['four'],
+          this.source,
+          userId,
+          true
+        )
+      })
+
+      it('should record the appended content as a tracked insert', async function () {
+        const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
+        expect(doc.lines).to.deep.equal(['one', 'two', 'three', 'four'])
+        expect(doc.ranges.changes).to.have.length(2)
+        expectPreservedTrackedDelete(doc.ranges.changes)
+        const insertChange = doc.ranges.changes.find(
+          change => change.op.i != null
+        )
+        expectNewTrackedChange(insertChange, { i: '\nfour', p: 13 })
+      })
+    })
+  })
+
+  describe('when track changes is requested without a user id', function () {
+    beforeEach(function () {
+      this.project_id = DocUpdaterClient.randomId()
+      this.doc_id = DocUpdaterClient.randomId()
+    })
+
+    it('should reject setting the document with a 400', async function () {
+      await expect(
+        DocUpdaterClient.setDocLines(
+          this.project_id,
+          this.doc_id,
+          this.newLines,
+          this.source,
+          undefined,
+          false,
+          true
+        )
+      )
+        .to.be.rejectedWith(RequestFailedError)
+        .and.eventually.have.nested.property('response.status', 400)
+    })
+
+    it('should reject appending to the document with a 400', async function () {
+      await expect(
+        DocUpdaterClient.appendToDoc(
+          this.project_id,
+          this.doc_id,
+          this.newLines,
+          this.source,
+          undefined,
+          true
+        )
+      )
+        .to.be.rejectedWith(RequestFailedError)
+        .and.eventually.have.nested.property('response.status', 400)
+    })
   })
   describe('when the first request returns a connection error', function () {
     beforeEach(function () {

services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js

@@ -111,7 +111,15 @@ module.exports = DocUpdaterClient = {
     )
   },
 
-  async setDocLines(projectId, docId, lines, source, userId, undoing) {
+  async setDocLines(
+    projectId,
+    docId,
+    lines,
+    source,
+    userId,
+    undoing,
+    trackChanges
+  ) {
     return await fetchJson(
       `http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
       {
@@ -121,6 +129,22 @@ module.exports = DocUpdaterClient = {
           source,
           user_id: userId,
           undoing,
+          trackChanges,
+        },
+      }
+    )
+  },
+
+  async appendToDoc(projectId, docId, lines, source, userId, trackChanges) {
+    return await fetchJson(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/append`,
+      {
+        method: 'POST',
+        json: {
+          lines,
+          source,
+          user_id: userId,
+          trackChanges,
         },
       }
     )

services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js

@@ -585,6 +585,39 @@ describe('DocumentManager', function () {
         })
       })
 
+      describe('with track changes enabled', function () {
+        beforeEach(async function () {
+          await this.DocumentManager.promises.setDoc(
+            this.project_id,
+            this.doc_id,
+            this.afterLines,
+            this.source,
+            this.user_id,
+            false,
+            true,
+            true
+          )
+        })
+
+        it('should apply the diff with a track changes id seed', function () {
+          this.UpdateManager.promises.applyUpdate.should.have.been.calledWith(
+            this.project_id,
+            this.doc_id,
+            sinon.match({
+              doc: this.doc_id,
+              v: this.version,
+              op: this.ops,
+              meta: sinon.match({
+                type: 'external',
+                source: this.source,
+                user_id: this.user_id,
+                tc: sinon.match(/^[0-9a-f]{18}$/),
+              }),
+            })
+          )
+        })
+      })
+
       describe('when not already loaded', function () {
         beforeEach(async function () {
           this.DocumentManager.promises.getDoc = sinon.stub().resolves({