[document-updater] add support for flush/restore of tc with history-ot (#35823)

GitOrigin-RevId: a79b0ca97c930ca2e2f3ffeea20defbc6973468c

services/document-updater/app/js/DocumentManager.js

@@ -8,6 +8,7 @@ const Metrics = require('./Metrics')
 const HistoryManager = require('./HistoryManager')
 const Errors = require('./Errors')
 const RangesManager = require('./RangesManager')
+const HistoryConversions = require('./HistoryConversions')
 const { extractOriginOrSource } = require('./Utils')
 const { getTotalSizeOfLines } = require('./Limits')
 const Settings = require('@overleaf/settings')
@@ -307,9 +308,7 @@ const DocumentManager = {
     logger.debug({ projectId, docId, version }, 'flushing doc')
     Metrics.inc('flush-doc-if-loaded', 1, { status: 'modified' })
     if (!Array.isArray(lines)) {
-      const file = StringFileData.fromRaw(lines)
-      // TODO(24596): tc support for history-ot
-      lines = file.getLines()
+      ;({ lines, ranges } = HistoryConversions.fromHistoryOT(lines))
     }
     const result = await PersistenceManager.promises.setDoc(
       projectId,

services/document-updater/app/js/HistoryConversions.js

@@ -1,11 +1,18 @@
 // @ts-check
 
 const _ = require('lodash')
-const { isDelete } = require('./Utils')
+const RangesTracker = require('@overleaf/ranges-tracker')
+const {
+  File,
+  StringFileData,
+  getDocUpdaterCompatibleRanges,
+} = require('overleaf-editor-core')
+const { addTrackedDeletesToContent, isDelete } = require('./Utils')
 
 /**
  * @import { Comment, HistoryComment, HistoryRanges, HistoryTrackedChange } from './types'
  * @import { Ranges, TrackedChange } from './types'
+ * @import { CommentRawData, StringFileRawData, TrackedChangeRawData } from 'overleaf-editor-core/lib/types'
  */
 
 /**
@@ -174,6 +181,98 @@ function toHistoryComment(comment, offset) {
   return historyComment
 }
 
+/**
+ * Convert a history-OT doc into editor lines and ranges
+ *
+ * Tracked deletes are removed from the content and the ranges use editor
+ * positions (as maintained by the RangesTracker).
+ *
+ * @param {StringFileRawData} raw
+ * @return {{lines: string[], ranges: Ranges}}
+ */
+function fromHistoryOT(raw) {
+  const fileData = StringFileData.fromRaw(raw)
+  const { changes, comments } = getDocUpdaterCompatibleRanges(
+    new File(fileData)
+  )
+  /** @type {Ranges} */
+  const ranges = {}
+  if (changes.length > 0) {
+    ranges.changes = changes.map(change => ({
+      ...change,
+      id: RangesTracker.generateId(),
+    }))
+  }
+  if (comments.length > 0) {
+    ranges.comments = comments.map(comment => ({
+      ...comment,
+      id: comment.op.t,
+    }))
+  }
+  return { lines: fileData.getLines(), ranges }
+}
+
+/**
+ * Convert editor lines and ranges into a history-OT doc
+ *
+ * Tracked deletes are added back to the content and the ranges use history
+ * positions.
+ *
+ * @param {string[]} lines
+ * @param {Ranges} ranges
+ * @param {string[]} resolvedCommentIds
+ * @return {StringFileRawData}
+ */
+function toHistoryOT(lines, ranges, resolvedCommentIds) {
+  const historyRanges = toHistoryRanges(ranges)
+
+  /** @type {StringFileRawData} */
+  const raw = {
+    content: addTrackedDeletesToContent(lines.join('\n'), ranges.changes ?? []),
+  }
+
+  const comments = (historyRanges.comments ?? []).map(comment => {
+    const length = comment.op.hlen ?? comment.op.c.length
+    /** @type {CommentRawData} */
+    const rawComment = {
+      id: comment.op.t,
+      // Translate zero length comments into detached comments
+      ranges:
+        length > 0 ? [{ pos: comment.op.hpos ?? comment.op.p, length }] : [],
+    }
+    if (resolvedCommentIds.includes(comment.op.t)) {
+      rawComment.resolved = true
+    }
+    return rawComment
+  })
+  if (comments.length > 0) {
+    raw.comments = comments
+  }
+
+  const trackedChanges = (historyRanges.changes ?? []).map(change => {
+    /** @type {TrackedChangeRawData} */
+    const trackedChange = {
+      range: {
+        pos: change.op.hpos ?? change.op.p,
+        length: isDelete(change.op) ? change.op.d.length : change.op.i.length,
+      },
+      tracking: {
+        type: isDelete(change.op) ? 'delete' : 'insert',
+        userId: change.metadata.user_id,
+        ts: new Date(change.metadata.ts).toISOString(),
+      },
+    }
+    return trackedChange
+  })
+  if (trackedChanges.length > 0) {
+    raw.trackedChanges = trackedChanges
+  }
+
+  return raw
+}
+
 module.exports = {
   toHistoryRanges,
+  toHistoryOT,
+  fromHistoryOT,
 }

services/document-updater/app/js/PersistenceManager.js

@@ -2,6 +2,7 @@
 const { setTimeout } = require('node:timers/promises')
 const Settings = require('@overleaf/settings')
 const Errors = require('./Errors')
+const HistoryConversions = require('./HistoryConversions')
 const OError = require('@overleaf/o-error')
 const Metrics = require('./Metrics')
 const logger = require('@overleaf/logger')
@@ -67,7 +68,11 @@ async function getDocOnce(projectId, docId, options = {}) {
 
     if (body.otMigrationStage > 0) {
       // Use history-ot
-      body.lines = { content: body.lines.join('\n') }
+      body.lines = HistoryConversions.toHistoryOT(
+        body.lines,
+        body.ranges || {},
+        body.resolvedCommentIds || []
+      )
       body.ranges = {}
     }
 

services/document-updater/test/unit/js/HistoryConversionsTests.js

@@ -67,6 +67,112 @@ describe('HistoryConversions', function () {
       ])
     })
   })
+
+  describe('toHistoryOT', function () {
+    it('handles empty ranges', function () {
+      expect(HistoryConversions.toHistoryOT(['one two'], {}, [])).to.deep.equal(
+        {
+          content: 'one two',
+        }
+      )
+    })
+
+    it('converts tracked changes and comments', function () {
+      const metadata = makeMetadata()
+      const ranges = {
+        changes: [
+          { id: 'change0', op: { p: 4, d: 'two ' }, metadata },
+          { id: 'change1', op: { p: 4, i: 'three' }, metadata },
+        ],
+        comments: [
+          { id: 'comment0', op: { p: 4, c: 'three', t: 'comment0' } },
+          { id: 'comment1', op: { p: 0, c: '', t: 'comment1' } },
+        ],
+      }
+      const raw = HistoryConversions.toHistoryOT(['one three'], ranges, [
+        'comment0',
+      ])
+      expect(raw).to.deep.equal({
+        content: 'one two three',
+        comments: [
+          {
+            id: 'comment1', // detached comment, sorted first by position
+            ranges: [],
+          },
+          {
+            id: 'comment0',
+            ranges: [{ pos: 8, length: 5 }],
+            resolved: true,
+          },
+        ],
+        trackedChanges: [
+          {
+            range: { pos: 4, length: 4 },
+            tracking: { type: 'delete', userId: 'user-id', ts: metadata.ts },
+          },
+          {
+            range: { pos: 8, length: 5 },
+            tracking: { type: 'insert', userId: 'user-id', ts: metadata.ts },
+          },
+        ],
+      })
+    })
+  })
+
+  describe('fromHistoryOT', function () {
+    it('handles a doc without ranges', function () {
+      expect(
+        HistoryConversions.fromHistoryOT({ content: 'one two' })
+      ).to.deep.equal({
+        lines: ['one two'],
+        ranges: {},
+      })
+    })
+
+    it('converts tracked changes and comments', function () {
+      const ts = new Date().toISOString()
+      const raw = {
+        content: 'one two three',
+        trackedChanges: [
+          {
+            range: { pos: 4, length: 4 },
+            tracking: { type: 'delete', userId: 'user-id', ts },
+          },
+          {
+            range: { pos: 8, length: 5 },
+            tracking: { type: 'insert', userId: 'user-id', ts },
+          },
+        ],
+        comments: [
+          { id: 'comment0', ranges: [{ pos: 8, length: 5 }], resolved: true },
+          { id: 'comment1', ranges: [] },
+        ],
+      }
+      const { lines, ranges } = HistoryConversions.fromHistoryOT(raw)
+      expect(lines).to.deep.equal(['one three'])
+
+      for (const change of ranges.changes) {
+        expect(change.id).to.be.a('string').and.to.have.length.above(0)
+      }
+      expect(
+        ranges.changes.map(({ op, metadata }) => ({ op, metadata }))
+      ).to.deep.equal([
+        { op: { p: 4, d: 'two ' }, metadata: { user_id: 'user-id', ts } },
+        { op: { p: 4, i: 'three' }, metadata: { user_id: 'user-id', ts } },
+      ])
+
+      expect(ranges.comments).to.deep.equal([
+        {
+          id: 'comment0',
+          op: { p: 4, c: 'three', t: 'comment0', resolved: true },
+        },
+        {
+          id: 'comment1',
+          op: { p: 0, c: '', t: 'comment1', resolved: false },
+        },
+      ])
+    })
+  })
 })
 
 function makeComment(id, pos, length) {