diff --git a/services/document-updater/app/js/DocumentManager.js b/services/document-updater/app/js/DocumentManager.js index 75a531d2a4..7b742b14ed 100644 --- a/services/document-updater/app/js/DocumentManager.js +++ b/services/document-updater/app/js/DocumentManager.js @@ -8,6 +8,7 @@ const Metrics = require('./Metrics') const HistoryManager = require('./HistoryManager') const Errors = require('./Errors') const RangesManager = require('./RangesManager') +const HistoryConversions = require('./HistoryConversions') const { extractOriginOrSource } = require('./Utils') const { getTotalSizeOfLines } = require('./Limits') const Settings = require('@overleaf/settings') @@ -307,9 +308,7 @@ const DocumentManager = { logger.debug({ projectId, docId, version }, 'flushing doc') Metrics.inc('flush-doc-if-loaded', 1, { status: 'modified' }) if (!Array.isArray(lines)) { - const file = StringFileData.fromRaw(lines) - // TODO(24596): tc support for history-ot - lines = file.getLines() + ;({ lines, ranges } = HistoryConversions.fromHistoryOT(lines)) } const result = await PersistenceManager.promises.setDoc( projectId, diff --git a/services/document-updater/app/js/HistoryConversions.js b/services/document-updater/app/js/HistoryConversions.js index 50db911ed6..f44948d5f9 100644 --- a/services/document-updater/app/js/HistoryConversions.js +++ b/services/document-updater/app/js/HistoryConversions.js @@ -1,11 +1,18 @@ // @ts-check const _ = require('lodash') -const { isDelete } = require('./Utils') +const RangesTracker = require('@overleaf/ranges-tracker') +const { + File, + StringFileData, + getDocUpdaterCompatibleRanges, +} = require('overleaf-editor-core') +const { addTrackedDeletesToContent, isDelete } = require('./Utils') /** * @import { Comment, HistoryComment, HistoryRanges, HistoryTrackedChange } from './types' * @import { Ranges, TrackedChange } from './types' + * @import { CommentRawData, StringFileRawData, TrackedChangeRawData } from 'overleaf-editor-core/lib/types' */ /** @@ -174,6 +181,98 @@ function toHistoryComment(comment, offset) { return historyComment } +/** + * Convert a history-OT doc into editor lines and ranges + * + * Tracked deletes are removed from the content and the ranges use editor + * positions (as maintained by the RangesTracker). + * + * @param {StringFileRawData} raw + * @return {{lines: string[], ranges: Ranges}} + */ +function fromHistoryOT(raw) { + const fileData = StringFileData.fromRaw(raw) + const { changes, comments } = getDocUpdaterCompatibleRanges( + new File(fileData) + ) + /** @type {Ranges} */ + const ranges = {} + if (changes.length > 0) { + ranges.changes = changes.map(change => ({ + ...change, + id: RangesTracker.generateId(), + })) + } + if (comments.length > 0) { + ranges.comments = comments.map(comment => ({ + ...comment, + id: comment.op.t, + })) + } + return { lines: fileData.getLines(), ranges } +} + +/** + * Convert editor lines and ranges into a history-OT doc + * + * Tracked deletes are added back to the content and the ranges use history + * positions. + * + * @param {string[]} lines + * @param {Ranges} ranges + * @param {string[]} resolvedCommentIds + * @return {StringFileRawData} + */ +function toHistoryOT(lines, ranges, resolvedCommentIds) { + const historyRanges = toHistoryRanges(ranges) + + /** @type {StringFileRawData} */ + const raw = { + content: addTrackedDeletesToContent(lines.join('\n'), ranges.changes ?? []), + } + + const comments = (historyRanges.comments ?? []).map(comment => { + const length = comment.op.hlen ?? comment.op.c.length + /** @type {CommentRawData} */ + const rawComment = { + id: comment.op.t, + // Translate zero length comments into detached comments + ranges: + length > 0 ? [{ pos: comment.op.hpos ?? comment.op.p, length }] : [], + } + if (resolvedCommentIds.includes(comment.op.t)) { + rawComment.resolved = true + } + return rawComment + }) + if (comments.length > 0) { + raw.comments = comments + } + + const trackedChanges = (historyRanges.changes ?? []).map(change => { + /** @type {TrackedChangeRawData} */ + const trackedChange = { + range: { + pos: change.op.hpos ?? change.op.p, + length: isDelete(change.op) ? change.op.d.length : change.op.i.length, + }, + tracking: { + type: isDelete(change.op) ? 'delete' : 'insert', + userId: change.metadata.user_id, + ts: new Date(change.metadata.ts).toISOString(), + }, + } + return trackedChange + }) + if (trackedChanges.length > 0) { + raw.trackedChanges = trackedChanges + } + + return raw +} + module.exports = { toHistoryRanges, + toHistoryOT, + fromHistoryOT, } diff --git a/services/document-updater/app/js/PersistenceManager.js b/services/document-updater/app/js/PersistenceManager.js index 3f4b03b68c..bf8f580ef4 100644 --- a/services/document-updater/app/js/PersistenceManager.js +++ b/services/document-updater/app/js/PersistenceManager.js @@ -2,6 +2,7 @@ const { setTimeout } = require('node:timers/promises') const Settings = require('@overleaf/settings') const Errors = require('./Errors') +const HistoryConversions = require('./HistoryConversions') const OError = require('@overleaf/o-error') const Metrics = require('./Metrics') const logger = require('@overleaf/logger') @@ -67,7 +68,11 @@ async function getDocOnce(projectId, docId, options = {}) { if (body.otMigrationStage > 0) { // Use history-ot - body.lines = { content: body.lines.join('\n') } + body.lines = HistoryConversions.toHistoryOT( + body.lines, + body.ranges || {}, + body.resolvedCommentIds || [] + ) body.ranges = {} } diff --git a/services/document-updater/test/unit/js/HistoryConversionsTests.js b/services/document-updater/test/unit/js/HistoryConversionsTests.js index 2bd18c31a2..5d101e1a5c 100644 --- a/services/document-updater/test/unit/js/HistoryConversionsTests.js +++ b/services/document-updater/test/unit/js/HistoryConversionsTests.js @@ -67,6 +67,112 @@ describe('HistoryConversions', function () { ]) }) }) + + describe('toHistoryOT', function () { + it('handles empty ranges', function () { + expect(HistoryConversions.toHistoryOT(['one two'], {}, [])).to.deep.equal( + { + content: 'one two', + } + ) + }) + + it('converts tracked changes and comments', function () { + const metadata = makeMetadata() + const ranges = { + changes: [ + { id: 'change0', op: { p: 4, d: 'two ' }, metadata }, + { id: 'change1', op: { p: 4, i: 'three' }, metadata }, + ], + comments: [ + { id: 'comment0', op: { p: 4, c: 'three', t: 'comment0' } }, + { id: 'comment1', op: { p: 0, c: '', t: 'comment1' } }, + ], + } + const raw = HistoryConversions.toHistoryOT(['one three'], ranges, [ + 'comment0', + ]) + expect(raw).to.deep.equal({ + content: 'one two three', + comments: [ + { + id: 'comment1', // detached comment, sorted first by position + ranges: [], + }, + { + id: 'comment0', + ranges: [{ pos: 8, length: 5 }], + resolved: true, + }, + ], + trackedChanges: [ + { + range: { pos: 4, length: 4 }, + tracking: { type: 'delete', userId: 'user-id', ts: metadata.ts }, + }, + { + range: { pos: 8, length: 5 }, + tracking: { type: 'insert', userId: 'user-id', ts: metadata.ts }, + }, + ], + }) + }) + }) + + describe('fromHistoryOT', function () { + it('handles a doc without ranges', function () { + expect( + HistoryConversions.fromHistoryOT({ content: 'one two' }) + ).to.deep.equal({ + lines: ['one two'], + ranges: {}, + }) + }) + + it('converts tracked changes and comments', function () { + const ts = new Date().toISOString() + const raw = { + content: 'one two three', + trackedChanges: [ + { + range: { pos: 4, length: 4 }, + tracking: { type: 'delete', userId: 'user-id', ts }, + }, + { + range: { pos: 8, length: 5 }, + tracking: { type: 'insert', userId: 'user-id', ts }, + }, + ], + comments: [ + { id: 'comment0', ranges: [{ pos: 8, length: 5 }], resolved: true }, + { id: 'comment1', ranges: [] }, + ], + } + const { lines, ranges } = HistoryConversions.fromHistoryOT(raw) + expect(lines).to.deep.equal(['one three']) + + for (const change of ranges.changes) { + expect(change.id).to.be.a('string').and.to.have.length.above(0) + } + expect( + ranges.changes.map(({ op, metadata }) => ({ op, metadata })) + ).to.deep.equal([ + { op: { p: 4, d: 'two ' }, metadata: { user_id: 'user-id', ts } }, + { op: { p: 4, i: 'three' }, metadata: { user_id: 'user-id', ts } }, + ]) + + expect(ranges.comments).to.deep.equal([ + { + id: 'comment0', + op: { p: 4, c: 'three', t: 'comment0', resolved: true }, + }, + { + id: 'comment1', + op: { p: 0, c: '', t: 'comment1', resolved: false }, + }, + ]) + }) + }) }) function makeComment(id, pos, length) {