diff --git a/libraries/overleaf-editor-core/lib/build_set_content_operations.js b/libraries/overleaf-editor-core/lib/build_set_content_operations.js new file mode 100644 index 0000000000..d3d7ac31d6 --- /dev/null +++ b/libraries/overleaf-editor-core/lib/build_set_content_operations.js @@ -0,0 +1,138 @@ +// @ts-check +'use strict' + +const _ = require('lodash') +const OError = require('@overleaf/o-error') +const File = require('./file') +const Operation = require('./operation') +const TrackingProps = require('./file_data/tracking_props') +const { diffAsTextOperation } = require('./diff_as_text_operation') + +/** + * @import Blob from './blob' + * @import StringFileData from './file_data/string_file_data' + * @import { BlobStore, RangesBlob } from './types' + */ + +/** + * Build the operations that set the content of a pathname (a setDoc-like + * upsert). + * + * Exactly one of `content` (editable doc) and `blob` (binary file) must be + * provided. An existing editable doc is edited in place with a minimal diff + * (see diffAsTextOperation); any other case replaces or creates the file at + * the pathname. New content is stored in the blob store and referenced by + * hash, so the returned operations are cheap to serialize. + * + * @param {object} args + * @param {File | null | undefined} args.file existing file at the pathname + * @param {string} args.pathname + * @param {string} [args.content] new doc content + * @param {Blob} [args.blob] blob with the new file content + * @param {Object} [args.metadata] metadata for the file; the file's metadata + * is always replaced with this value (defaults to empty metadata) + * @param {{userId: string, ts: Date}} [args.tracking] record the change as + * tracked changes (ignored when replacing with a blob) + * @param {BlobStore} args.blobStore + * @return {Promise<{operations: Operation[], status: 'applied' | 'noop'}>} + */ +async function buildSetContentOperations({ + file, + pathname, + content, + blob, + metadata = {}, + tracking, + blobStore, +}) { + if ((content == null) === (blob == null)) { + throw new OError( + 'buildSetContentOperations: exactly one of content and blob must be given', + { pathname } + ) + } + + if (content != null) { + if (file) { + // Eager loading resolves both the editability of files whose data is a + // bare hash and the content to diff against. + await file.load('eager', blobStore) + if (file.isEditable()) { + return buildEditOperations(file, pathname, content, metadata, tracking) + } + } + blob = await blobStore.putString(content) + } + if (blob == null) { + throw new OError('buildSetContentOperations: no blob', { pathname }) + } + + if (file && file.getHash() === blob.getHash()) { + // LazyStringFileData yields no hash with buffered changes, no false-positive hash matches possible. + if (_.isEqual(file.getMetadata(), metadata)) { + return { operations: [], status: 'noop' } + } + return { + operations: [Operation.setFileMetadata(pathname, metadata)], + status: 'applied', + } + } + + let rangesBlob + if (content != null && content.length > 0 && tracking) { + // Record the whole new doc as a tracked insert. + /** @type {RangesBlob} */ + const ranges = { + comments: [], + trackedChanges: [ + { + range: { pos: 0, length: content.length }, + tracking: new TrackingProps( + 'insert', + tracking.userId, + tracking.ts + ).toRaw(), + }, + ], + } + rangesBlob = await blobStore.putObject(ranges) + } + + const newFile = File.createLazyFromBlobs(blob, rangesBlob, metadata) + const operations = [] + if (file) { + operations.push(Operation.removeFile(pathname)) + } + operations.push(Operation.addFile(pathname, newFile)) + return { operations, status: 'applied' } +} + +/** + * @param {File} file eagerly loaded editable file + * @param {string} pathname + * @param {string} content + * @param {Object} metadata metadata to set on the doc + * @param {{userId: string, ts: Date}} [tracking] + * @return {{operations: Operation[], status: 'applied' | 'noop'}} + */ +function buildEditOperations(file, pathname, content, metadata, tracking) { + const fileData = /** @type {StringFileData} */ (file.data) + const textOperation = diffAsTextOperation( + fileData, + content, + tracking ? { tracking } : {} + ) + const operations = [] + if (!textOperation.isNoop()) { + operations.push(Operation.editFile(pathname, textOperation)) + } + if (!_.isEqual(file.getMetadata(), metadata)) { + operations.push(Operation.setFileMetadata(pathname, metadata)) + } + if (operations.length === 0) { + return { operations, status: 'noop' } + } + return { operations, status: 'applied' } +} + +module.exports = { buildSetContentOperations } diff --git a/libraries/overleaf-editor-core/lib/diff_as_text_operation.js b/libraries/overleaf-editor-core/lib/diff_as_text_operation.js new file mode 100644 index 0000000000..1313a8b3e4 --- /dev/null +++ b/libraries/overleaf-editor-core/lib/diff_as_text_operation.js @@ -0,0 +1,177 @@ +// @ts-check +'use strict' + +const DMP = require('diff-match-patch') +const OError = require('@overleaf/o-error') +const TextOperation = require('./operation/text_operation') +const TrackingProps = require('./file_data/tracking_props') + +/** + * @import StringFileData from './file_data/string_file_data' + */ + +const ADDED = 1 +const REMOVED = -1 +const UNCHANGED = 0 + +const dmp = new DMP() + +// Do not attempt to produce a diff for more than 100ms +dmp.Diff_Timeout = 0.1 + +/** + * Diff the file's content against `after` and return a minimal TextOperation + * turning the file into it. The diff is computed against the content with + * tracked deletes filtered out (the content external sources see); the + * returned operation preserves the file's tracked deletes. + * + * @param {StringFileData} file eagerly loaded file data + * @param {string} after + * @param {object} [opts] + * @param {{userId: string, ts: Date}} [opts.tracking] record the edit as + * tracked changes + * @return {TextOperation} + */ +function diffAsTextOperation(file, after, opts = {}) { + const beforeWithoutTrackedDeletes = file.getContent({ + filterTrackedDeletes: true, + }) + const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after) + dmp.diff_cleanupSemantic(diffs) + return diffsToTextOperation(file, diffs, opts) +} + +/** + * Convert diff-match-patch style diffs into a TextOperation. + * + * The diffs must have been computed against the file content with tracked + * deletes filtered out, i.e. `file.getContent({ filterTrackedDeletes: true })`, + * which is the content that external sources (uploads, github-sync, tpds) see. + * The returned operation applies to the full content: positions are mapped to + * step over tracked-delete spans with plain retains, preserving them. + * + * When `opts.tracking` is set, the edit is recorded as tracked changes: + * insertions become tracked inserts and removals become tracked deletes, + * except inside existing tracked inserts, where removals stay regular + * deletes, and over existing tracked deletes, which are left untouched + * (keeping the original author). This mirrors the semantics of updates with + * track changes enabled in project-history's UpdateTranslator. + * + * Exported for tests: use diffAsTextOperation instead. + * + * @param {StringFileData} file eagerly loaded file data + * @param {[number, string][]} diffs diff-match-patch diffs + * @param {object} [opts] + * @param {{userId: string, ts: Date}} [opts.tracking] record the edit as + * tracked changes + * @return {TextOperation} + */ +function diffsToTextOperation(file, diffs, opts = {}) { + const insertTracking = opts.tracking + ? new TrackingProps('insert', opts.tracking.userId, opts.tracking.ts) + : undefined + const deleteTracking = opts.tracking + ? new TrackingProps('delete', opts.tracking.userId, opts.tracking.ts) + : undefined + + const trackedChanges = file.trackedChanges.asSorted() + let tcIndex = 0 + + const op = new TextOperation() + + /** + * Consume removed content: a plain remove, or a tracked delete when + * recording tracked changes. + * + * @param {number} length + */ + function removeContent(length) { + if (deleteTracking) { + op.retain(length, { tracking: deleteTracking }) + } else { + op.remove(length) + } + } + + for (const diff of diffs) { + const type = diff[0] + let content = diff[1] + if (type === ADDED) { + op.insert(content, insertTracking ? { tracking: insertTracking } : {}) + continue + } + if (type !== REMOVED && type !== UNCHANGED) { + throw new Error('Unknown type') + } + + while (tcIndex < trackedChanges.length) { + const tc = trackedChanges[tcIndex] + const segmentEnd = op.baseLength + content.length + if (tc.range.start >= segmentEnd) break + if (tc.tracking.type === 'delete') { + // Tracked deletes are invisible in the diffed content. Step over + // them with a plain retain, preserving the original tracked delete. + const before = tc.range.start - op.baseLength + if (type === REMOVED) { + removeContent(before) + } else { + op.retain(before) + } + op.retain(tc.range.length) + content = content.slice(before) + tcIndex++ + } else if (type === REMOVED && deleteTracking) { + // Removals inside existing tracked inserts are always regular + // deletes, even when recording tracked changes. + const before = Math.max(0, tc.range.start - op.baseLength) + removeContent(before) + const overlap = Math.min(tc.range.end, segmentEnd) - op.baseLength + op.remove(overlap) + content = content.slice(before + overlap) + if (tc.range.end <= op.baseLength) { + tcIndex++ + } else { + // The tracked insert extends beyond this diff segment. + break + } + } else { + // Tracked inserts are ordinary visible content otherwise. Only move + // past them once the whole range has been covered by diff segments. + if (tc.range.end <= segmentEnd) { + tcIndex++ + } else { + break + } + } + } + + if (type === REMOVED) { + removeContent(content.length) + } else { + op.retain(content.length) + } + } + + // Any tracked deletes after the end of the diffed content must be retained. + while (tcIndex < trackedChanges.length) { + const tc = trackedChanges[tcIndex] + if (tc.tracking.type !== 'delete' || tc.range.start !== op.baseLength) { + throw new OError( + 'StringFileData.trackedChanges out of sync: unexpected range after end of diff', + { nextTc: tc, baseLength: op.baseLength } + ) + } + op.retain(tc.range.length) + tcIndex++ + } + + return op +} + +module.exports = { + diffAsTextOperation, + diffsToTextOperation, + ADDED, + REMOVED, + UNCHANGED, +} diff --git a/libraries/overleaf-editor-core/package.json b/libraries/overleaf-editor-core/package.json index 444579a0d4..f3ba1b0be6 100644 --- a/libraries/overleaf-editor-core/package.json +++ b/libraries/overleaf-editor-core/package.json @@ -36,6 +36,7 @@ "@overleaf/o-error": "workspace:*", "@overleaf/validation-tools": "workspace:*", "check-types": "^5.1.0", + "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688", "lodash": "^4.18.1", "p-map": "^4.0.0", "path-browserify": "^1.0.1" diff --git a/libraries/overleaf-editor-core/test/unit/build_set_content_operations.test.js b/libraries/overleaf-editor-core/test/unit/build_set_content_operations.test.js new file mode 100644 index 0000000000..5c75a509d0 Binary files /dev/null and b/libraries/overleaf-editor-core/test/unit/build_set_content_operations.test.js differ diff --git a/libraries/overleaf-editor-core/test/unit/diff_as_text_operation.test.js b/libraries/overleaf-editor-core/test/unit/diff_as_text_operation.test.js new file mode 100644 index 0000000000..b99873d727 --- /dev/null +++ b/libraries/overleaf-editor-core/test/unit/diff_as_text_operation.test.js @@ -0,0 +1,362 @@ +// @ts-check +'use strict' + +const { expect } = require('chai') + +const StringFileData = require('../../lib/file_data/string_file_data') +const { + diffAsTextOperation, + diffsToTextOperation, + ADDED, + REMOVED, + UNCHANGED, +} = require('../../lib/diff_as_text_operation') + +/** @import { TrackedChangeRawData } from '../../lib/types' */ + +const TS = new Date('2026-07-10T00:00:00.000Z') +const OTHER_TS = new Date('2026-07-09T00:00:00.000Z') +const TRACKING = { userId: 'user-1', ts: TS } + +/** + * @param {'insert' | 'delete'} type + * @param {number} pos + * @param {number} length + * @param {string} [userId] + * @return {TrackedChangeRawData} + */ +function trackedChange(type, pos, length, userId = 'user-2') { + return { + range: { pos, length }, + tracking: { type, userId, ts: OTHER_TS.toISOString() }, + } +} + +describe('diffAsTextOperation', function () { + /** + * @param {StringFileData} file + * @param {string} after + * @param {object} [opts] + */ + function applyDiff(file, after, opts) { + const operation = diffAsTextOperation(file, after, opts) + file.edit(operation) + return operation + } + + it('returns a noop operation for unchanged content', function () { + const file = new StringFileData('hello world') + const operation = diffAsTextOperation(file, 'hello world') + expect(operation.isNoop()).to.be.true + }) + + it('builds a minimal edit operation', function () { + const file = new StringFileData('hello world') + const operation = applyDiff(file, 'hello brave world') + expect(file.getContent()).to.equal('hello brave world') + expect(operation.toJSON()).to.eql({ textOperation: [6, 'brave ', 5] }) + }) + + it('diffs against the content without tracked deletes and preserves them', function () { + const file = StringFileData.fromRaw({ + content: 'hello cruel world', + trackedChanges: [trackedChange('delete', 6, 6)], + }) + applyDiff(file, 'hello brave world') + expect(file.getContent()).to.equal('hello brave cruel world') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello brave world' + ) + expect(file.trackedChanges.toRaw()).to.eql([trackedChange('delete', 12, 6)]) + }) + + it('records an insertion as a tracked insert when tracking is given', function () { + const file = new StringFileData('hello world') + applyDiff(file, 'hello brave world', { tracking: TRACKING }) + expect(file.getContent()).to.equal('hello brave world') + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 6, length: 6 }, + tracking: { type: 'insert', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('records a removal as a tracked delete when tracking is given', function () { + const file = new StringFileData('hello cruel world') + applyDiff(file, 'hello world', { tracking: TRACKING }) + expect(file.getContent()).to.equal('hello cruel world') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello world' + ) + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 6, length: 6 }, + tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) +}) + +describe('diffsToTextOperation', function () { + /** + * @param {StringFileData} file + * @param {[number, string][]} diffs + * @param {object} [opts] + */ + function applyDiffs(file, diffs, opts) { + const operation = diffsToTextOperation(file, diffs, opts) + file.edit(operation) + return operation + } + + describe('without tracked changes in the file', function () { + it('converts an insertion', function () { + const file = new StringFileData('hello world') + applyDiffs(file, [ + [UNCHANGED, 'hello '], + [ADDED, 'brave '], + [UNCHANGED, 'world'], + ]) + expect(file.getContent()).to.equal('hello brave world') + expect(file.trackedChanges.toRaw()).to.eql([]) + }) + + it('converts a removal', function () { + const file = new StringFileData('hello cruel world') + applyDiffs(file, [ + [UNCHANGED, 'hello '], + [REMOVED, 'cruel '], + [UNCHANGED, 'world'], + ]) + expect(file.getContent()).to.equal('hello world') + }) + + it('returns a noop operation for identical content', function () { + const file = new StringFileData('hello world') + const operation = diffsToTextOperation(file, [[UNCHANGED, 'hello world']]) + expect(operation.isNoop()).to.be.true + }) + + it('throws on unknown diff types', function () { + const file = new StringFileData('hello') + expect(() => diffsToTextOperation(file, [[2, 'hello']])).to.throw( + 'Unknown type' + ) + }) + }) + + describe('with tracked deletes in the file', function () { + it('retains a tracked delete inside an unchanged region', function () { + const file = new StringFileData('hello XXXworld', undefined, [ + trackedChange('delete', 6, 3), + ]) + applyDiffs(file, [ + [UNCHANGED, 'hello '], + [ADDED, 'brave '], + [UNCHANGED, 'world'], + ]) + expect(file.getContent()).to.equal('hello brave XXXworld') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello brave world' + ) + expect(file.trackedChanges.toRaw()).to.eql([ + trackedChange('delete', 12, 3), + ]) + }) + + it('retains a tracked delete inside a removed region', function () { + const file = new StringFileData('abCCCcdef', undefined, [ + trackedChange('delete', 2, 3), + ]) + applyDiffs(file, [ + [UNCHANGED, 'a'], + [REMOVED, 'bc'], + [UNCHANGED, 'def'], + ]) + expect(file.getContent()).to.equal('aCCCdef') + expect(file.trackedChanges.toRaw()).to.eql([ + trackedChange('delete', 1, 3), + ]) + }) + + it('retains tracked deletes after the end of the diff', function () { + const file = new StringFileData('abcDDD', undefined, [ + trackedChange('delete', 3, 3), + ]) + const operation = applyDiffs(file, [[UNCHANGED, 'abc']]) + expect(operation.baseLength).to.equal(6) + expect(file.getContent()).to.equal('abcDDD') + expect(file.trackedChanges.toRaw()).to.eql([ + trackedChange('delete', 3, 3), + ]) + }) + + it('throws when the tracked changes are out of sync', function () { + const file = new StringFileData('abcDE', undefined, [ + trackedChange('delete', 4, 1), + ]) + expect(() => diffsToTextOperation(file, [[UNCHANGED, 'abc']])).to.throw( + 'StringFileData.trackedChanges out of sync' + ) + }) + }) + + describe('recording the edit as tracked changes', function () { + it('records insertions as tracked inserts', function () { + const file = new StringFileData('hello world') + applyDiffs( + file, + [ + [UNCHANGED, 'hello '], + [ADDED, 'brave '], + [UNCHANGED, 'world'], + ], + { tracking: TRACKING } + ) + expect(file.getContent()).to.equal('hello brave world') + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 6, length: 6 }, + tracking: { type: 'insert', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('records removals as tracked deletes', function () { + const file = new StringFileData('hello cruel world') + applyDiffs( + file, + [ + [UNCHANGED, 'hello '], + [REMOVED, 'cruel '], + [UNCHANGED, 'world'], + ], + { tracking: TRACKING } + ) + expect(file.getContent()).to.equal('hello cruel world') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello world' + ) + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 6, length: 6 }, + tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('records a whole-document removal as a tracked delete', function () { + const file = new StringFileData('abc') + const operation = diffsToTextOperation(file, [[REMOVED, 'abc']], { + tracking: TRACKING, + }) + file.edit(operation) + expect(file.getContent({ filterTrackedDeletes: true })).to.equal('') + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 0, length: 3 }, + tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('leaves existing tracked deletes untouched', function () { + const file = new StringFileData('hello CCCworld', undefined, [ + trackedChange('delete', 6, 3), + ]) + applyDiffs( + file, + [ + [UNCHANGED, 'hello '], + [REMOVED, 'wor'], + [UNCHANGED, 'ld'], + ], + { tracking: TRACKING } + ) + expect(file.getContent()).to.equal('hello CCCworld') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello ld' + ) + expect(file.trackedChanges.toRaw()).to.eql([ + trackedChange('delete', 6, 3), + { + range: { pos: 9, length: 3 }, + tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('removes content inside existing tracked inserts as a regular delete', function () { + const file = new StringFileData('hello INSworld', undefined, [ + trackedChange('insert', 6, 3), + ]) + applyDiffs( + file, + [ + [UNCHANGED, 'hello '], + [REMOVED, 'INS'], + [UNCHANGED, 'world'], + ], + { tracking: TRACKING } + ) + expect(file.getContent()).to.equal('hello world') + expect(file.trackedChanges.toRaw()).to.eql([]) + }) + + it('splits a removal spanning an existing tracked insert', function () { + const file = new StringFileData('abXYcd', undefined, [ + trackedChange('insert', 2, 2), + ]) + applyDiffs( + file, + [ + [UNCHANGED, 'a'], + [REMOVED, 'bXYc'], + [UNCHANGED, 'd'], + ], + { tracking: TRACKING } + ) + // The tracked insert is removed for real; b and c become tracked deletes + expect(file.getContent()).to.equal('abcd') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal('ad') + expect(file.trackedChanges.toRaw()).to.eql([ + { + range: { pos: 1, length: 2 }, + tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() }, + }, + ]) + }) + + it('handles a tracked insert straddling removed and unchanged regions', function () { + const file = new StringFileData('hello INSins', undefined, [ + trackedChange('insert', 6, 6), + ]) + applyDiffs( + file, + [ + [UNCHANGED, 'hello '], + [REMOVED, 'INS'], + [UNCHANGED, 'ins'], + ], + { tracking: TRACKING } + ) + expect(file.getContent()).to.equal('hello ins') + expect(file.trackedChanges.toRaw()).to.eql([ + trackedChange('insert', 6, 3), + ]) + }) + + it('keeps identical content a noop', function () { + const file = new StringFileData('hello world') + const operation = diffsToTextOperation( + file, + [[UNCHANGED, 'hello world']], + { tracking: TRACKING } + ) + expect(operation.isNoop()).to.be.true + file.edit(operation) + expect(file.trackedChanges.toRaw()).to.eql([]) + }) + }) +}) diff --git a/services/document-updater/app/js/DiffCodec.js b/services/document-updater/app/js/DiffCodec.js index 17da409386..207154d2d5 100644 --- a/services/document-updater/app/js/DiffCodec.js +++ b/services/document-updater/app/js/DiffCodec.js @@ -1,6 +1,4 @@ -const OError = require('@overleaf/o-error') const DMP = require('diff-match-patch') -const { TextOperation } = require('overleaf-editor-core') const dmp = new DMP() // Do not attempt to produce a diff for more than 100ms @@ -38,63 +36,4 @@ module.exports = { } return ops }, - - /** - * @param {import("overleaf-editor-core").StringFileData} file - * @param {string} after - * @return {TextOperation} - */ - diffAsHistoryOTEditOperation(file, after) { - const beforeWithoutTrackedDeletes = file.getContent({ - filterTrackedDeletes: true, - }) - const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after) - dmp.diff_cleanupSemantic(diffs) - - const trackedChanges = file.trackedChanges.asSorted() - let nextTc = trackedChanges.shift() - - const op = new TextOperation() - for (const diff of diffs) { - let [type, content] = diff - if (type === this.ADDED) { - op.insert(content) - } else if (type === this.REMOVED || type === this.UNCHANGED) { - while (op.baseLength + content.length > nextTc?.range.start) { - if (nextTc.tracking.type === 'delete') { - const untilRange = nextTc.range.start - op.baseLength - if (type === this.REMOVED) { - op.remove(untilRange) - } else if (type === this.UNCHANGED) { - op.retain(untilRange) - } - op.retain(nextTc.range.end - nextTc.range.start) - content = content.slice(untilRange) - } - nextTc = trackedChanges.shift() - } - if (type === this.REMOVED) { - op.remove(content.length) - } else if (type === this.UNCHANGED) { - op.retain(content.length) - } - } else { - throw new Error('Unknown type') - } - } - while (nextTc) { - if ( - nextTc.tracking.type !== 'delete' || - nextTc.range.start !== op.baseLength - ) { - throw new OError( - 'StringFileData.trackedChanges out of sync: unexpected range after end of diff', - { nextTc, baseLength: op.baseLength } - ) - } - op.retain(nextTc.range.end - nextTc.range.start) - nextTc = trackedChanges.shift() - } - return op - }, } diff --git a/services/document-updater/app/js/DocumentManager.js b/services/document-updater/app/js/DocumentManager.js index 11b71f601b..8f1f48db4a 100644 --- a/services/document-updater/app/js/DocumentManager.js +++ b/services/document-updater/app/js/DocumentManager.js @@ -12,6 +12,9 @@ const { extractOriginOrSource } = require('./Utils') const { getTotalSizeOfLines } = require('./Limits') const Settings = require('@overleaf/settings') const { StringFileData } = require('overleaf-editor-core') +const { + diffAsTextOperation, +} = require('overleaf-editor-core/lib/diff_as_text_operation') const MAX_UNFLUSHED_AGE = Settings.maxUnflushedAgeMs // document should be flushed to mongo this time after a change @@ -194,10 +197,7 @@ const DocumentManager = { let op if (type === 'history-ot') { const file = StringFileData.fromRaw(oldLines) - const operation = DiffCodec.diffAsHistoryOTEditOperation( - file, - newLines.join('\n') - ) + const operation = diffAsTextOperation(file, newLines.join('\n')) if (operation.isNoop()) { op = [] } else { diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js index dfc0c40159..8d585220b7 100644 --- a/services/history-v1/api/controllers/project_import.js +++ b/services/history-v1/api/controllers/project_import.js @@ -12,10 +12,12 @@ const Change = core.Change const Chunk = core.Chunk const File = core.File const FileMap = core.FileMap +const Origin = core.Origin const Snapshot = core.Snapshot const TextOperation = core.TextOperation const logger = require('@overleaf/logger') +const OError = require('@overleaf/o-error') const storage = require('../../storage') const BatchBlobStore = storage.BatchBlobStore @@ -24,6 +26,7 @@ const chunkStore = storage.chunkStore const HashCheckBlobStore = storage.HashCheckBlobStore const commitChanges = storage.commitChanges const persistBuffer = storage.persistBuffer +const buildSetContentChange = storage.buildSetContentChange const InvalidChangeError = storage.InvalidChangeError const render = require('./render') @@ -81,6 +84,17 @@ async function importSnapshot(req, res) { res.status(HTTPStatus.OK).json({ projectId: historyId }) } +// Limits that force us to persist all of the changes. +function getPersistLimits() { + const farFuture = new Date() + farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) + return { + maxChanges: 0, + minChangeTimestamp: farFuture, + maxChangeTimestamp: farFuture, + } +} + async function importChanges(req, res, next) { const { params, query, body } = parseReq(req, schemas.importChanges) const projectId = params.project_id @@ -97,14 +111,7 @@ async function importChanges(req, res, next) { return render.unprocessableEntity(res) } - // Set limits to force us to persist all of the changes. - const farFuture = new Date() - farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) - const limits = { - maxChanges: 0, - minChangeTimestamp: farFuture, - maxChangeTimestamp: farFuture, - } + const limits = getPersistLimits() const blobStore = new BlobStore(projectId) const batchBlobStore = new BatchBlobStore(blobStore) @@ -176,16 +183,64 @@ async function importChanges(req, res, next) { } } +async function setContent(req, res) { + const { params, body } = parseReq(req, schemas.setContent) + const projectId = params.project_id + const { pathname, source, userId, timestamp, metadata, trackChanges } = body + const content = 'content' in body ? body.content : undefined + const blobHash = 'blobHash' in body ? body.blobHash : undefined + if (timestamp == null) { + // The schema requires a timestamp; this narrows the type for the checker. + return render.unprocessableEntity(res) + } + + let result + try { + result = await buildSetContentChange(projectId, pathname, { + content, + blobHash, + metadata, + userId, + timestamp, + origin: new Origin(source), + trackChanges, + }) + } catch (err) { + if (err instanceof storage.ContentTooLargeError) { + return render.requestEntityTooLarge(res) + } else if (err instanceof Chunk.NotFoundError) { + logger.warn({ err, projectId }, 'chunk not found') + return render.notFound(res) + } else if ( + err instanceof storage.BlobNotFoundError || + err instanceof TextOperation.UnprocessableError || + err instanceof File.NotEditableError || + err instanceof FileMap.PathnameError || + err instanceof Snapshot.EditMissingFileError || + err instanceof InvalidChangeError || + (err instanceof OError && + err.message.startsWith('StringFileData.trackedChanges out of sync')) + ) { + logger.warn({ err, projectId, pathname }, 'set content rejected') + return render.unprocessableEntity(res) + } else { + throw err + } + } + + const { change, baseVersion } = result + res.status(HTTPStatus.OK).json({ + baseVersion, + change: change ? change.toRaw() : null, + }) +} + async function flushChanges(req, res, next) { const { params } = parseReq(req, schemas.flushChanges) const projectId = params.project_id // Use the same limits importChanges, since these are passed to persistChanges - const farFuture = new Date() - farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) const limits = { - maxChanges: 0, - minChangeTimestamp: farFuture, - maxChangeTimestamp: farFuture, + ...getPersistLimits(), autoResync: true, } try { @@ -209,5 +264,6 @@ async function expireProject(req, res, next) { exports.importSnapshot = expressify(importSnapshot) exports.importChanges = expressify(importChanges) +exports.setContent = expressify(setContent) exports.flushChanges = expressify(flushChanges) exports.expireProject = expressify(expireProject) diff --git a/services/history-v1/api/routes/project_import.js b/services/history-v1/api/routes/project_import.js index ad3f730711..c5fcfbc986 100644 --- a/services/history-v1/api/routes/project_import.js +++ b/services/history-v1/api/routes/project_import.js @@ -31,6 +31,12 @@ router.post( projectImportController.importChanges ) +router.post( + '/projects/:project_id/set_content', + handleBasicAuth, + projectImportController.setContent +) + router.post( '/projects/:project_id/flush', handleBasicAuth, diff --git a/services/history-v1/api/schema.js b/services/history-v1/api/schema.js index b462373a02..b05fe8e0d3 100644 --- a/services/history-v1/api/schema.js +++ b/services/history-v1/api/schema.js @@ -65,6 +65,15 @@ const changeSchema = z }) .passthrough() +const setContentBodyBase = z.object({ + pathname: z.string().min(1), + source: z.string(), + userId: zz.objectId().optional(), + timestamp: zz.datetime(), + metadata: z.object({}).passthrough().optional(), + trackChanges: z.boolean().optional(), +}) + const schemas = { projectId: z.object({ params: z @@ -252,6 +261,31 @@ const schemas = { body: z.array(changeSchema), }), + setContent: z.object({ + params: z.object({ + project_id: z.string(), + }), + // strict() rejects unknown keys, so a mixed payload with both content and + // blobHash fails both union members and is rejected. + body: z + .union([ + setContentBodyBase.extend({ content: z.string() }).strict(), + setContentBodyBase + .extend({ blobHash: z.string().regex(hexHashPattern) }) + .strict(), + ]) + .refine( + body => { + if (body.trackChanges && !body.userId) return false + return true + }, + { + message: '"userId" is required when trackChanges is set', + path: ['userId'], + } + ), + }), + flushChanges: z.object({ params: z.object({ project_id: z.string(), diff --git a/services/history-v1/storage/index.js b/services/history-v1/storage/index.js index 6bc81f60e8..5424315892 100644 --- a/services/history-v1/storage/index.js +++ b/services/history-v1/storage/index.js @@ -24,5 +24,14 @@ exports.loadGlobalBlobs = loadGlobalBlobs const { InvalidChangeError } = require('./lib/errors') exports.InvalidChangeError = InvalidChangeError +const { + buildSetContentChange, + ContentTooLargeError, + BlobNotFoundError, +} = require('./lib/build_set_content_change') +exports.buildSetContentChange = buildSetContentChange +exports.ContentTooLargeError = ContentTooLargeError +exports.BlobNotFoundError = BlobNotFoundError + const { ChunkVersionConflictError } = require('./lib/chunk_store/errors') exports.ChunkVersionConflictError = ChunkVersionConflictError diff --git a/services/history-v1/storage/lib/build_set_content_change.js b/services/history-v1/storage/lib/build_set_content_change.js new file mode 100644 index 0000000000..0e6de77c7c --- /dev/null +++ b/services/history-v1/storage/lib/build_set_content_change.js @@ -0,0 +1,129 @@ +// @ts-check + +'use strict' + +const OError = require('@overleaf/o-error') +const core = require('overleaf-editor-core') +const { + buildSetContentOperations, +} = require('overleaf-editor-core/lib/build_set_content_operations') +const assert = require('./assert') +const chunkStore = require('./chunk_store') +const { BlobStore } = require('./blob_store') + +const { Change, TextOperation } = core + +/** + * @import { Change, Origin } from 'overleaf-editor-core' + */ + +class ContentTooLargeError extends OError {} +class BlobNotFoundError extends OError {} + +/** + * Build a change that sets the content of a pathname (a setDoc-like upsert). + * + * Computes the change against the current head (persisted chunk plus buffered + * changes) without committing it. The caller is responsible for committing + * the change with `baseVersion` as the expected end version, rebasing or + * transforming it first if the head has moved on in the meantime. + * + * New content is stored in the blob store as a side effect, so the returned + * change is cheap to serialize and can be committed as-is later. + * + * @param {string} projectId + * @param {string} pathname + * @param {object} opts + * @param {string} [opts.content] new doc content (exactly one of content and + * blobHash must be given) + * @param {string} [opts.blobHash] hash of an already created blob with the + * new binary file content + * @param {Object} [opts.metadata] metadata for the file; the file's metadata + * is always replaced with this value (defaults to empty metadata) + * @param {string} [opts.userId] + * @param {Date} opts.timestamp + * @param {Origin} opts.origin + * @param {boolean} [opts.trackChanges] record the change as tracked changes + * (ignored when blobHash is given) + * @return {Promise<{change: Change | null, baseVersion: number}>} the change + * to commit (null when the content and metadata already match) and + * the version of the head it was built against + */ +async function buildSetContentChange(projectId, pathname, opts) { + const { + content, + blobHash, + metadata, + userId, + timestamp, + origin, + trackChanges = false, + } = opts + + assert.projectId(projectId, 'bad projectId') + if (content != null && content.length > TextOperation.MAX_STRING_LENGTH) { + throw new ContentTooLargeError('content is too large', { + projectId, + pathname, + contentLength: content.length, + }) + } + if ((content == null) === (blobHash == null)) { + throw new OError( + 'buildSetContentChange: exactly one of content and blobHash must be given', + { projectId, pathname } + ) + } + if (trackChanges && !userId) { + throw new OError('buildSetContentChange: trackChanges requires a userId', { + projectId, + pathname, + }) + } + + const blobStore = new BlobStore(projectId) + let blob + if (blobHash != null) { + blob = await blobStore.getBlob(blobHash) + if (blob == null) { + throw new BlobNotFoundError('blob not found', { projectId, blobHash }) + } + } + + const tracking = trackChanges + ? { userId: /** @type {string} */ (userId), ts: timestamp } + : undefined + + const chunk = await chunkStore.loadLatest(projectId) + const snapshot = chunk.getSnapshot() + snapshot.applyAll(chunk.getChanges()) + const baseVersion = chunk.getEndVersion() + + const { operations, status } = await buildSetContentOperations({ + file: snapshot.getFile(pathname), + pathname, + content, + blob, + metadata, + tracking, + blobStore, + }) + if (status === 'noop') { + return { change: null, baseVersion } + } + + const change = new Change( + operations, + timestamp, + [], + origin, + userId ? [userId] : [] + ) + return { change, baseVersion } +} + +module.exports = { + buildSetContentChange, + ContentTooLargeError, + BlobNotFoundError, +} diff --git a/services/history-v1/test/acceptance/js/api/set_content.test.js b/services/history-v1/test/acceptance/js/api/set_content.test.js new file mode 100644 index 0000000000..b995635852 --- /dev/null +++ b/services/history-v1/test/acceptance/js/api/set_content.test.js @@ -0,0 +1,325 @@ +'use strict' + +const { expect } = require('chai') +const HTTPStatus = require('http-status') +const fetch = require('node-fetch') +const fs = require('node:fs') + +const cleanup = require('../storage/support/cleanup') +const fixtures = require('../storage/support/fixtures') +const testFiles = require('../storage/support/test_files') +const testProjects = require('./support/test_projects') +const testServer = require('./support/test_server') + +const storage = require('../../../../storage') +const chunkStore = storage.chunkStore +const BlobStore = storage.BlobStore + +const { Change, Operation, TextOperation } = require('overleaf-editor-core') + +describe('set content', function () { + beforeEach(cleanup.everything) + beforeEach(fixtures.create) + + function setContent(projectId, body, opts = {}) { + const headers = { 'Content-Type': 'application/json' } + if (!opts.anonymous) { + headers.Authorization = testServer.basicAuthHeader + } + return fetch(testServer.url(`/api/projects/${projectId}/set_content`), { + method: 'POST', + body: JSON.stringify({ + source: 'test-source', + userId: 'abcdef0123456789abcdef01', + timestamp: new Date().toISOString(), + ...body, + }), + headers, + }) + } + + function postChanges(projectId, changes, endVersion) { + return fetch( + testServer.url(`/api/projects/${projectId}/changes`) + + `?end_version=${endVersion}`, + { + method: 'POST', + body: JSON.stringify(changes), + headers: { + 'Content-Type': 'application/json', + Authorization: testServer.basicAuthHeader, + }, + } + ) + } + + // Commit a change returned by set_content, as a caller is expected to do + async function applyChange(projectId, { change, baseVersion }) { + const response = await postChanges(projectId, [change], baseVersion) + expect(response.status).to.equal(HTTPStatus.CREATED) + } + + async function getFileContent(projectId, pathname) { + const blobStore = new BlobStore(projectId) + const chunk = await chunkStore.loadLatest(projectId) + const snapshot = chunk.getSnapshot() + snapshot.applyAll(chunk.getChanges()) + const file = snapshot.getFile(pathname) + await file.load('eager', blobStore) + return file + } + + it('requires authentication', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent( + projectId, + { pathname: 'main.tex', content: 'hello' }, + { anonymous: true } + ) + expect(response.status).to.equal(HTTPStatus.UNAUTHORIZED) + }) + + it('builds create and edit changes and reports noops for a doc', async function () { + const projectId = await testProjects.createEmptyProject() + + // Create the doc + let response = await setContent(projectId, { + pathname: 'main.tex', + content: 'one\ntwo\n', + }) + expect(response.status).to.equal(HTTPStatus.OK) + let result = await response.json() + expect(result.baseVersion).to.equal(0) + expect(result.change.origin).to.deep.equal({ kind: 'test-source' }) + expect(result.change.v2Authors).to.deep.equal(['abcdef0123456789abcdef01']) + expect(result.change.operations).to.have.length(1) + expect(result.change.operations[0].pathname).to.equal('main.tex') + expect(result.change.operations[0].file.hash).to.exist + + // The change is not applied until the caller commits it + await applyChange(projectId, result) + + // Edit the doc + response = await setContent(projectId, { + pathname: 'main.tex', + content: 'one\ntwo\nthree\n', + }) + expect(response.status).to.equal(HTTPStatus.OK) + result = await response.json() + expect(result.baseVersion).to.equal(1) + expect(result.change.operations).to.have.length(1) + expect(result.change.operations[0].pathname).to.equal('main.tex') + expect(result.change.operations[0].textOperation).to.exist + await applyChange(projectId, result) + + // Identical content is a noop + response = await setContent(projectId, { + pathname: 'main.tex', + content: 'one\ntwo\nthree\n', + }) + expect(response.status).to.equal(HTTPStatus.OK) + expect(await response.json()).to.deep.equal({ + baseVersion: 2, + change: null, + }) + + const file = await getFileContent(projectId, 'main.tex') + expect(file.getContent()).to.equal('one\ntwo\nthree\n') + }) + + it('returns a change that can be rebased on a concurrent change', async function () { + const projectId = await testProjects.createEmptyProject() + + // Create the doc + let response = await setContent(projectId, { + pathname: 'main.tex', + content: 'one\ntwo\nthree\n', + }) + expect(response.status).to.equal(HTTPStatus.OK) + await applyChange(projectId, await response.json()) + + // Build a change that appends a line, without committing it yet + response = await setContent(projectId, { + pathname: 'main.tex', + content: 'one\ntwo\nthree\nfour\n', + }) + expect(response.status).to.equal(HTTPStatus.OK) + const result = await response.json() + expect(result.baseVersion).to.equal(1) + + // Meanwhile, a concurrent change inserts a line at the top + const concurrentChange = new Change( + [ + Operation.editFile( + 'main.tex', + TextOperation.fromJSON({ + textOperation: ['zero\n', 'one\ntwo\nthree\n'.length], + }) + ), + ], + new Date(), + [] + ) + let commitResponse = await postChanges( + projectId, + [concurrentChange.toRaw()], + result.baseVersion + ) + expect(commitResponse.status).to.equal(HTTPStatus.CREATED) + + // The returned change no longer applies at its base version + commitResponse = await postChanges( + projectId, + [result.change], + result.baseVersion + ) + expect(commitResponse.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + + // Rebase the returned change on the concurrent change and commit it + const rebasedChange = Change.fromRaw(result.change) + Operation.transformMultiple( + rebasedChange.getOperations(), + concurrentChange.getOperations() + ) + commitResponse = await postChanges( + projectId, + [rebasedChange.toRaw()], + result.baseVersion + 1 + ) + expect(commitResponse.status).to.equal(HTTPStatus.CREATED) + + const file = await getFileContent(projectId, 'main.tex') + expect(file.getContent()).to.equal('zero\none\ntwo\nthree\nfour\n') + }) + + it('records tracked changes when trackChanges is set', async function () { + const projectId = await testProjects.createEmptyProject() + + let response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello world', + }) + expect(response.status).to.equal(HTTPStatus.OK) + await applyChange(projectId, await response.json()) + + response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello brave world', + trackChanges: true, + }) + expect(response.status).to.equal(HTTPStatus.OK) + const result = await response.json() + expect(result.baseVersion).to.equal(1) + expect(result.change).to.exist + await applyChange(projectId, result) + + const file = await getFileContent(projectId, 'main.tex') + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello brave world' + ) + const trackedChanges = file.getTrackedChanges().asSorted() + expect(trackedChanges).to.have.length(1) + expect(trackedChanges[0].tracking.type).to.equal('insert') + expect(trackedChanges[0].tracking.userId).to.equal( + 'abcdef0123456789abcdef01' + ) + }) + + it('sets binary file content from an uploaded blob', async function () { + const projectId = await testProjects.createEmptyProject() + + // Create the blob first, as in the importChanges flow + const uploadResponse = await fetch( + testServer.url( + `/api/projects/${projectId}/blobs/${testFiles.GRAPH_PNG_HASH}` + ), + { + method: 'PUT', + body: fs.createReadStream(testFiles.path('graph.png')), + headers: { Authorization: testServer.basicAuthHeader }, + } + ) + expect(uploadResponse.ok).to.be.true + + const response = await setContent(projectId, { + pathname: 'images/graph.png', + blobHash: testFiles.GRAPH_PNG_HASH, + }) + expect(response.status).to.equal(HTTPStatus.OK) + const result = await response.json() + expect(result.baseVersion).to.equal(0) + expect(result.change.operations).to.have.length(1) + expect(result.change.operations[0].file.hash).to.equal( + testFiles.GRAPH_PNG_HASH + ) + await applyChange(projectId, result) + + const file = await getFileContent(projectId, 'images/graph.png') + expect(file.isEditable()).to.be.false + expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH) + }) + + it('rejects a payload with both content and blobHash', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello', + blobHash: testFiles.GRAPH_PNG_HASH, + }) + expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + }) + + it('rejects a payload with neither content nor blobHash', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { pathname: 'main.tex' }) + expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + }) + + it('rejects a payload without a timestamp', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello', + timestamp: undefined, + }) + expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + }) + + it('rejects trackChanges without a userId', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello', + trackChanges: true, + userId: undefined, + }) + expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + }) + + it('rejects an unknown blob hash', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { + pathname: 'images/graph.png', + blobHash: testFiles.GRAPH_PNG_HASH, + }) + expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY) + }) + + it('rejects content that is too large', async function () { + const projectId = await testProjects.createEmptyProject() + const response = await setContent(projectId, { + pathname: 'main.tex', + content: 'x'.repeat(TextOperation.MAX_STRING_LENGTH + 1), + }) + expect(response.status).to.equal(HTTPStatus.REQUEST_ENTITY_TOO_LARGE) + }) + + it('returns 404 for an uninitialized project', async function () { + const projectId = fixtures.docs.uninitializedProject.id + const response = await setContent(projectId, { + pathname: 'main.tex', + content: 'hello', + }) + expect(response.status).to.equal(HTTPStatus.NOT_FOUND) + }) +}) diff --git a/services/history-v1/test/acceptance/js/storage/build_set_content_change.test.js b/services/history-v1/test/acceptance/js/storage/build_set_content_change.test.js new file mode 100644 index 0000000000..b93346fe13 --- /dev/null +++ b/services/history-v1/test/acceptance/js/storage/build_set_content_change.test.js @@ -0,0 +1,515 @@ +'use strict' + +const { expect } = require('chai') + +const cleanup = require('./support/cleanup') +const fixtures = require('./support/fixtures') +const testFiles = require('./support/test_files.js') +const storage = require('../../../../storage') +const chunkStore = storage.chunkStore +const queueChanges = storage.queueChanges +const persistChanges = storage.persistChanges +const commitChanges = storage.commitChanges +const redisBackend = require('../../../../storage/lib/chunk_store/redis') +const { buildSetContentChange, ContentTooLargeError, BlobNotFoundError } = + storage +const BlobStore = storage.BlobStore + +const core = require('overleaf-editor-core') +const AddFileOperation = core.AddFileOperation +const Change = core.Change +const Chunk = core.Chunk +const File = core.File +const Origin = core.Origin +const TextOperation = core.TextOperation + +describe('buildSetContentChange', function () { + const userId = 'abcdef0123456789abcdef01' + const source = 'test-source' + + let limitsToPersistImmediately + before(function () { + // Used to provide a limit which forces us to persist all of the changes + const farFuture = new Date() + farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) + limitsToPersistImmediately = { + minChangeTimestamp: farFuture, + maxChangeTimestamp: farFuture, + maxChanges: 10, + maxChunkChanges: 10, + } + }) + + beforeEach(cleanup.everything) + beforeEach(fixtures.create) + + function makeOpts(opts = {}) { + return { + userId, + timestamp: new Date(), + origin: new Origin(source), + ...opts, + } + } + + async function setupProjectWithDoc(pathname, content) { + const projectId = fixtures.docs.uninitializedProject.id + await chunkStore.initializeProject(projectId) + const blobStore = new BlobStore(projectId) + const blob = await blobStore.putString(content) + await persistChanges( + projectId, + [ + new Change( + [new AddFileOperation(pathname, File.createLazyFromBlobs(blob))], + new Date(), + [] + ), + ], + limitsToPersistImmediately, + 0 + ) + return { projectId, blobStore } + } + + async function commit(projectId, { change, baseVersion }) { + await commitChanges( + projectId, + [change], + limitsToPersistImmediately, + baseVersion, + { historyBufferLevel: 4 } + ) + } + + async function getFile(projectId, pathname) { + const blobStore = new BlobStore(projectId) + const chunk = await chunkStore.loadLatest(projectId) + const snapshot = chunk.getSnapshot() + snapshot.applyAll(chunk.getChanges()) + const file = snapshot.getFile(pathname) + if (file) { + await file.load('eager', blobStore) + } + return { file, version: chunk.getEndVersion() } + } + + describe('with string content', function () { + it('edits an existing doc with a minimal diff', async function () { + const { projectId } = await setupProjectWithDoc( + 'main.tex', + 'one\ntwo\nthree\n' + ) + + const result = await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ content: 'one\ntwo and a half\nthree\n' }) + ) + expect(result.baseVersion).to.equal(1) + + // The change carries the given metadata + const rawChange = result.change.toRaw() + expect(rawChange.origin).to.deep.equal({ kind: source }) + expect(rawChange.v2Authors).to.deep.equal([userId]) + expect(rawChange.operations).to.have.length(1) + expect(rawChange.operations[0].pathname).to.equal('main.tex') + expect(rawChange.operations[0].textOperation).to.exist + + // Nothing was committed yet + const redisState = await redisBackend.getState(projectId) + expect(redisState.headVersion).to.be.null + + await commit(projectId, result) + const { file, version } = await getFile(projectId, 'main.tex') + expect(version).to.equal(2) + expect(file.getContent()).to.equal('one\ntwo and a half\nthree\n') + }) + + it('reports identical content as a noop', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'same\n') + + const result = await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ content: 'same\n' }) + ) + expect(result).to.deep.equal({ change: null, baseVersion: 1 }) + }) + + it('updates doc metadata when explicitly provided', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'same\n') + + const result = await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ content: 'same\n', metadata: { main: true } }) + ) + expect(result.baseVersion).to.equal(1) + expect(result.change.toRaw().operations).to.deep.equal([ + { + pathname: 'main.tex', + metadata: { main: true }, + }, + ]) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'main.tex') + expect(file.getContent()).to.equal('same\n') + expect(file.getMetadata()).to.deep.equal({ main: true }) + }) + + it('diffs against content pending in the redis buffer', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'one\ntwo\n') + + // Queue an edit in the redis buffer without persisting it + const editOperation = core.Operation.editFile( + 'main.tex', + TextOperation.fromJSON({ textOperation: [8, 'three\n'] }) + ) + const editChange = new Change([editOperation], new Date(), []) + await queueChanges(projectId, [editChange], 1) + + const result = await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ content: 'one\ntwo\nthree\nfour\n' }) + ) + expect(result.baseVersion).to.equal(2) + + await commit(projectId, result) + const { file, version } = await getFile(projectId, 'main.tex') + expect(version).to.equal(3) + expect(file.getContent()).to.equal('one\ntwo\nthree\nfour\n') + }) + + it('replaces a binary file with an editable doc', async function () { + const projectId = fixtures.docs.uninitializedProject.id + await chunkStore.initializeProject(projectId) + const blobStore = new BlobStore(projectId) + await blobStore.putFile(testFiles.path('graph.png')) + await persistChanges( + projectId, + [ + new Change( + [ + new AddFileOperation( + 'figure.tex', + File.fromHash(testFiles.GRAPH_PNG_HASH) + ), + ], + new Date(), + [] + ), + ], + limitsToPersistImmediately, + 0 + ) + + const result = await buildSetContentChange( + projectId, + 'figure.tex', + makeOpts({ content: 'hello world', metadata: { main: true } }) + ) + expect(result.baseVersion).to.equal(1) + const rawChange = result.change.toRaw() + expect(rawChange.operations).to.have.length(2) + expect(rawChange.operations[0]).to.deep.equal({ + pathname: 'figure.tex', + newPathname: '', + }) + expect(rawChange.operations[1].file.hash).to.exist + + await commit(projectId, result) + const { file } = await getFile(projectId, 'figure.tex') + expect(file.isEditable()).to.be.true + expect(file.getContent()).to.equal('hello world') + expect(file.getMetadata()).to.deep.equal({ main: true }) + }) + + it('creates a new doc when there is no file at the pathname', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'main\n') + + const result = await buildSetContentChange( + projectId, + 'chapters/one.tex', + makeOpts({ content: 'chapter one\n' }) + ) + expect(result.baseVersion).to.equal(1) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'chapters/one.tex') + expect(file.getContent()).to.equal('chapter one\n') + }) + + it('records the edit as tracked changes when requested', async function () { + const timestamp = new Date() + const { projectId } = await setupProjectWithDoc( + 'main.tex', + 'hello cruel world' + ) + + const result = await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ + content: 'hello brave world', + trackChanges: true, + timestamp, + }) + ) + expect(result.baseVersion).to.equal(1) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'main.tex') + // The removed content is retained as a tracked delete + expect(file.getContent({ filterTrackedDeletes: true })).to.equal( + 'hello brave world' + ) + expect(file.getContent()).to.contain('cruel') + const trackedChanges = file.getTrackedChanges().asSorted() + expect(trackedChanges).to.have.length(2) + const trackedDelete = trackedChanges.find( + tc => tc.tracking.type === 'delete' + ) + const trackedInsert = trackedChanges.find( + tc => tc.tracking.type === 'insert' + ) + expect(trackedDelete).to.exist + expect(trackedInsert).to.exist + for (const trackedChange of trackedChanges) { + expect(trackedChange.tracking.userId).to.equal(userId) + expect(trackedChange.tracking.ts.toISOString()).to.equal( + timestamp.toISOString() + ) + } + }) + + it('records new doc content as a tracked insert when requested', async function () { + const timestamp = new Date() + const { projectId } = await setupProjectWithDoc('main.tex', 'main\n') + + const result = await buildSetContentChange( + projectId, + 'new.tex', + makeOpts({ content: 'new content', trackChanges: true, timestamp }) + ) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'new.tex') + expect(file.getContent()).to.equal('new content') + expect(file.getTrackedChanges().toRaw()).to.deep.equal([ + { + range: { pos: 0, length: 'new content'.length }, + tracking: { + type: 'insert', + userId, + ts: timestamp.toISOString(), + }, + }, + ]) + }) + + it('rejects content that is too large', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'main\n') + + let error + try { + await buildSetContentChange( + projectId, + 'main.tex', + makeOpts({ content: 'x'.repeat(TextOperation.MAX_STRING_LENGTH + 1) }) + ) + } catch (err) { + error = err + } + expect(error).to.be.an.instanceof(ContentTooLargeError) + }) + + it('throws when the project does not exist', async function () { + let error + try { + await buildSetContentChange( + fixtures.docs.uninitializedProject.id, + 'main.tex', + makeOpts({ content: 'hello' }) + ) + } catch (err) { + error = err + } + expect(error).to.be.an.instanceof(Chunk.NotFoundError) + }) + }) + + describe('with a blob', function () { + it('replaces a doc with a binary file', async function () { + const { projectId, blobStore } = await setupProjectWithDoc( + 'figure.png', + 'placeholder' + ) + await blobStore.putFile(testFiles.path('graph.png')) + + const result = await buildSetContentChange( + projectId, + 'figure.png', + makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH }) + ) + expect(result.baseVersion).to.equal(1) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'figure.png') + expect(file.isEditable()).to.be.false + expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH) + }) + + it('reports the same hash and metadata as a noop', async function () { + const projectId = fixtures.docs.uninitializedProject.id + await chunkStore.initializeProject(projectId) + const blobStore = new BlobStore(projectId) + const blob = await blobStore.putFile(testFiles.path('graph.png')) + await persistChanges( + projectId, + [ + new Change( + [ + new AddFileOperation( + 'figure.png', + File.createLazyFromBlobs(blob) + ), + ], + new Date(), + [] + ), + ], + limitsToPersistImmediately, + 0 + ) + + const result = await buildSetContentChange( + projectId, + 'figure.png', + makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH }) + ) + expect(result).to.deep.equal({ change: null, baseVersion: 1 }) + }) + + it('only sets metadata when the hash matches but metadata differs', async function () { + const projectId = fixtures.docs.uninitializedProject.id + await chunkStore.initializeProject(projectId) + const blobStore = new BlobStore(projectId) + const blob = await blobStore.putFile(testFiles.path('graph.png')) + await persistChanges( + projectId, + [ + new Change( + [ + new AddFileOperation( + 'figure.png', + File.createLazyFromBlobs(blob) + ), + ], + new Date(), + [] + ), + ], + limitsToPersistImmediately, + 0 + ) + + const result = await buildSetContentChange( + projectId, + 'figure.png', + makeOpts({ + blobHash: testFiles.GRAPH_PNG_HASH, + metadata: { importer: 'github' }, + }) + ) + expect(result.baseVersion).to.equal(1) + expect(result.change.toRaw().operations).to.deep.equal([ + { + pathname: 'figure.png', + metadata: { importer: 'github' }, + }, + ]) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'figure.png') + expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH) + expect(file.getMetadata()).to.deep.equal({ importer: 'github' }) + }) + + it('adds a new file when there is no file at the pathname', async function () { + const { projectId, blobStore } = await setupProjectWithDoc( + 'main.tex', + 'main\n' + ) + await blobStore.putFile(testFiles.path('graph.png')) + + const result = await buildSetContentChange( + projectId, + 'images/graph.png', + makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH }) + ) + expect(result.baseVersion).to.equal(1) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'images/graph.png') + expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH) + }) + + it('throws when the blob does not exist', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'main\n') + + let error + try { + await buildSetContentChange( + projectId, + 'images/graph.png', + makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH }) + ) + } catch (err) { + error = err + } + expect(error).to.be.an.instanceof(BlobNotFoundError) + }) + + it('ignores the trackChanges flag', async function () { + const { projectId, blobStore } = await setupProjectWithDoc( + 'main.tex', + 'main\n' + ) + await blobStore.putFile(testFiles.path('graph.png')) + + const result = await buildSetContentChange( + projectId, + 'images/graph.png', + makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH, trackChanges: true }) + ) + expect(result.baseVersion).to.equal(1) + + await commit(projectId, result) + const { file } = await getFile(projectId, 'images/graph.png') + expect(file.getRangesHash()).to.not.exist + }) + }) + + it('requires exactly one of content and blobHash', async function () { + const { projectId } = await setupProjectWithDoc('main.tex', 'main\n') + + for (const opts of [ + {}, + { content: 'a', blobHash: testFiles.GRAPH_PNG_HASH }, + ]) { + let error + try { + await buildSetContentChange(projectId, 'main.tex', makeOpts(opts)) + } catch (err) { + error = err + } + expect(error) + .to.be.an('error') + .with.property('message') + .that.contains('exactly one of content and blobHash') + } + }) +}) diff --git a/yarn.lock b/yarn.lock index cc8098afc7..9373175367 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25756,6 +25756,7 @@ __metadata: "@types/path-browserify": "npm:^1.0.3" chai: "npm:^3.3.0" check-types: "npm:^5.1.0" + diff-match-patch: "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688" eslint: "npm:^10.4.0" lodash: "npm:^4.18.1" mocha: "npm:^11.1.0"