diff --git a/libraries/overleaf-editor-core/lib/operation/text_operation.js b/libraries/overleaf-editor-core/lib/operation/text_operation.js index 4f556012c6..148570fa42 100644 --- a/libraries/overleaf-editor-core/lib/operation/text_operation.js +++ b/libraries/overleaf-editor-core/lib/operation/text_operation.js @@ -56,18 +56,34 @@ class TextOperation extends EditOperation { constructor() { super() - // When an operation is applied to an input string, you can think of this as - // if an imaginary cursor runs over the entire string and skips over some - // parts, removes some parts and inserts characters at some positions. These - // actions (skip/remove/insert) are stored as an array in the "ops" property. - /** @type {ScanOp[]} */ + + /** + * When an operation is applied to an input string, you can think of this as + * if an imaginary cursor runs over the entire string and skips over some + * parts, removes some parts and inserts characters at some positions. These + * actions (skip/remove/insert) are stored as an array in the "ops" property. + * @type {ScanOp[]} + */ this.ops = [] - // An operation's baseLength is the length of every string the operation - // can be applied to. + + /** + * An operation's baseLength is the length of every string the operation + * can be applied to. + */ this.baseLength = 0 - // The targetLength is the length of every string that results from applying - // the operation on a valid input string. + + /** + * The targetLength is the length of every string that results from applying + * the operation on a valid input string. + */ this.targetLength = 0 + + /** + * The expected content hash after this operation is applied + * + * @type {string | null} + */ + this.contentHash = null } /** @@ -223,7 +239,12 @@ class TextOperation extends EditOperation { * @returns {RawTextOperation} */ toJSON() { - return { textOperation: this.ops.map(op => op.toJSON()) } + /** @type {RawTextOperation} */ + const json = { textOperation: this.ops.map(op => op.toJSON()) } + if (this.contentHash != null) { + json.contentHash = this.contentHash + } + return json } /** @@ -231,7 +252,7 @@ class TextOperation extends EditOperation { * @param {RawTextOperation} obj * @returns {TextOperation} */ - static fromJSON = function ({ textOperation: ops }) { + static fromJSON = function ({ textOperation: ops, contentHash }) { const o = new TextOperation() for (const op of ops) { if (isRetain(op)) { @@ -250,6 +271,9 @@ class TextOperation extends EditOperation { throw new UnprocessableError('unknown operation: ' + JSON.stringify(op)) } } + if (contentHash != null) { + o.contentHash = contentHash + } return o } diff --git a/libraries/overleaf-editor-core/lib/types.ts b/libraries/overleaf-editor-core/lib/types.ts index 604cc93414..53bf0622c4 100644 --- a/libraries/overleaf-editor-core/lib/types.ts +++ b/libraries/overleaf-editor-core/lib/types.ts @@ -132,6 +132,7 @@ export type RawScanOp = RawInsertOp | RawRemoveOp | RawRetainOp export type RawTextOperation = { textOperation: RawScanOp[] + contentHash?: string } export type RawAddCommentOperation = { diff --git a/services/project-history/app/js/UpdateCompressor.js b/services/project-history/app/js/UpdateCompressor.js index c6ab91f959..9010348598 100644 --- a/services/project-history/app/js/UpdateCompressor.js +++ b/services/project-history/app/js/UpdateCompressor.js @@ -29,11 +29,16 @@ const cloneWithOp = function (update, op) { return update } const mergeUpdatesWithOp = function (firstUpdate, secondUpdate, op) { - // We want to take doc_length and ts from the firstUpdate, v from the second + // We want to take doc_length and ts from the firstUpdate, v and doc_hash from the second const update = cloneWithOp(firstUpdate, op) if (secondUpdate.v != null) { update.v = secondUpdate.v } + if (secondUpdate.meta.doc_hash != null) { + update.meta.doc_hash = secondUpdate.meta.doc_hash + } else { + delete update.meta.doc_hash + } return update } @@ -112,8 +117,11 @@ export function convertToSingleOpUpdates(updates) { if (docLength === -1) { docLength = 0 } + const docHash = update.meta.doc_hash for (const op of ops) { const splitUpdate = cloneWithOp(update, op) + // Only the last update will keep the doc_hash property + delete splitUpdate.meta.doc_hash if (docLength != null) { splitUpdate.meta.doc_length = docLength docLength = adjustLengthByOp(docLength, op, { @@ -123,6 +131,9 @@ export function convertToSingleOpUpdates(updates) { } splitUpdates.push(splitUpdate) } + if (docHash != null && splitUpdates.length > 0) { + splitUpdates[splitUpdates.length - 1].meta.doc_hash = docHash + } } return splitUpdates } @@ -153,6 +164,11 @@ export function concatUpdatesWithSameVersion(updates) { lastUpdate.pathname === update.pathname ) { lastUpdate.op = lastUpdate.op.concat(update.op) + if (update.meta.doc_hash == null) { + delete lastUpdate.meta.doc_hash + } else { + lastUpdate.meta.doc_hash = update.meta.doc_hash + } } else { concattedUpdates.push(update) } diff --git a/services/project-history/app/js/UpdateTranslator.js b/services/project-history/app/js/UpdateTranslator.js index 82a1de05ff..38e65f6968 100644 --- a/services/project-history/app/js/UpdateTranslator.js +++ b/services/project-history/app/js/UpdateTranslator.js @@ -70,6 +70,12 @@ function _convertToChange(projectId, updateWithBlob) { for (const op of update.op) { builder.addOp(op, update) } + // add doc hash if present + if (update.meta.doc_hash != null) { + // This will commit the text operation that the builder is currently + // building and set the contentHash property. + builder.commitTextOperation({ contentHash: update.meta.doc_hash }) + } operations = builder.finish() // add doc version information if present if (update.v != null) { @@ -285,8 +291,8 @@ class OperationsBuilder { const pos = Math.min(op.hpos ?? op.p, this.docLength) if (isComment(op)) { - // Close the current text operation - this.pushTextOperation() + // Commit the current text operation + this.commitTextOperation() // Add a comment operation const commentLength = op.hlen ?? op.c.length @@ -307,7 +313,7 @@ class OperationsBuilder { } if (pos < this.cursor) { - this.pushTextOperation() + this.commitTextOperation() // At this point, this.cursor === 0 and we can continue } @@ -450,23 +456,32 @@ class OperationsBuilder { this.docLength -= length } - pushTextOperation() { - if (this.textOperation.length > 0) - if (this.cursor < this.docLength) { - this.retain(this.docLength - this.cursor) - } + /** + * Finalize the current text operation and push it to the queue + * + * @param {object} [opts] + * @param {string} [opts.contentHash] + */ + commitTextOperation(opts = {}) { + if (this.textOperation.length > 0 && this.cursor < this.docLength) { + this.retain(this.docLength - this.cursor) + } if (this.textOperation.length > 0) { - this.operations.push({ + const operation = { pathname: this.pathname, textOperation: this.textOperation, - }) + } + if (opts.contentHash != null) { + operation.contentHash = opts.contentHash + } + this.operations.push(operation) this.textOperation = [] } this.cursor = 0 } finish() { - this.pushTextOperation() + this.commitTextOperation() return this.operations } } diff --git a/services/project-history/app/js/types.ts b/services/project-history/app/js/types.ts index 206ccfdcd5..c2b0d83728 100644 --- a/services/project-history/app/js/types.ts +++ b/services/project-history/app/js/types.ts @@ -35,6 +35,7 @@ export type TextUpdate = { meta: UpdateMeta & { pathname: string doc_length: number + doc_hash?: string history_doc_length?: number } } diff --git a/services/project-history/test/unit/js/UpdateCompressor/UpdateCompressorTests.js b/services/project-history/test/unit/js/UpdateCompressor/UpdateCompressorTests.js index 8124e30e0b..04d5163476 100644 --- a/services/project-history/test/unit/js/UpdateCompressor/UpdateCompressorTests.js +++ b/services/project-history/test/unit/js/UpdateCompressor/UpdateCompressorTests.js @@ -12,6 +12,7 @@ describe('UpdateCompressor', function () { this.user_id = 'user-id-1' this.other_user_id = 'user-id-2' this.doc_id = 'mock-doc-id' + this.doc_hash = 'doc-hash' this.ts1 = Date.now() this.ts2 = Date.now() + 1000 }) @@ -247,6 +248,50 @@ describe('UpdateCompressor', function () { }, ]) }) + + it('should set the doc hash on the last split update only', function () { + const meta = { + ts: this.ts1, + user_id: this.user_id, + } + expect( + this.UpdateCompressor.convertToSingleOpUpdates([ + { + op: [ + { p: 0, i: 'foo' }, + { p: 6, i: 'bar' }, + ], + meta: { ...meta, doc_hash: 'hash1' }, + v: 42, + }, + { + op: [{ p: 10, i: 'baz' }], + meta: { ...meta, doc_hash: 'hash2' }, + v: 43, + }, + { + op: [ + { p: 0, d: 'foo' }, + { p: 20, i: 'quux' }, + { p: 3, d: 'bar' }, + ], + meta: { ...meta, doc_hash: 'hash3' }, + v: 44, + }, + ]) + ).to.deep.equal([ + { op: { p: 0, i: 'foo' }, meta, v: 42 }, + { op: { p: 6, i: 'bar' }, meta: { ...meta, doc_hash: 'hash1' }, v: 42 }, + { + op: { p: 10, i: 'baz' }, + meta: { ...meta, doc_hash: 'hash2' }, + v: 43, + }, + { op: { p: 0, d: 'foo' }, meta, v: 44 }, + { op: { p: 20, i: 'quux' }, meta, v: 44 }, + { op: { p: 3, d: 'bar' }, meta: { ...meta, doc_hash: 'hash3' }, v: 44 }, + ]) + }) }) describe('concatUpdatesWithSameVersion', function () { @@ -376,6 +421,48 @@ describe('UpdateCompressor', function () { }, ]) }) + + it("should keep the doc hash only when it's on the last update", function () { + const meta = { ts: this.ts1, user_id: this.user_id } + const baseUpdate = { doc: this.doc_id, pathname: 'main.tex', meta } + const updates = [ + { ...baseUpdate, op: { p: 0, i: 'foo' }, v: 1 }, + { + ...baseUpdate, + op: { p: 10, i: 'bar' }, + meta: { ...meta, doc_hash: 'hash1' }, + v: 1, + }, + { + ...baseUpdate, + op: { p: 20, i: 'baz' }, + meta: { ...meta, doc_hash: 'hash2' }, + v: 2, + }, + { ...baseUpdate, op: { p: 30, i: 'quux' }, v: 2 }, + ] + expect( + this.UpdateCompressor.concatUpdatesWithSameVersion(updates) + ).to.deep.equal([ + { + ...baseUpdate, + op: [ + { p: 0, i: 'foo' }, + { p: 10, i: 'bar' }, + ], + meta: { ...meta, doc_hash: 'hash1' }, + v: 1, + }, + { + ...baseUpdate, + op: [ + { p: 20, i: 'baz' }, + { p: 30, i: 'quux' }, + ], + v: 2, + }, + ]) + }) }) describe('compress', function () { @@ -1437,5 +1524,47 @@ describe('UpdateCompressor', function () { ]) }) }) + + describe('doc hash', function () { + it("should keep the doc hash if it's on the last update", function () { + const meta = { ts: this.ts1, user_id: this.user_id } + expect( + this.UpdateCompressor.compressUpdates([ + { op: { p: 3, i: 'foo' }, meta, v: 42 }, + { + op: { p: 6, i: 'bar' }, + meta: { ...meta, doc_hash: 'hash1' }, + v: 43, + }, + ]) + ).to.deep.equal([ + { + op: { p: 3, i: 'foobar' }, + meta: { ...meta, doc_hash: 'hash1' }, + v: 43, + }, + ]) + }) + + it("should not keep the doc hash if it's not on the last update", function () { + const meta = { ts: this.ts1, user_id: this.user_id } + expect( + this.UpdateCompressor.compressUpdates([ + { + op: { p: 3, i: 'foo' }, + meta: { ...meta, doc_hash: 'hash1' }, + v: 42, + }, + { op: { p: 6, i: 'bar' }, meta, v: 43 }, + ]) + ).to.deep.equal([ + { + op: { p: 3, i: 'foobar' }, + meta, + v: 43, + }, + ]) + }) + }) }) })