[history-v1] add setContent endpoint (#35799)

* [history-v1] add setContent endpoint

* [history-v1] refactor setContent to return a change+v

* [history-v1] make typescript happy

GitOrigin-RevId: e2cb0281367c01aa9cd842f250f2016d591ec0fc

libraries/overleaf-editor-core/lib/build_set_content_operations.js

@@ -0,0 +1,138 @@
+// @ts-check
+'use strict'
+
+const _ = require('lodash')
+const OError = require('@overleaf/o-error')
+const File = require('./file')
+const Operation = require('./operation')
+const TrackingProps = require('./file_data/tracking_props')
+const { diffAsTextOperation } = require('./diff_as_text_operation')
+
+/**
+ * @import Blob from './blob'
+ * @import StringFileData from './file_data/string_file_data'
+ * @import { BlobStore, RangesBlob } from './types'
+ */
+
+/**
+ * Build the operations that set the content of a pathname (a setDoc-like
+ * upsert).
+ *
+ * Exactly one of `content` (editable doc) and `blob` (binary file) must be
+ * provided. An existing editable doc is edited in place with a minimal diff
+ * (see diffAsTextOperation); any other case replaces or creates the file at
+ * the pathname. New content is stored in the blob store and referenced by
+ * hash, so the returned operations are cheap to serialize.
+ *
+ * @param {object} args
+ * @param {File | null | undefined} args.file existing file at the pathname
+ * @param {string} args.pathname
+ * @param {string} [args.content] new doc content
+ * @param {Blob} [args.blob] blob with the new file content
+ * @param {Object} [args.metadata] metadata for the file; the file's metadata
+ *        is always replaced with this value (defaults to empty metadata)
+ * @param {{userId: string, ts: Date}} [args.tracking] record the change as
+ *        tracked changes (ignored when replacing with a blob)
+ * @param {BlobStore} args.blobStore
+ * @return {Promise<{operations: Operation[], status: 'applied' | 'noop'}>}
+ */
+async function buildSetContentOperations({
+  file,
+  pathname,
+  content,
+  blob,
+  metadata = {},
+  tracking,
+  blobStore,
+}) {
+  if ((content == null) === (blob == null)) {
+    throw new OError(
+      'buildSetContentOperations: exactly one of content and blob must be given',
+      { pathname }
+    )
+  }
+
+  if (content != null) {
+    if (file) {
+      // Eager loading resolves both the editability of files whose data is a
+      // bare hash and the content to diff against.
+      await file.load('eager', blobStore)
+      if (file.isEditable()) {
+        return buildEditOperations(file, pathname, content, metadata, tracking)
+      }
+    }
+    blob = await blobStore.putString(content)
+  }
+  if (blob == null) {
+    throw new OError('buildSetContentOperations: no blob', { pathname })
+  }
+
+  if (file && file.getHash() === blob.getHash()) {
+    // LazyStringFileData yields no hash with buffered changes, no false-positive hash matches possible.
+    if (_.isEqual(file.getMetadata(), metadata)) {
+      return { operations: [], status: 'noop' }
+    }
+    return {
+      operations: [Operation.setFileMetadata(pathname, metadata)],
+      status: 'applied',
+    }
+  }
+
+  let rangesBlob
+  if (content != null && content.length > 0 && tracking) {
+    // Record the whole new doc as a tracked insert.
+    /** @type {RangesBlob} */
+    const ranges = {
+      comments: [],
+      trackedChanges: [
+        {
+          range: { pos: 0, length: content.length },
+          tracking: new TrackingProps(
+            'insert',
+            tracking.userId,
+            tracking.ts
+          ).toRaw(),
+        },
+      ],
+    }
+    rangesBlob = await blobStore.putObject(ranges)
+  }
+
+  const newFile = File.createLazyFromBlobs(blob, rangesBlob, metadata)
+  const operations = []
+  if (file) {
+    operations.push(Operation.removeFile(pathname))
+  }
+  operations.push(Operation.addFile(pathname, newFile))
+  return { operations, status: 'applied' }
+}
+
+/**
+ * @param {File} file eagerly loaded editable file
+ * @param {string} pathname
+ * @param {string} content
+ * @param {Object} metadata metadata to set on the doc
+ * @param {{userId: string, ts: Date}} [tracking]
+ * @return {{operations: Operation[], status: 'applied' | 'noop'}}
+ */
+function buildEditOperations(file, pathname, content, metadata, tracking) {
+  const fileData = /** @type {StringFileData} */ (file.data)
+  const textOperation = diffAsTextOperation(
+    fileData,
+    content,
+    tracking ? { tracking } : {}
+  )
+  const operations = []
+  if (!textOperation.isNoop()) {
+    operations.push(Operation.editFile(pathname, textOperation))
+  }
+  if (!_.isEqual(file.getMetadata(), metadata)) {
+    operations.push(Operation.setFileMetadata(pathname, metadata))
+  }
+  if (operations.length === 0) {
+    return { operations, status: 'noop' }
+  }
+  return { operations, status: 'applied' }
+}
+
+module.exports = { buildSetContentOperations }

libraries/overleaf-editor-core/lib/diff_as_text_operation.js

@@ -0,0 +1,177 @@
+// @ts-check
+'use strict'
+
+const DMP = require('diff-match-patch')
+const OError = require('@overleaf/o-error')
+const TextOperation = require('./operation/text_operation')
+const TrackingProps = require('./file_data/tracking_props')
+
+/**
+ * @import StringFileData from './file_data/string_file_data'
+ */
+
+const ADDED = 1
+const REMOVED = -1
+const UNCHANGED = 0
+
+const dmp = new DMP()
+
+// Do not attempt to produce a diff for more than 100ms
+dmp.Diff_Timeout = 0.1
+
+/**
+ * Diff the file's content against `after` and return a minimal TextOperation
+ * turning the file into it. The diff is computed against the content with
+ * tracked deletes filtered out (the content external sources see); the
+ * returned operation preserves the file's tracked deletes.
+ *
+ * @param {StringFileData} file eagerly loaded file data
+ * @param {string} after
+ * @param {object} [opts]
+ * @param {{userId: string, ts: Date}} [opts.tracking] record the edit as
+ *        tracked changes
+ * @return {TextOperation}
+ */
+function diffAsTextOperation(file, after, opts = {}) {
+  const beforeWithoutTrackedDeletes = file.getContent({
+    filterTrackedDeletes: true,
+  })
+  const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after)
+  dmp.diff_cleanupSemantic(diffs)
+  return diffsToTextOperation(file, diffs, opts)
+}
+
+/**
+ * Convert diff-match-patch style diffs into a TextOperation.
+ *
+ * The diffs must have been computed against the file content with tracked
+ * deletes filtered out, i.e. `file.getContent({ filterTrackedDeletes: true })`,
+ * which is the content that external sources (uploads, github-sync, tpds) see.
+ * The returned operation applies to the full content: positions are mapped to
+ * step over tracked-delete spans with plain retains, preserving them.
+ *
+ * When `opts.tracking` is set, the edit is recorded as tracked changes:
+ * insertions become tracked inserts and removals become tracked deletes,
+ * except inside existing tracked inserts, where removals stay regular
+ * deletes, and over existing tracked deletes, which are left untouched
+ * (keeping the original author). This mirrors the semantics of updates with
+ * track changes enabled in project-history's UpdateTranslator.
+ *
+ * Exported for tests: use diffAsTextOperation instead.
+ *
+ * @param {StringFileData} file eagerly loaded file data
+ * @param {[number, string][]} diffs diff-match-patch diffs
+ * @param {object} [opts]
+ * @param {{userId: string, ts: Date}} [opts.tracking] record the edit as
+ *        tracked changes
+ * @return {TextOperation}
+ */
+function diffsToTextOperation(file, diffs, opts = {}) {
+  const insertTracking = opts.tracking
+    ? new TrackingProps('insert', opts.tracking.userId, opts.tracking.ts)
+    : undefined
+  const deleteTracking = opts.tracking
+    ? new TrackingProps('delete', opts.tracking.userId, opts.tracking.ts)
+    : undefined
+
+  const trackedChanges = file.trackedChanges.asSorted()
+  let tcIndex = 0
+
+  const op = new TextOperation()
+
+  /**
+   * Consume removed content: a plain remove, or a tracked delete when
+   * recording tracked changes.
+   *
+   * @param {number} length
+   */
+  function removeContent(length) {
+    if (deleteTracking) {
+      op.retain(length, { tracking: deleteTracking })
+    } else {
+      op.remove(length)
+    }
+  }
+
+  for (const diff of diffs) {
+    const type = diff[0]
+    let content = diff[1]
+    if (type === ADDED) {
+      op.insert(content, insertTracking ? { tracking: insertTracking } : {})
+      continue
+    }
+    if (type !== REMOVED && type !== UNCHANGED) {
+      throw new Error('Unknown type')
+    }
+
+    while (tcIndex < trackedChanges.length) {
+      const tc = trackedChanges[tcIndex]
+      const segmentEnd = op.baseLength + content.length
+      if (tc.range.start >= segmentEnd) break
+      if (tc.tracking.type === 'delete') {
+        // Tracked deletes are invisible in the diffed content. Step over
+        // them with a plain retain, preserving the original tracked delete.
+        const before = tc.range.start - op.baseLength
+        if (type === REMOVED) {
+          removeContent(before)
+        } else {
+          op.retain(before)
+        }
+        op.retain(tc.range.length)
+        content = content.slice(before)
+        tcIndex++
+      } else if (type === REMOVED && deleteTracking) {
+        // Removals inside existing tracked inserts are always regular
+        // deletes, even when recording tracked changes.
+        const before = Math.max(0, tc.range.start - op.baseLength)
+        removeContent(before)
+        const overlap = Math.min(tc.range.end, segmentEnd) - op.baseLength
+        op.remove(overlap)
+        content = content.slice(before + overlap)
+        if (tc.range.end <= op.baseLength) {
+          tcIndex++
+        } else {
+          // The tracked insert extends beyond this diff segment.
+          break
+        }
+      } else {
+        // Tracked inserts are ordinary visible content otherwise. Only move
+        // past them once the whole range has been covered by diff segments.
+        if (tc.range.end <= segmentEnd) {
+          tcIndex++
+        } else {
+          break
+        }
+      }
+    }
+
+    if (type === REMOVED) {
+      removeContent(content.length)
+    } else {
+      op.retain(content.length)
+    }
+  }
+
+  // Any tracked deletes after the end of the diffed content must be retained.
+  while (tcIndex < trackedChanges.length) {
+    const tc = trackedChanges[tcIndex]
+    if (tc.tracking.type !== 'delete' || tc.range.start !== op.baseLength) {
+      throw new OError(
+        'StringFileData.trackedChanges out of sync: unexpected range after end of diff',
+        { nextTc: tc, baseLength: op.baseLength }
+      )
+    }
+    op.retain(tc.range.length)
+    tcIndex++
+  }
+
+  return op
+}
+
+module.exports = {
+  diffAsTextOperation,
+  diffsToTextOperation,
+  ADDED,
+  REMOVED,
+  UNCHANGED,
+}

libraries/overleaf-editor-core/package.json

@@ -36,6 +36,7 @@
     "@overleaf/o-error": "workspace:*",
     "@overleaf/validation-tools": "workspace:*",
     "check-types": "^5.1.0",
+    "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
     "lodash": "^4.18.1",
     "p-map": "^4.0.0",
     "path-browserify": "^1.0.1"

libraries/overleaf-editor-core/test/unit/diff_as_text_operation.test.js

@@ -0,0 +1,362 @@
+// @ts-check
+'use strict'
+
+const { expect } = require('chai')
+
+const StringFileData = require('../../lib/file_data/string_file_data')
+const {
+  diffAsTextOperation,
+  diffsToTextOperation,
+  ADDED,
+  REMOVED,
+  UNCHANGED,
+} = require('../../lib/diff_as_text_operation')
+
+/** @import { TrackedChangeRawData } from '../../lib/types' */
+
+const TS = new Date('2026-07-10T00:00:00.000Z')
+const OTHER_TS = new Date('2026-07-09T00:00:00.000Z')
+const TRACKING = { userId: 'user-1', ts: TS }
+
+/**
+ * @param {'insert' | 'delete'} type
+ * @param {number} pos
+ * @param {number} length
+ * @param {string} [userId]
+ * @return {TrackedChangeRawData}
+ */
+function trackedChange(type, pos, length, userId = 'user-2') {
+  return {
+    range: { pos, length },
+    tracking: { type, userId, ts: OTHER_TS.toISOString() },
+  }
+}
+
+describe('diffAsTextOperation', function () {
+  /**
+   * @param {StringFileData} file
+   * @param {string} after
+   * @param {object} [opts]
+   */
+  function applyDiff(file, after, opts) {
+    const operation = diffAsTextOperation(file, after, opts)
+    file.edit(operation)
+    return operation
+  }
+
+  it('returns a noop operation for unchanged content', function () {
+    const file = new StringFileData('hello world')
+    const operation = diffAsTextOperation(file, 'hello world')
+    expect(operation.isNoop()).to.be.true
+  })
+
+  it('builds a minimal edit operation', function () {
+    const file = new StringFileData('hello world')
+    const operation = applyDiff(file, 'hello brave world')
+    expect(file.getContent()).to.equal('hello brave world')
+    expect(operation.toJSON()).to.eql({ textOperation: [6, 'brave ', 5] })
+  })
+
+  it('diffs against the content without tracked deletes and preserves them', function () {
+    const file = StringFileData.fromRaw({
+      content: 'hello cruel world',
+      trackedChanges: [trackedChange('delete', 6, 6)],
+    })
+    applyDiff(file, 'hello brave world')
+    expect(file.getContent()).to.equal('hello brave cruel world')
+    expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+      'hello brave world'
+    )
+    expect(file.trackedChanges.toRaw()).to.eql([trackedChange('delete', 12, 6)])
+  })
+
+  it('records an insertion as a tracked insert when tracking is given', function () {
+    const file = new StringFileData('hello world')
+    applyDiff(file, 'hello brave world', { tracking: TRACKING })
+    expect(file.getContent()).to.equal('hello brave world')
+    expect(file.trackedChanges.toRaw()).to.eql([
+      {
+        range: { pos: 6, length: 6 },
+        tracking: { type: 'insert', userId: 'user-1', ts: TS.toISOString() },
+      },
+    ])
+  })
+
+  it('records a removal as a tracked delete when tracking is given', function () {
+    const file = new StringFileData('hello cruel world')
+    applyDiff(file, 'hello world', { tracking: TRACKING })
+    expect(file.getContent()).to.equal('hello cruel world')
+    expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+      'hello world'
+    )
+    expect(file.trackedChanges.toRaw()).to.eql([
+      {
+        range: { pos: 6, length: 6 },
+        tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() },
+      },
+    ])
+  })
+})
+
+describe('diffsToTextOperation', function () {
+  /**
+   * @param {StringFileData} file
+   * @param {[number, string][]} diffs
+   * @param {object} [opts]
+   */
+  function applyDiffs(file, diffs, opts) {
+    const operation = diffsToTextOperation(file, diffs, opts)
+    file.edit(operation)
+    return operation
+  }
+
+  describe('without tracked changes in the file', function () {
+    it('converts an insertion', function () {
+      const file = new StringFileData('hello world')
+      applyDiffs(file, [
+        [UNCHANGED, 'hello '],
+        [ADDED, 'brave '],
+        [UNCHANGED, 'world'],
+      ])
+      expect(file.getContent()).to.equal('hello brave world')
+      expect(file.trackedChanges.toRaw()).to.eql([])
+    })
+
+    it('converts a removal', function () {
+      const file = new StringFileData('hello cruel world')
+      applyDiffs(file, [
+        [UNCHANGED, 'hello '],
+        [REMOVED, 'cruel '],
+        [UNCHANGED, 'world'],
+      ])
+      expect(file.getContent()).to.equal('hello world')
+    })
+
+    it('returns a noop operation for identical content', function () {
+      const file = new StringFileData('hello world')
+      const operation = diffsToTextOperation(file, [[UNCHANGED, 'hello world']])
+      expect(operation.isNoop()).to.be.true
+    })
+
+    it('throws on unknown diff types', function () {
+      const file = new StringFileData('hello')
+      expect(() => diffsToTextOperation(file, [[2, 'hello']])).to.throw(
+        'Unknown type'
+      )
+    })
+  })
+
+  describe('with tracked deletes in the file', function () {
+    it('retains a tracked delete inside an unchanged region', function () {
+      const file = new StringFileData('hello XXXworld', undefined, [
+        trackedChange('delete', 6, 3),
+      ])
+      applyDiffs(file, [
+        [UNCHANGED, 'hello '],
+        [ADDED, 'brave '],
+        [UNCHANGED, 'world'],
+      ])
+      expect(file.getContent()).to.equal('hello brave XXXworld')
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+        'hello brave world'
+      )
+      expect(file.trackedChanges.toRaw()).to.eql([
+        trackedChange('delete', 12, 3),
+      ])
+    })
+
+    it('retains a tracked delete inside a removed region', function () {
+      const file = new StringFileData('abCCCcdef', undefined, [
+        trackedChange('delete', 2, 3),
+      ])
+      applyDiffs(file, [
+        [UNCHANGED, 'a'],
+        [REMOVED, 'bc'],
+        [UNCHANGED, 'def'],
+      ])
+      expect(file.getContent()).to.equal('aCCCdef')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        trackedChange('delete', 1, 3),
+      ])
+    })
+
+    it('retains tracked deletes after the end of the diff', function () {
+      const file = new StringFileData('abcDDD', undefined, [
+        trackedChange('delete', 3, 3),
+      ])
+      const operation = applyDiffs(file, [[UNCHANGED, 'abc']])
+      expect(operation.baseLength).to.equal(6)
+      expect(file.getContent()).to.equal('abcDDD')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        trackedChange('delete', 3, 3),
+      ])
+    })
+
+    it('throws when the tracked changes are out of sync', function () {
+      const file = new StringFileData('abcDE', undefined, [
+        trackedChange('delete', 4, 1),
+      ])
+      expect(() => diffsToTextOperation(file, [[UNCHANGED, 'abc']])).to.throw(
+        'StringFileData.trackedChanges out of sync'
+      )
+    })
+  })
+
+  describe('recording the edit as tracked changes', function () {
+    it('records insertions as tracked inserts', function () {
+      const file = new StringFileData('hello world')
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'hello '],
+          [ADDED, 'brave '],
+          [UNCHANGED, 'world'],
+        ],
+        { tracking: TRACKING }
+      )
+      expect(file.getContent()).to.equal('hello brave world')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        {
+          range: { pos: 6, length: 6 },
+          tracking: { type: 'insert', userId: 'user-1', ts: TS.toISOString() },
+        },
+      ])
+    })
+
+    it('records removals as tracked deletes', function () {
+      const file = new StringFileData('hello cruel world')
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'hello '],
+          [REMOVED, 'cruel '],
+          [UNCHANGED, 'world'],
+        ],
+        { tracking: TRACKING }
+      )
+      expect(file.getContent()).to.equal('hello cruel world')
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+        'hello world'
+      )
+      expect(file.trackedChanges.toRaw()).to.eql([
+        {
+          range: { pos: 6, length: 6 },
+          tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() },
+        },
+      ])
+    })
+
+    it('records a whole-document removal as a tracked delete', function () {
+      const file = new StringFileData('abc')
+      const operation = diffsToTextOperation(file, [[REMOVED, 'abc']], {
+        tracking: TRACKING,
+      })
+      file.edit(operation)
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal('')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        {
+          range: { pos: 0, length: 3 },
+          tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() },
+        },
+      ])
+    })
+
+    it('leaves existing tracked deletes untouched', function () {
+      const file = new StringFileData('hello CCCworld', undefined, [
+        trackedChange('delete', 6, 3),
+      ])
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'hello '],
+          [REMOVED, 'wor'],
+          [UNCHANGED, 'ld'],
+        ],
+        { tracking: TRACKING }
+      )
+      expect(file.getContent()).to.equal('hello CCCworld')
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+        'hello ld'
+      )
+      expect(file.trackedChanges.toRaw()).to.eql([
+        trackedChange('delete', 6, 3),
+        {
+          range: { pos: 9, length: 3 },
+          tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() },
+        },
+      ])
+    })
+
+    it('removes content inside existing tracked inserts as a regular delete', function () {
+      const file = new StringFileData('hello INSworld', undefined, [
+        trackedChange('insert', 6, 3),
+      ])
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'hello '],
+          [REMOVED, 'INS'],
+          [UNCHANGED, 'world'],
+        ],
+        { tracking: TRACKING }
+      )
+      expect(file.getContent()).to.equal('hello world')
+      expect(file.trackedChanges.toRaw()).to.eql([])
+    })
+
+    it('splits a removal spanning an existing tracked insert', function () {
+      const file = new StringFileData('abXYcd', undefined, [
+        trackedChange('insert', 2, 2),
+      ])
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'a'],
+          [REMOVED, 'bXYc'],
+          [UNCHANGED, 'd'],
+        ],
+        { tracking: TRACKING }
+      )
+      // The tracked insert is removed for real; b and c become tracked deletes
+      expect(file.getContent()).to.equal('abcd')
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal('ad')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        {
+          range: { pos: 1, length: 2 },
+          tracking: { type: 'delete', userId: 'user-1', ts: TS.toISOString() },
+        },
+      ])
+    })
+
+    it('handles a tracked insert straddling removed and unchanged regions', function () {
+      const file = new StringFileData('hello INSins', undefined, [
+        trackedChange('insert', 6, 6),
+      ])
+      applyDiffs(
+        file,
+        [
+          [UNCHANGED, 'hello '],
+          [REMOVED, 'INS'],
+          [UNCHANGED, 'ins'],
+        ],
+        { tracking: TRACKING }
+      )
+      expect(file.getContent()).to.equal('hello ins')
+      expect(file.trackedChanges.toRaw()).to.eql([
+        trackedChange('insert', 6, 3),
+      ])
+    })
+
+    it('keeps identical content a noop', function () {
+      const file = new StringFileData('hello world')
+      const operation = diffsToTextOperation(
+        file,
+        [[UNCHANGED, 'hello world']],
+        { tracking: TRACKING }
+      )
+      expect(operation.isNoop()).to.be.true
+      file.edit(operation)
+      expect(file.trackedChanges.toRaw()).to.eql([])
+    })
+  })
+})

services/document-updater/app/js/DiffCodec.js

@@ -1,6 +1,4 @@
-const OError = require('@overleaf/o-error')
 const DMP = require('diff-match-patch')
-const { TextOperation } = require('overleaf-editor-core')
 const dmp = new DMP()
 
 // Do not attempt to produce a diff for more than 100ms
@@ -38,63 +36,4 @@ module.exports = {
     }
     return ops
   },
-
-  /**
-   * @param {import("overleaf-editor-core").StringFileData} file
-   * @param {string} after
-   * @return {TextOperation}
-   */
-  diffAsHistoryOTEditOperation(file, after) {
-    const beforeWithoutTrackedDeletes = file.getContent({
-      filterTrackedDeletes: true,
-    })
-    const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after)
-    dmp.diff_cleanupSemantic(diffs)
-
-    const trackedChanges = file.trackedChanges.asSorted()
-    let nextTc = trackedChanges.shift()
-
-    const op = new TextOperation()
-    for (const diff of diffs) {
-      let [type, content] = diff
-      if (type === this.ADDED) {
-        op.insert(content)
-      } else if (type === this.REMOVED || type === this.UNCHANGED) {
-        while (op.baseLength + content.length > nextTc?.range.start) {
-          if (nextTc.tracking.type === 'delete') {
-            const untilRange = nextTc.range.start - op.baseLength
-            if (type === this.REMOVED) {
-              op.remove(untilRange)
-            } else if (type === this.UNCHANGED) {
-              op.retain(untilRange)
-            }
-            op.retain(nextTc.range.end - nextTc.range.start)
-            content = content.slice(untilRange)
-          }
-          nextTc = trackedChanges.shift()
-        }
-        if (type === this.REMOVED) {
-          op.remove(content.length)
-        } else if (type === this.UNCHANGED) {
-          op.retain(content.length)
-        }
-      } else {
-        throw new Error('Unknown type')
-      }
-    }
-    while (nextTc) {
-      if (
-        nextTc.tracking.type !== 'delete' ||
-        nextTc.range.start !== op.baseLength
-      ) {
-        throw new OError(
-          'StringFileData.trackedChanges out of sync: unexpected range after end of diff',
-          { nextTc, baseLength: op.baseLength }
-        )
-      }
-      op.retain(nextTc.range.end - nextTc.range.start)
-      nextTc = trackedChanges.shift()
-    }
-    return op
-  },
 }

services/document-updater/app/js/DocumentManager.js

@@ -12,6 +12,9 @@ const { extractOriginOrSource } = require('./Utils')
 const { getTotalSizeOfLines } = require('./Limits')
 const Settings = require('@overleaf/settings')
 const { StringFileData } = require('overleaf-editor-core')
+const {
+  diffAsTextOperation,
+} = require('overleaf-editor-core/lib/diff_as_text_operation')
 
 const MAX_UNFLUSHED_AGE = Settings.maxUnflushedAgeMs // document should be flushed to mongo this time after a change
 
@@ -194,10 +197,7 @@ const DocumentManager = {
     let op
     if (type === 'history-ot') {
       const file = StringFileData.fromRaw(oldLines)
-      const operation = DiffCodec.diffAsHistoryOTEditOperation(
-        file,
-        newLines.join('\n')
-      )
+      const operation = diffAsTextOperation(file, newLines.join('\n'))
       if (operation.isNoop()) {
         op = []
       } else {

services/history-v1/api/controllers/project_import.js

@@ -12,10 +12,12 @@ const Change = core.Change
 const Chunk = core.Chunk
 const File = core.File
 const FileMap = core.FileMap
+const Origin = core.Origin
 const Snapshot = core.Snapshot
 const TextOperation = core.TextOperation
 
 const logger = require('@overleaf/logger')
+const OError = require('@overleaf/o-error')
 
 const storage = require('../../storage')
 const BatchBlobStore = storage.BatchBlobStore
@@ -24,6 +26,7 @@ const chunkStore = storage.chunkStore
 const HashCheckBlobStore = storage.HashCheckBlobStore
 const commitChanges = storage.commitChanges
 const persistBuffer = storage.persistBuffer
+const buildSetContentChange = storage.buildSetContentChange
 const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
@@ -81,6 +84,17 @@ async function importSnapshot(req, res) {
   res.status(HTTPStatus.OK).json({ projectId: historyId })
 }
 
+// Limits that force us to persist all of the changes.
+function getPersistLimits() {
+  const farFuture = new Date()
+  farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+  return {
+    maxChanges: 0,
+    minChangeTimestamp: farFuture,
+    maxChangeTimestamp: farFuture,
+  }
+}
+
 async function importChanges(req, res, next) {
   const { params, query, body } = parseReq(req, schemas.importChanges)
   const projectId = params.project_id
@@ -97,14 +111,7 @@ async function importChanges(req, res, next) {
     return render.unprocessableEntity(res)
   }
 
-  // Set limits to force us to persist all of the changes.
-  const farFuture = new Date()
-  farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
-  const limits = {
-    maxChanges: 0,
-    minChangeTimestamp: farFuture,
-    maxChangeTimestamp: farFuture,
-  }
+  const limits = getPersistLimits()
 
   const blobStore = new BlobStore(projectId)
   const batchBlobStore = new BatchBlobStore(blobStore)
@@ -176,16 +183,64 @@ async function importChanges(req, res, next) {
   }
 }
 
+async function setContent(req, res) {
+  const { params, body } = parseReq(req, schemas.setContent)
+  const projectId = params.project_id
+  const { pathname, source, userId, timestamp, metadata, trackChanges } = body
+  const content = 'content' in body ? body.content : undefined
+  const blobHash = 'blobHash' in body ? body.blobHash : undefined
+  if (timestamp == null) {
+    // The schema requires a timestamp; this narrows the type for the checker.
+    return render.unprocessableEntity(res)
+  }
+
+  let result
+  try {
+    result = await buildSetContentChange(projectId, pathname, {
+      content,
+      blobHash,
+      metadata,
+      userId,
+      timestamp,
+      origin: new Origin(source),
+      trackChanges,
+    })
+  } catch (err) {
+    if (err instanceof storage.ContentTooLargeError) {
+      return render.requestEntityTooLarge(res)
+    } else if (err instanceof Chunk.NotFoundError) {
+      logger.warn({ err, projectId }, 'chunk not found')
+      return render.notFound(res)
+    } else if (
+      err instanceof storage.BlobNotFoundError ||
+      err instanceof TextOperation.UnprocessableError ||
+      err instanceof File.NotEditableError ||
+      err instanceof FileMap.PathnameError ||
+      err instanceof Snapshot.EditMissingFileError ||
+      err instanceof InvalidChangeError ||
+      (err instanceof OError &&
+        err.message.startsWith('StringFileData.trackedChanges out of sync'))
+    ) {
+      logger.warn({ err, projectId, pathname }, 'set content rejected')
+      return render.unprocessableEntity(res)
+    } else {
+      throw err
+    }
+  }
+
+  const { change, baseVersion } = result
+  res.status(HTTPStatus.OK).json({
+    baseVersion,
+    change: change ? change.toRaw() : null,
+  })
+}
+
 async function flushChanges(req, res, next) {
   const { params } = parseReq(req, schemas.flushChanges)
   const projectId = params.project_id
   // Use the same limits importChanges, since these are passed to persistChanges
-  const farFuture = new Date()
-  farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
   const limits = {
-    maxChanges: 0,
-    minChangeTimestamp: farFuture,
-    maxChangeTimestamp: farFuture,
+    ...getPersistLimits(),
     autoResync: true,
   }
   try {
@@ -209,5 +264,6 @@ async function expireProject(req, res, next) {
 
 exports.importSnapshot = expressify(importSnapshot)
 exports.importChanges = expressify(importChanges)
+exports.setContent = expressify(setContent)
 exports.flushChanges = expressify(flushChanges)
 exports.expireProject = expressify(expireProject)

services/history-v1/api/routes/project_import.js

@@ -31,6 +31,12 @@ router.post(
   projectImportController.importChanges
 )
 
+router.post(
+  '/projects/:project_id/set_content',
+  handleBasicAuth,
+  projectImportController.setContent
+)
+
 router.post(
   '/projects/:project_id/flush',
   handleBasicAuth,

services/history-v1/api/schema.js

@@ -65,6 +65,15 @@ const changeSchema = z
   })
   .passthrough()
 
+const setContentBodyBase = z.object({
+  pathname: z.string().min(1),
+  source: z.string(),
+  userId: zz.objectId().optional(),
+  timestamp: zz.datetime(),
+  metadata: z.object({}).passthrough().optional(),
+  trackChanges: z.boolean().optional(),
+})
+
 const schemas = {
   projectId: z.object({
     params: z
@@ -252,6 +261,31 @@ const schemas = {
     body: z.array(changeSchema),
   }),
 
+  setContent: z.object({
+    params: z.object({
+      project_id: z.string(),
+    }),
+    // strict() rejects unknown keys, so a mixed payload with both content and
+    // blobHash fails both union members and is rejected.
+    body: z
+      .union([
+        setContentBodyBase.extend({ content: z.string() }).strict(),
+        setContentBodyBase
+          .extend({ blobHash: z.string().regex(hexHashPattern) })
+          .strict(),
+      ])
+      .refine(
+        body => {
+          if (body.trackChanges && !body.userId) return false
+          return true
+        },
+        {
+          message: '"userId" is required when trackChanges is set',
+          path: ['userId'],
+        }
+      ),
+  }),
+
   flushChanges: z.object({
     params: z.object({
       project_id: z.string(),

services/history-v1/storage/index.js

@@ -24,5 +24,14 @@ exports.loadGlobalBlobs = loadGlobalBlobs
 const { InvalidChangeError } = require('./lib/errors')
 exports.InvalidChangeError = InvalidChangeError
 
+const {
+  buildSetContentChange,
+  ContentTooLargeError,
+  BlobNotFoundError,
+} = require('./lib/build_set_content_change')
+exports.buildSetContentChange = buildSetContentChange
+exports.ContentTooLargeError = ContentTooLargeError
+exports.BlobNotFoundError = BlobNotFoundError
+
 const { ChunkVersionConflictError } = require('./lib/chunk_store/errors')
 exports.ChunkVersionConflictError = ChunkVersionConflictError

services/history-v1/storage/lib/build_set_content_change.js

@@ -0,0 +1,129 @@
+// @ts-check
+
+'use strict'
+
+const OError = require('@overleaf/o-error')
+const core = require('overleaf-editor-core')
+const {
+  buildSetContentOperations,
+} = require('overleaf-editor-core/lib/build_set_content_operations')
+const assert = require('./assert')
+const chunkStore = require('./chunk_store')
+const { BlobStore } = require('./blob_store')
+
+const { Change, TextOperation } = core
+
+/**
+ * @import { Change, Origin } from 'overleaf-editor-core'
+ */
+
+class ContentTooLargeError extends OError {}
+class BlobNotFoundError extends OError {}
+
+/**
+ * Build a change that sets the content of a pathname (a setDoc-like upsert).
+ *
+ * Computes the change against the current head (persisted chunk plus buffered
+ * changes) without committing it. The caller is responsible for committing
+ * the change with `baseVersion` as the expected end version, rebasing or
+ * transforming it first if the head has moved on in the meantime.
+ *
+ * New content is stored in the blob store as a side effect, so the returned
+ * change is cheap to serialize and can be committed as-is later.
+ *
+ * @param {string} projectId
+ * @param {string} pathname
+ * @param {object} opts
+ * @param {string} [opts.content] new doc content (exactly one of content and
+ *        blobHash must be given)
+ * @param {string} [opts.blobHash] hash of an already created blob with the
+ *        new binary file content
+ * @param {Object} [opts.metadata] metadata for the file; the file's metadata
+ *        is always replaced with this value (defaults to empty metadata)
+ * @param {string} [opts.userId]
+ * @param {Date} opts.timestamp
+ * @param {Origin} opts.origin
+ * @param {boolean} [opts.trackChanges] record the change as tracked changes
+ *        (ignored when blobHash is given)
+ * @return {Promise<{change: Change | null, baseVersion: number}>} the change
+ *         to commit (null when the content and metadata already match) and
+ *         the version of the head it was built against
+ */
+async function buildSetContentChange(projectId, pathname, opts) {
+  const {
+    content,
+    blobHash,
+    metadata,
+    userId,
+    timestamp,
+    origin,
+    trackChanges = false,
+  } = opts
+
+  assert.projectId(projectId, 'bad projectId')
+  if (content != null && content.length > TextOperation.MAX_STRING_LENGTH) {
+    throw new ContentTooLargeError('content is too large', {
+      projectId,
+      pathname,
+      contentLength: content.length,
+    })
+  }
+  if ((content == null) === (blobHash == null)) {
+    throw new OError(
+      'buildSetContentChange: exactly one of content and blobHash must be given',
+      { projectId, pathname }
+    )
+  }
+  if (trackChanges && !userId) {
+    throw new OError('buildSetContentChange: trackChanges requires a userId', {
+      projectId,
+      pathname,
+    })
+  }
+
+  const blobStore = new BlobStore(projectId)
+  let blob
+  if (blobHash != null) {
+    blob = await blobStore.getBlob(blobHash)
+    if (blob == null) {
+      throw new BlobNotFoundError('blob not found', { projectId, blobHash })
+    }
+  }
+
+  const tracking = trackChanges
+    ? { userId: /** @type {string} */ (userId), ts: timestamp }
+    : undefined
+
+  const chunk = await chunkStore.loadLatest(projectId)
+  const snapshot = chunk.getSnapshot()
+  snapshot.applyAll(chunk.getChanges())
+  const baseVersion = chunk.getEndVersion()
+
+  const { operations, status } = await buildSetContentOperations({
+    file: snapshot.getFile(pathname),
+    pathname,
+    content,
+    blob,
+    metadata,
+    tracking,
+    blobStore,
+  })
+  if (status === 'noop') {
+    return { change: null, baseVersion }
+  }
+
+  const change = new Change(
+    operations,
+    timestamp,
+    [],
+    origin,
+    userId ? [userId] : []
+  )
+  return { change, baseVersion }
+}
+
+module.exports = {
+  buildSetContentChange,
+  ContentTooLargeError,
+  BlobNotFoundError,
+}

services/history-v1/test/acceptance/js/api/set_content.test.js

@@ -0,0 +1,325 @@
+'use strict'
+
+const { expect } = require('chai')
+const HTTPStatus = require('http-status')
+const fetch = require('node-fetch')
+const fs = require('node:fs')
+
+const cleanup = require('../storage/support/cleanup')
+const fixtures = require('../storage/support/fixtures')
+const testFiles = require('../storage/support/test_files')
+const testProjects = require('./support/test_projects')
+const testServer = require('./support/test_server')
+
+const storage = require('../../../../storage')
+const chunkStore = storage.chunkStore
+const BlobStore = storage.BlobStore
+
+const { Change, Operation, TextOperation } = require('overleaf-editor-core')
+
+describe('set content', function () {
+  beforeEach(cleanup.everything)
+  beforeEach(fixtures.create)
+
+  function setContent(projectId, body, opts = {}) {
+    const headers = { 'Content-Type': 'application/json' }
+    if (!opts.anonymous) {
+      headers.Authorization = testServer.basicAuthHeader
+    }
+    return fetch(testServer.url(`/api/projects/${projectId}/set_content`), {
+      method: 'POST',
+      body: JSON.stringify({
+        source: 'test-source',
+        userId: 'abcdef0123456789abcdef01',
+        timestamp: new Date().toISOString(),
+        ...body,
+      }),
+      headers,
+    })
+  }
+
+  function postChanges(projectId, changes, endVersion) {
+    return fetch(
+      testServer.url(`/api/projects/${projectId}/changes`) +
+        `?end_version=${endVersion}`,
+      {
+        method: 'POST',
+        body: JSON.stringify(changes),
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: testServer.basicAuthHeader,
+        },
+      }
+    )
+  }
+
+  // Commit a change returned by set_content, as a caller is expected to do
+  async function applyChange(projectId, { change, baseVersion }) {
+    const response = await postChanges(projectId, [change], baseVersion)
+    expect(response.status).to.equal(HTTPStatus.CREATED)
+  }
+
+  async function getFileContent(projectId, pathname) {
+    const blobStore = new BlobStore(projectId)
+    const chunk = await chunkStore.loadLatest(projectId)
+    const snapshot = chunk.getSnapshot()
+    snapshot.applyAll(chunk.getChanges())
+    const file = snapshot.getFile(pathname)
+    await file.load('eager', blobStore)
+    return file
+  }
+
+  it('requires authentication', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(
+      projectId,
+      { pathname: 'main.tex', content: 'hello' },
+      { anonymous: true }
+    )
+    expect(response.status).to.equal(HTTPStatus.UNAUTHORIZED)
+  })
+
+  it('builds create and edit changes and reports noops for a doc', async function () {
+    const projectId = await testProjects.createEmptyProject()
+
+    // Create the doc
+    let response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'one\ntwo\n',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    let result = await response.json()
+    expect(result.baseVersion).to.equal(0)
+    expect(result.change.origin).to.deep.equal({ kind: 'test-source' })
+    expect(result.change.v2Authors).to.deep.equal(['abcdef0123456789abcdef01'])
+    expect(result.change.operations).to.have.length(1)
+    expect(result.change.operations[0].pathname).to.equal('main.tex')
+    expect(result.change.operations[0].file.hash).to.exist
+
+    // The change is not applied until the caller commits it
+    await applyChange(projectId, result)
+
+    // Edit the doc
+    response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'one\ntwo\nthree\n',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    result = await response.json()
+    expect(result.baseVersion).to.equal(1)
+    expect(result.change.operations).to.have.length(1)
+    expect(result.change.operations[0].pathname).to.equal('main.tex')
+    expect(result.change.operations[0].textOperation).to.exist
+    await applyChange(projectId, result)
+
+    // Identical content is a noop
+    response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'one\ntwo\nthree\n',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    expect(await response.json()).to.deep.equal({
+      baseVersion: 2,
+      change: null,
+    })
+
+    const file = await getFileContent(projectId, 'main.tex')
+    expect(file.getContent()).to.equal('one\ntwo\nthree\n')
+  })
+
+  it('returns a change that can be rebased on a concurrent change', async function () {
+    const projectId = await testProjects.createEmptyProject()
+
+    // Create the doc
+    let response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'one\ntwo\nthree\n',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    await applyChange(projectId, await response.json())
+
+    // Build a change that appends a line, without committing it yet
+    response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'one\ntwo\nthree\nfour\n',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    const result = await response.json()
+    expect(result.baseVersion).to.equal(1)
+
+    // Meanwhile, a concurrent change inserts a line at the top
+    const concurrentChange = new Change(
+      [
+        Operation.editFile(
+          'main.tex',
+          TextOperation.fromJSON({
+            textOperation: ['zero\n', 'one\ntwo\nthree\n'.length],
+          })
+        ),
+      ],
+      new Date(),
+      []
+    )
+    let commitResponse = await postChanges(
+      projectId,
+      [concurrentChange.toRaw()],
+      result.baseVersion
+    )
+    expect(commitResponse.status).to.equal(HTTPStatus.CREATED)
+
+    // The returned change no longer applies at its base version
+    commitResponse = await postChanges(
+      projectId,
+      [result.change],
+      result.baseVersion
+    )
+    expect(commitResponse.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+
+    // Rebase the returned change on the concurrent change and commit it
+    const rebasedChange = Change.fromRaw(result.change)
+    Operation.transformMultiple(
+      rebasedChange.getOperations(),
+      concurrentChange.getOperations()
+    )
+    commitResponse = await postChanges(
+      projectId,
+      [rebasedChange.toRaw()],
+      result.baseVersion + 1
+    )
+    expect(commitResponse.status).to.equal(HTTPStatus.CREATED)
+
+    const file = await getFileContent(projectId, 'main.tex')
+    expect(file.getContent()).to.equal('zero\none\ntwo\nthree\nfour\n')
+  })
+
+  it('records tracked changes when trackChanges is set', async function () {
+    const projectId = await testProjects.createEmptyProject()
+
+    let response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello world',
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    await applyChange(projectId, await response.json())
+
+    response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello brave world',
+      trackChanges: true,
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    const result = await response.json()
+    expect(result.baseVersion).to.equal(1)
+    expect(result.change).to.exist
+    await applyChange(projectId, result)
+
+    const file = await getFileContent(projectId, 'main.tex')
+    expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+      'hello brave world'
+    )
+    const trackedChanges = file.getTrackedChanges().asSorted()
+    expect(trackedChanges).to.have.length(1)
+    expect(trackedChanges[0].tracking.type).to.equal('insert')
+    expect(trackedChanges[0].tracking.userId).to.equal(
+      'abcdef0123456789abcdef01'
+    )
+  })
+
+  it('sets binary file content from an uploaded blob', async function () {
+    const projectId = await testProjects.createEmptyProject()
+
+    // Create the blob first, as in the importChanges flow
+    const uploadResponse = await fetch(
+      testServer.url(
+        `/api/projects/${projectId}/blobs/${testFiles.GRAPH_PNG_HASH}`
+      ),
+      {
+        method: 'PUT',
+        body: fs.createReadStream(testFiles.path('graph.png')),
+        headers: { Authorization: testServer.basicAuthHeader },
+      }
+    )
+    expect(uploadResponse.ok).to.be.true
+
+    const response = await setContent(projectId, {
+      pathname: 'images/graph.png',
+      blobHash: testFiles.GRAPH_PNG_HASH,
+    })
+    expect(response.status).to.equal(HTTPStatus.OK)
+    const result = await response.json()
+    expect(result.baseVersion).to.equal(0)
+    expect(result.change.operations).to.have.length(1)
+    expect(result.change.operations[0].file.hash).to.equal(
+      testFiles.GRAPH_PNG_HASH
+    )
+    await applyChange(projectId, result)
+
+    const file = await getFileContent(projectId, 'images/graph.png')
+    expect(file.isEditable()).to.be.false
+    expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH)
+  })
+
+  it('rejects a payload with both content and blobHash', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello',
+      blobHash: testFiles.GRAPH_PNG_HASH,
+    })
+    expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+  })
+
+  it('rejects a payload with neither content nor blobHash', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, { pathname: 'main.tex' })
+    expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+  })
+
+  it('rejects a payload without a timestamp', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello',
+      timestamp: undefined,
+    })
+    expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+  })
+
+  it('rejects trackChanges without a userId', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello',
+      trackChanges: true,
+      userId: undefined,
+    })
+    expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+  })
+
+  it('rejects an unknown blob hash', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, {
+      pathname: 'images/graph.png',
+      blobHash: testFiles.GRAPH_PNG_HASH,
+    })
+    expect(response.status).to.equal(HTTPStatus.UNPROCESSABLE_ENTITY)
+  })
+
+  it('rejects content that is too large', async function () {
+    const projectId = await testProjects.createEmptyProject()
+    const response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'x'.repeat(TextOperation.MAX_STRING_LENGTH + 1),
+    })
+    expect(response.status).to.equal(HTTPStatus.REQUEST_ENTITY_TOO_LARGE)
+  })
+
+  it('returns 404 for an uninitialized project', async function () {
+    const projectId = fixtures.docs.uninitializedProject.id
+    const response = await setContent(projectId, {
+      pathname: 'main.tex',
+      content: 'hello',
+    })
+    expect(response.status).to.equal(HTTPStatus.NOT_FOUND)
+  })
+})

services/history-v1/test/acceptance/js/storage/build_set_content_change.test.js

@@ -0,0 +1,515 @@
+'use strict'
+
+const { expect } = require('chai')
+
+const cleanup = require('./support/cleanup')
+const fixtures = require('./support/fixtures')
+const testFiles = require('./support/test_files.js')
+const storage = require('../../../../storage')
+const chunkStore = storage.chunkStore
+const queueChanges = storage.queueChanges
+const persistChanges = storage.persistChanges
+const commitChanges = storage.commitChanges
+const redisBackend = require('../../../../storage/lib/chunk_store/redis')
+const { buildSetContentChange, ContentTooLargeError, BlobNotFoundError } =
+  storage
+const BlobStore = storage.BlobStore
+
+const core = require('overleaf-editor-core')
+const AddFileOperation = core.AddFileOperation
+const Change = core.Change
+const Chunk = core.Chunk
+const File = core.File
+const Origin = core.Origin
+const TextOperation = core.TextOperation
+
+describe('buildSetContentChange', function () {
+  const userId = 'abcdef0123456789abcdef01'
+  const source = 'test-source'
+
+  let limitsToPersistImmediately
+  before(function () {
+    // Used to provide a limit which forces us to persist all of the changes
+    const farFuture = new Date()
+    farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+    limitsToPersistImmediately = {
+      minChangeTimestamp: farFuture,
+      maxChangeTimestamp: farFuture,
+      maxChanges: 10,
+      maxChunkChanges: 10,
+    }
+  })
+
+  beforeEach(cleanup.everything)
+  beforeEach(fixtures.create)
+
+  function makeOpts(opts = {}) {
+    return {
+      userId,
+      timestamp: new Date(),
+      origin: new Origin(source),
+      ...opts,
+    }
+  }
+
+  async function setupProjectWithDoc(pathname, content) {
+    const projectId = fixtures.docs.uninitializedProject.id
+    await chunkStore.initializeProject(projectId)
+    const blobStore = new BlobStore(projectId)
+    const blob = await blobStore.putString(content)
+    await persistChanges(
+      projectId,
+      [
+        new Change(
+          [new AddFileOperation(pathname, File.createLazyFromBlobs(blob))],
+          new Date(),
+          []
+        ),
+      ],
+      limitsToPersistImmediately,
+      0
+    )
+    return { projectId, blobStore }
+  }
+
+  async function commit(projectId, { change, baseVersion }) {
+    await commitChanges(
+      projectId,
+      [change],
+      limitsToPersistImmediately,
+      baseVersion,
+      { historyBufferLevel: 4 }
+    )
+  }
+
+  async function getFile(projectId, pathname) {
+    const blobStore = new BlobStore(projectId)
+    const chunk = await chunkStore.loadLatest(projectId)
+    const snapshot = chunk.getSnapshot()
+    snapshot.applyAll(chunk.getChanges())
+    const file = snapshot.getFile(pathname)
+    if (file) {
+      await file.load('eager', blobStore)
+    }
+    return { file, version: chunk.getEndVersion() }
+  }
+
+  describe('with string content', function () {
+    it('edits an existing doc with a minimal diff', async function () {
+      const { projectId } = await setupProjectWithDoc(
+        'main.tex',
+        'one\ntwo\nthree\n'
+      )
+
+      const result = await buildSetContentChange(
+        projectId,
+        'main.tex',
+        makeOpts({ content: 'one\ntwo and a half\nthree\n' })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      // The change carries the given metadata
+      const rawChange = result.change.toRaw()
+      expect(rawChange.origin).to.deep.equal({ kind: source })
+      expect(rawChange.v2Authors).to.deep.equal([userId])
+      expect(rawChange.operations).to.have.length(1)
+      expect(rawChange.operations[0].pathname).to.equal('main.tex')
+      expect(rawChange.operations[0].textOperation).to.exist
+
+      // Nothing was committed yet
+      const redisState = await redisBackend.getState(projectId)
+      expect(redisState.headVersion).to.be.null
+
+      await commit(projectId, result)
+      const { file, version } = await getFile(projectId, 'main.tex')
+      expect(version).to.equal(2)
+      expect(file.getContent()).to.equal('one\ntwo and a half\nthree\n')
+    })
+
+    it('reports identical content as a noop', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'same\n')
+
+      const result = await buildSetContentChange(
+        projectId,
+        'main.tex',
+        makeOpts({ content: 'same\n' })
+      )
+      expect(result).to.deep.equal({ change: null, baseVersion: 1 })
+    })
+
+    it('updates doc metadata when explicitly provided', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'same\n')
+
+      const result = await buildSetContentChange(
+        projectId,
+        'main.tex',
+        makeOpts({ content: 'same\n', metadata: { main: true } })
+      )
+      expect(result.baseVersion).to.equal(1)
+      expect(result.change.toRaw().operations).to.deep.equal([
+        {
+          pathname: 'main.tex',
+          metadata: { main: true },
+        },
+      ])
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'main.tex')
+      expect(file.getContent()).to.equal('same\n')
+      expect(file.getMetadata()).to.deep.equal({ main: true })
+    })
+
+    it('diffs against content pending in the redis buffer', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'one\ntwo\n')
+
+      // Queue an edit in the redis buffer without persisting it
+      const editOperation = core.Operation.editFile(
+        'main.tex',
+        TextOperation.fromJSON({ textOperation: [8, 'three\n'] })
+      )
+      const editChange = new Change([editOperation], new Date(), [])
+      await queueChanges(projectId, [editChange], 1)
+
+      const result = await buildSetContentChange(
+        projectId,
+        'main.tex',
+        makeOpts({ content: 'one\ntwo\nthree\nfour\n' })
+      )
+      expect(result.baseVersion).to.equal(2)
+
+      await commit(projectId, result)
+      const { file, version } = await getFile(projectId, 'main.tex')
+      expect(version).to.equal(3)
+      expect(file.getContent()).to.equal('one\ntwo\nthree\nfour\n')
+    })
+
+    it('replaces a binary file with an editable doc', async function () {
+      const projectId = fixtures.docs.uninitializedProject.id
+      await chunkStore.initializeProject(projectId)
+      const blobStore = new BlobStore(projectId)
+      await blobStore.putFile(testFiles.path('graph.png'))
+      await persistChanges(
+        projectId,
+        [
+          new Change(
+            [
+              new AddFileOperation(
+                'figure.tex',
+                File.fromHash(testFiles.GRAPH_PNG_HASH)
+              ),
+            ],
+            new Date(),
+            []
+          ),
+        ],
+        limitsToPersistImmediately,
+        0
+      )
+
+      const result = await buildSetContentChange(
+        projectId,
+        'figure.tex',
+        makeOpts({ content: 'hello world', metadata: { main: true } })
+      )
+      expect(result.baseVersion).to.equal(1)
+      const rawChange = result.change.toRaw()
+      expect(rawChange.operations).to.have.length(2)
+      expect(rawChange.operations[0]).to.deep.equal({
+        pathname: 'figure.tex',
+        newPathname: '',
+      })
+      expect(rawChange.operations[1].file.hash).to.exist
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'figure.tex')
+      expect(file.isEditable()).to.be.true
+      expect(file.getContent()).to.equal('hello world')
+      expect(file.getMetadata()).to.deep.equal({ main: true })
+    })
+
+    it('creates a new doc when there is no file at the pathname', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'main\n')
+
+      const result = await buildSetContentChange(
+        projectId,
+        'chapters/one.tex',
+        makeOpts({ content: 'chapter one\n' })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'chapters/one.tex')
+      expect(file.getContent()).to.equal('chapter one\n')
+    })
+
+    it('records the edit as tracked changes when requested', async function () {
+      const timestamp = new Date()
+      const { projectId } = await setupProjectWithDoc(
+        'main.tex',
+        'hello cruel world'
+      )
+
+      const result = await buildSetContentChange(
+        projectId,
+        'main.tex',
+        makeOpts({
+          content: 'hello brave world',
+          trackChanges: true,
+          timestamp,
+        })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'main.tex')
+      // The removed content is retained as a tracked delete
+      expect(file.getContent({ filterTrackedDeletes: true })).to.equal(
+        'hello brave world'
+      )
+      expect(file.getContent()).to.contain('cruel')
+      const trackedChanges = file.getTrackedChanges().asSorted()
+      expect(trackedChanges).to.have.length(2)
+      const trackedDelete = trackedChanges.find(
+        tc => tc.tracking.type === 'delete'
+      )
+      const trackedInsert = trackedChanges.find(
+        tc => tc.tracking.type === 'insert'
+      )
+      expect(trackedDelete).to.exist
+      expect(trackedInsert).to.exist
+      for (const trackedChange of trackedChanges) {
+        expect(trackedChange.tracking.userId).to.equal(userId)
+        expect(trackedChange.tracking.ts.toISOString()).to.equal(
+          timestamp.toISOString()
+        )
+      }
+    })
+
+    it('records new doc content as a tracked insert when requested', async function () {
+      const timestamp = new Date()
+      const { projectId } = await setupProjectWithDoc('main.tex', 'main\n')
+
+      const result = await buildSetContentChange(
+        projectId,
+        'new.tex',
+        makeOpts({ content: 'new content', trackChanges: true, timestamp })
+      )
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'new.tex')
+      expect(file.getContent()).to.equal('new content')
+      expect(file.getTrackedChanges().toRaw()).to.deep.equal([
+        {
+          range: { pos: 0, length: 'new content'.length },
+          tracking: {
+            type: 'insert',
+            userId,
+            ts: timestamp.toISOString(),
+          },
+        },
+      ])
+    })
+
+    it('rejects content that is too large', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'main\n')
+
+      let error
+      try {
+        await buildSetContentChange(
+          projectId,
+          'main.tex',
+          makeOpts({ content: 'x'.repeat(TextOperation.MAX_STRING_LENGTH + 1) })
+        )
+      } catch (err) {
+        error = err
+      }
+      expect(error).to.be.an.instanceof(ContentTooLargeError)
+    })
+
+    it('throws when the project does not exist', async function () {
+      let error
+      try {
+        await buildSetContentChange(
+          fixtures.docs.uninitializedProject.id,
+          'main.tex',
+          makeOpts({ content: 'hello' })
+        )
+      } catch (err) {
+        error = err
+      }
+      expect(error).to.be.an.instanceof(Chunk.NotFoundError)
+    })
+  })
+
+  describe('with a blob', function () {
+    it('replaces a doc with a binary file', async function () {
+      const { projectId, blobStore } = await setupProjectWithDoc(
+        'figure.png',
+        'placeholder'
+      )
+      await blobStore.putFile(testFiles.path('graph.png'))
+
+      const result = await buildSetContentChange(
+        projectId,
+        'figure.png',
+        makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'figure.png')
+      expect(file.isEditable()).to.be.false
+      expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH)
+    })
+
+    it('reports the same hash and metadata as a noop', async function () {
+      const projectId = fixtures.docs.uninitializedProject.id
+      await chunkStore.initializeProject(projectId)
+      const blobStore = new BlobStore(projectId)
+      const blob = await blobStore.putFile(testFiles.path('graph.png'))
+      await persistChanges(
+        projectId,
+        [
+          new Change(
+            [
+              new AddFileOperation(
+                'figure.png',
+                File.createLazyFromBlobs(blob)
+              ),
+            ],
+            new Date(),
+            []
+          ),
+        ],
+        limitsToPersistImmediately,
+        0
+      )
+
+      const result = await buildSetContentChange(
+        projectId,
+        'figure.png',
+        makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH })
+      )
+      expect(result).to.deep.equal({ change: null, baseVersion: 1 })
+    })
+
+    it('only sets metadata when the hash matches but metadata differs', async function () {
+      const projectId = fixtures.docs.uninitializedProject.id
+      await chunkStore.initializeProject(projectId)
+      const blobStore = new BlobStore(projectId)
+      const blob = await blobStore.putFile(testFiles.path('graph.png'))
+      await persistChanges(
+        projectId,
+        [
+          new Change(
+            [
+              new AddFileOperation(
+                'figure.png',
+                File.createLazyFromBlobs(blob)
+              ),
+            ],
+            new Date(),
+            []
+          ),
+        ],
+        limitsToPersistImmediately,
+        0
+      )
+
+      const result = await buildSetContentChange(
+        projectId,
+        'figure.png',
+        makeOpts({
+          blobHash: testFiles.GRAPH_PNG_HASH,
+          metadata: { importer: 'github' },
+        })
+      )
+      expect(result.baseVersion).to.equal(1)
+      expect(result.change.toRaw().operations).to.deep.equal([
+        {
+          pathname: 'figure.png',
+          metadata: { importer: 'github' },
+        },
+      ])
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'figure.png')
+      expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH)
+      expect(file.getMetadata()).to.deep.equal({ importer: 'github' })
+    })
+
+    it('adds a new file when there is no file at the pathname', async function () {
+      const { projectId, blobStore } = await setupProjectWithDoc(
+        'main.tex',
+        'main\n'
+      )
+      await blobStore.putFile(testFiles.path('graph.png'))
+
+      const result = await buildSetContentChange(
+        projectId,
+        'images/graph.png',
+        makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'images/graph.png')
+      expect(file.getHash()).to.equal(testFiles.GRAPH_PNG_HASH)
+    })
+
+    it('throws when the blob does not exist', async function () {
+      const { projectId } = await setupProjectWithDoc('main.tex', 'main\n')
+
+      let error
+      try {
+        await buildSetContentChange(
+          projectId,
+          'images/graph.png',
+          makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH })
+        )
+      } catch (err) {
+        error = err
+      }
+      expect(error).to.be.an.instanceof(BlobNotFoundError)
+    })
+
+    it('ignores the trackChanges flag', async function () {
+      const { projectId, blobStore } = await setupProjectWithDoc(
+        'main.tex',
+        'main\n'
+      )
+      await blobStore.putFile(testFiles.path('graph.png'))
+
+      const result = await buildSetContentChange(
+        projectId,
+        'images/graph.png',
+        makeOpts({ blobHash: testFiles.GRAPH_PNG_HASH, trackChanges: true })
+      )
+      expect(result.baseVersion).to.equal(1)
+
+      await commit(projectId, result)
+      const { file } = await getFile(projectId, 'images/graph.png')
+      expect(file.getRangesHash()).to.not.exist
+    })
+  })
+
+  it('requires exactly one of content and blobHash', async function () {
+    const { projectId } = await setupProjectWithDoc('main.tex', 'main\n')
+
+    for (const opts of [
+      {},
+      { content: 'a', blobHash: testFiles.GRAPH_PNG_HASH },
+    ]) {
+      let error
+      try {
+        await buildSetContentChange(projectId, 'main.tex', makeOpts(opts))
+      } catch (err) {
+        error = err
+      }
+      expect(error)
+        .to.be.an('error')
+        .with.property('message')
+        .that.contains('exactly one of content and blobHash')
+    }
+  })
+})

yarn.lock

@@ -25756,6 +25756,7 @@ __metadata:
     "@types/path-browserify": "npm:^1.0.3"
     chai: "npm:^3.3.0"
     check-types: "npm:^5.1.0"
+    diff-match-patch: "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688"
     eslint: "npm:^10.4.0"
     lodash: "npm:^4.18.1"
     mocha: "npm:^11.1.0"