[history-ot] initial implementation of using doc-level history-ot (#25054)

* [history-v1-ot] initial implementation of using doc-level history-v1-ot * [web] fix advancing of the otMigrationStage Use 'nextStage' for the user provided, desired stage when advancing. Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [document-updater] document size check in editor-core * [history-ot] rename history-v1-ot to history-ot and add types * [history-ot] apply review feedback - remove extra !! - merge variable assignment when processing diff-match-match output - add helper function for getting docstore lines view of StringFileData Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> * Revert "[document-updater] add safe rollback point for history-ot (#25283)" This reverts commit d7230dd14a379a27d2c6ab03a006463a18979d06 Signed-off-by: Jakob Ackermann <jakob.ackermann@overleaf.com> --------- Signed-off-by: Jakob Ackermann <jakob.ackermann@overleaf.com> Co-authored-by: Brian Gough <brian.gough@overleaf.com> Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> GitOrigin-RevId: 89c497782adb0427635d50d02263d6f535b12481
2026-11-14 14:13:43 +01:00 · 2025-05-08 08:05:44 +00:00
parent 4d93187e58
commit e8b5ee2ff9
48 changed files with 1828 additions and 223 deletions
@@ -52,6 +52,11 @@ async function getDocument(req, res) {
      'overleaf.history.rangesSupportEnabled',
      false
    )
+    const otMigrationStage = _.get(
+      project,
+      'overleaf.history.otMigrationStage',
+      0
+    )

    // all projects are now migrated to Full Project History, keeping the field
    // for API compatibility
@@ -65,6 +70,7 @@ async function getDocument(req, res) {
      projectHistoryId,
      projectHistoryType,
      historyRangesSupport,
+      otMigrationStage,
      resolvedCommentIds,
    })
  }
@@ -300,6 +300,18 @@ class NonDeletableEntityError extends OError {
  }
 }

+class FoundConnectedClientsError extends OError {
+  constructor(nConnectedClients) {
+    super(`found ${nConnectedClients} remaining connected clients`)
+  }
+}
+
+class ConcurrentLoadingOfDocsDetectedError extends OError {
+  constructor() {
+    super('concurrent loading of docs detected')
+  }
+}
+
 module.exports = {
  OError,
  BackwardCompatibleError,
@@ -356,4 +368,6 @@ module.exports = {
  InvalidEmailError,
  InvalidInstitutionalEmailError,
  NonDeletableEntityError,
+  FoundConnectedClientsError,
+  ConcurrentLoadingOfDocsDetectedError,
 }
@@ -0,0 +1,56 @@
+import ProjectGetter from '../Project/ProjectGetter.js'
+import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.js'
+import HistoryManager from '../History/HistoryManager.js'
+import * as RealTimeHandler from '../References/RealTime/RealTimeHandler.mjs'
+import ProjectOptionsHandler from '../Project/ProjectOptionsHandler.js'
+import {
+  NotFoundError,
+  FoundConnectedClientsError,
+  ConcurrentLoadingOfDocsDetectedError,
+} from '../Errors/Errors.js'
+
+async function ensureNoConnectedClients(projectId) {
+  const n = await RealTimeHandler.countConnectedClients(projectId)
+  if (n > 0) throw new FoundConnectedClientsError(n)
+}
+
+/**
+ * @param {string} projectId
+ * @param {number} nextStage
+ * @return {Promise<{otMigrationStage: number}>}
+ */
+export async function advanceOTMigrationStage(projectId, nextStage) {
+  const project = await ProjectGetter.promises.getProject(projectId, {
+    overleaf: true,
+  })
+  if (!project) throw new NotFoundError()
+  const { otMigrationStage } = project?.overleaf?.history || {}
+  if (otMigrationStage >= nextStage) return { otMigrationStage }
+
+  // NOTE: For the single connected client case, we could emit a pub/sub event here asking any (inactive) client without pending edits to disconnect briefly.
+  // e.g. EditorRealTimeController.emitToRoom(projectId, 'attempt-history-ot-migration')
+
+  // Ensure we can perform the hard migration
+  await ensureNoConnectedClients(projectId)
+
+  // Flush ahead of migrating to keep the time under lock down.
+  await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete(projectId)
+  // Avoid mixing update types
+  await HistoryManager.promises.flushProject(projectId)
+
+  // Obtain lock
+  if (!(await DocumentUpdaterHandler.promises.blockProject(projectId))) {
+    throw new ConcurrentLoadingOfDocsDetectedError()
+  }
+
+  try {
+    // Perform the mongo update and tell caller about the latest stage.
+    return await ProjectOptionsHandler.promises.setOTMigrationStage(
+      projectId,
+      nextStage
+    )
+  } finally {
+    // Unlock again (The lock will expire after 30s otherwise)
+    await DocumentUpdaterHandler.promises.unblockProject(projectId)
+  }
+}
@@ -2,6 +2,8 @@ const { Project } = require('../../models/Project')
 const settings = require('@overleaf/settings')
 const { callbackify } = require('util')
 const { db, ObjectId } = require('../../infrastructure/mongodb')
+const Errors = require('../Errors/Errors')
+const { ReturnDocument } = require('mongodb-legacy')
 const safeCompilers = ['xelatex', 'pdflatex', 'latex', 'lualatex']

 const ProjectOptionsHandler = {
@@ -73,6 +75,21 @@ const ProjectOptionsHandler = {
    // because rangesSupportEnabled is not part of the schema?
    return db.projects.updateOne(conditions, update)
  },
+
+  async setOTMigrationStage(projectId, nextStage) {
+    const project = await db.projects.findOneAndUpdate(
+      { _id: new ObjectId(projectId) },
+      // Use $max to ensure that we never downgrade the migration stage.
+      { $max: { 'overleaf.history.otMigrationStage': nextStage } },
+      {
+        returnDocument: ReturnDocument.AFTER,
+        projection: { 'overleaf.history.otMigrationStage': 1 },
+      }
+    )
+    if (!project) throw new Errors.NotFoundError('project does not exist')
+    const { otMigrationStage } = project.overleaf.history
+    return { otMigrationStage }
+  },
 }

 module.exports = {
@@ -0,0 +1,9 @@
+import Settings from '@overleaf/settings'
+import { fetchJson } from '@overleaf/fetch-utils'
+
+export async function countConnectedClients(projectId) {
+  const url = new URL(Settings.apis.realTime.url)
+  url.pathname = `/project/${projectId}/count-connected-clients`
+  const { nConnectedClients } = await fetchJson(url)
+  return nConnectedClients
+}
@@ -99,6 +99,7 @@ const ProjectSchema = new Schema(
        allowDowngrade: { type: Boolean },
        zipFileArchivedInProject: { type: Boolean },
        rangesSupportEnabled: { type: Boolean },
+        otMigrationStage: { type: Number },
      },
    },
    collabratecUsers: [
@@ -2,7 +2,7 @@
 // Migrated from services/web/frontend/js/ide/editor/Document.js

 import RangesTracker from '@overleaf/ranges-tracker'
-import { ShareJsDoc } from './share-js-doc'
+import { OTType, ShareJsDoc } from './share-js-doc'
 import { debugConsole } from '@/utils/debugging'
 import { Socket } from '@/features/ide-react/connection/types/socket'
 import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
@@ -28,6 +28,7 @@ import {
 } from '@/features/ide-react/editor/types/document'
 import { ThreadId } from '../../../../../types/review-panel/review-panel'
 import getMeta from '@/utils/meta'
+import OError from '@overleaf/o-error'

 const MAX_PENDING_OP_SIZE = 64

@@ -447,16 +448,36 @@ export class DocumentContainer extends EventEmitter {
        'joinDoc',
        this.doc_id,
        this.doc.getVersion(),
-        { encodeRanges: true, age: this.doc.getTimeSinceLastServerActivity() },
-        (error, docLines, version, updates, ranges) => {
+        {
+          encodeRanges: true,
+          age: this.doc.getTimeSinceLastServerActivity(),
+          supportsHistoryV1OT: true,
+        },
+        (
+          error,
+          docLines,
+          version,
+          updates,
+          ranges,
+          type = 'sharejs-text-ot'
+        ) => {
          if (error) {
            callback?.(error)
            return
          }
          this.joined = true
          this.doc?.catchUp(updates)
-          this.decodeRanges(ranges)
-          this.catchUpRanges(ranges?.changes, ranges?.comments)
+          if (this.doc?.getType() !== type) {
+            // TODO(24596): page reload after checking for pending ops?
+            throw new OError('ot type mismatch', {
+              got: type,
+              want: this.doc?.getType(),
+            })
+          }
+          if (type === 'sharejs-text-ot') {
+            this.decodeRanges(ranges)
+            this.catchUpRanges(ranges?.changes, ranges?.comments)
+          }
          callback?.()
        }
      )
@@ -464,8 +485,18 @@ export class DocumentContainer extends EventEmitter {
      this.socket.emit(
        'joinDoc',
        this.doc_id,
-        { encodeRanges: true },
-        (error, docLines, version, updates, ranges) => {
+        {
+          encodeRanges: true,
+          supportsHistoryV1OT: true,
+        },
+        (
+          error,
+          docLines,
+          version,
+          updates,
+          ranges,
+          type: OTType = 'sharejs-text-ot'
+        ) => {
          if (error) {
            callback?.(error)
            return
@@ -477,9 +508,12 @@ export class DocumentContainer extends EventEmitter {
            version,
            this.socket,
            this.globalEditorWatchdogManager,
-            this.ideEventEmitter
+            this.ideEventEmitter,
+            type
          )
-          this.decodeRanges(ranges)
+          if (type === 'sharejs-text-ot') {
+            this.decodeRanges(ranges)
+          }
          this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
          this.bindToShareJsDocEvents()
          callback?.()
@@ -580,7 +614,9 @@ export class DocumentContainer extends EventEmitter {
    this.doc.on(
      'change',
      (ops: AnyOperation[], oldSnapshot: any, msg: Message) => {
-        this.applyOpsToRanges(ops, msg)
+        if (this.getType() === 'sharejs-text-ot') {
+          this.applyOpsToRanges(ops, msg)
+        }
        if (docChangedTimeout) {
          window.clearTimeout(docChangedTimeout)
        }
@@ -2,7 +2,7 @@
 // Migrated from services/web/frontend/js/ide/editor/ShareJsDoc.js

 import EventEmitter from '../../../utils/EventEmitter'
-import { Doc } from '@/vendor/libs/sharejs'
+import sharejs, { Doc } from '@/vendor/libs/sharejs'
 import { Socket } from '@/features/ide-react/connection/types/socket'
 import { debugConsole } from '@/utils/debugging'
 import { decodeUtf8 } from '@/utils/decode-utf8'
@@ -12,11 +12,18 @@ import {
  Message,
  ShareJsConnectionState,
  ShareJsOperation,
+  ShareJsTextType,
  TrackChangesIdSeeds,
 } from '@/features/ide-react/editor/types/document'
 import { EditorFacade } from '@/features/source-editor/extensions/realtime'
 import { recordDocumentFirstChangeEvent } from '@/features/event-tracking/document-first-change-event'
 import getMeta from '@/utils/meta'
+import { HistoryOTType } from './share-js-history-ot-type'
+import { StringFileData } from 'overleaf-editor-core/index'
+import {
+  RawEditOperation,
+  StringFileRawData,
+} from 'overleaf-editor-core/lib/types'

 // All times below are in milliseconds
 const SINGLE_USER_FLUSH_DELAY = 2000
@@ -27,6 +34,7 @@ const FATAL_OP_TIMEOUT = 45000
 const RECENT_ACK_LIMIT = 2 * SINGLE_USER_FLUSH_DELAY

 type Update = Record<string, any>
+export type OTType = 'sharejs-text-ot' | 'history-ot'

 type Connection = {
  send: (update: Update) => void
@@ -35,7 +43,6 @@ type Connection = {
 }

 export class ShareJsDoc extends EventEmitter {
-  type: string
  track_changes = false
  track_changes_id_seeds: TrackChangesIdSeeds | null = null
  connection: Connection
@@ -57,12 +64,24 @@ export class ShareJsDoc extends EventEmitter {
    version: number,
    readonly socket: Socket,
    private readonly globalEditorWatchdogManager: EditorWatchdogManager,
-    private readonly eventEmitter: IdeEventEmitter
+    private readonly eventEmitter: IdeEventEmitter,
+    readonly type: OTType = 'sharejs-text-ot'
  ) {
    super()
-    this.type = 'text'
+    let sharejsType: ShareJsTextType = sharejs.types.text
    // Decode any binary bits of data
-    const snapshot = docLines.map(line => decodeUtf8(line)).join('\n')
+    let snapshot: string | StringFileData
+    if (this.type === 'history-ot') {
+      snapshot = StringFileData.fromRaw(
+        docLines as unknown as StringFileRawData
+      )
+      sharejsType = new HistoryOTType(snapshot) as ShareJsTextType<
+        StringFileData,
+        RawEditOperation[]
+      >
+    } else {
+      snapshot = docLines.map(line => decodeUtf8(line)).join('\n')
+    }

    this.connection = {
      send: (update: Update) => {
@@ -89,7 +108,7 @@ export class ShareJsDoc extends EventEmitter {
    }

    this._doc = new Doc(this.connection, this.doc_id, {
-      type: this.type,
+      type: sharejsType,
    })
    this._doc.setFlushDelay(SINGLE_USER_FLUSH_DELAY)
    this._doc.on('change', (...args: any[]) => {
@@ -0,0 +1,131 @@
+import EventEmitter from '@/utils/EventEmitter'
+import {
+  EditOperationBuilder,
+  InsertOp,
+  RemoveOp,
+  RetainOp,
+  StringFileData,
+  TextOperation,
+} from 'overleaf-editor-core'
+import { RawEditOperation } from 'overleaf-editor-core/lib/types'
+
+function loadTextOperation(raw: RawEditOperation): TextOperation {
+  const operation = EditOperationBuilder.fromJSON(raw)
+  if (!(operation instanceof TextOperation)) {
+    throw new Error(`operation not supported: ${operation.constructor.name}`)
+  }
+  return operation
+}
+
+export class HistoryOTType extends EventEmitter {
+  // stub interface, these are actually on the Doc
+  api: HistoryOTType
+  snapshot: StringFileData
+
+  constructor(snapshot: StringFileData) {
+    super()
+    this.api = this
+    this.snapshot = snapshot
+  }
+
+  transformX(raw1: RawEditOperation[], raw2: RawEditOperation[]) {
+    const [a, b] = TextOperation.transform(
+      loadTextOperation(raw1[0]),
+      loadTextOperation(raw2[0])
+    )
+    return [[a.toJSON()], [b.toJSON()]]
+  }
+
+  apply(snapshot: StringFileData, rawEditOperation: RawEditOperation[]) {
+    const operation = loadTextOperation(rawEditOperation[0])
+    const afterFile = StringFileData.fromRaw(snapshot.toRaw())
+    afterFile.edit(operation)
+    this.snapshot = afterFile
+    return afterFile
+  }
+
+  compose(op1: RawEditOperation[], op2: RawEditOperation[]) {
+    return [
+      loadTextOperation(op1[0]).compose(loadTextOperation(op2[0])).toJSON(),
+    ]
+  }
+
+  // Do not provide normalize, used by submitOp to fixup bad input.
+  // normalize(op: TextOperation) {}
+
+  // Do not provide invert, only needed for reverting a rejected update.
+  // We are displaying an out-of-sync modal when an op is rejected.
+  // invert(op: TextOperation) {}
+
+  // API
+  insert(pos: number, text: string, fromUndo: boolean) {
+    const old = this.getText()
+    const op = new TextOperation()
+    op.retain(pos)
+    op.insert(text)
+    op.retain(old.length - pos)
+    this.submitOp([op.toJSON()])
+  }
+
+  del(pos: number, length: number, fromUndo: boolean) {
+    const old = this.getText()
+    const op = new TextOperation()
+    op.retain(pos)
+    op.remove(length)
+    op.retain(old.length - pos - length)
+    this.submitOp([op.toJSON()])
+  }
+
+  getText() {
+    return this.snapshot.getContent({ filterTrackedDeletes: true })
+  }
+
+  getLength() {
+    return this.getText().length
+  }
+
+  _register() {
+    this.on(
+      'remoteop',
+      (rawEditOperation: RawEditOperation[], oldSnapshot: StringFileData) => {
+        const operation = loadTextOperation(rawEditOperation[0])
+        const str = oldSnapshot.getContent()
+        if (str.length !== operation.baseLength)
+          throw new TextOperation.ApplyError(
+            "The operation's base length must be equal to the string's length.",
+            operation,
+            str
+          )
+
+        let outputCursor = 0
+        let inputCursor = 0
+        for (const op of operation.ops) {
+          if (op instanceof RetainOp) {
+            inputCursor += op.length
+            outputCursor += op.length
+          } else if (op instanceof InsertOp) {
+            this.emit('insert', outputCursor, op.insertion, op.insertion.length)
+            outputCursor += op.insertion.length
+          } else if (op instanceof RemoveOp) {
+            this.emit(
+              'delete',
+              outputCursor,
+              str.slice(inputCursor, inputCursor + op.length)
+            )
+            inputCursor += op.length
+          }
+        }
+
+        if (inputCursor !== str.length)
+          throw new TextOperation.ApplyError(
+            "The operation didn't operate on the whole string.",
+            operation,
+            str
+          )
+      }
+    )
+  }
+
+  // stub-interface, provided by sharejs.Doc
+  submitOp(op: RawEditOperation[]) {}
+}
@@ -1,3 +1,4 @@
+import { StringFileData } from 'overleaf-editor-core'
 import { AnyOperation } from '../../../../../../types/change'

 export type Version = number
@@ -8,6 +9,23 @@ export type ShareJsOperation = AnyOperation[]

 export type TrackChangesIdSeeds = { inflight: string; pending: string }

+export interface ShareJsTextType<Snapshot = any, Operation = any> {
+  transformX(op1: Operation, op2: Operation): Operation[]
+  apply(snapshot: Snapshot, op: Operation): Snapshot
+  compose(op1: Operation, op2: Operation): Operation
+
+  api: {
+    insert(pos: number, text: string, fromUndo: boolean): void
+    del(pos: number, length: number, fromUndo: boolean): void
+    getText(): string
+    getLength(): number
+    _register(): void
+  }
+
+  // stub-interface, provided by sharejs.Doc
+  submitOp(op: Operation): void
+}
+
 // TODO: check the properties of this type
 export type Message = {
  v: Version
@@ -16,5 +34,6 @@ export type Message = {
    type?: string
  }
  doc?: string
-  snapshot?: string
+  snapshot?: string | StringFileData
+  type?: ShareJsTextType
 }
@@ -88,6 +88,7 @@ function getSandboxedModuleRequires() {
    'sshpk',
    'xml2js',
    'mongodb',
+    'mongodb-legacy',
  ]
  for (const modulePath of internalModules) {
    requires[Path.resolve(__dirname, modulePath)] = require(modulePath)
@@ -126,6 +126,7 @@ describe('DocumentController', function () {
          projectHistoryType: 'project-history',
          resolvedCommentIds: ['comment2'],
          historyRangesSupport: false,
+          otMigrationStage: 0,
        })
      })
    })