[web] Recover offline doc backup (#35647)

GitOrigin-RevId: 999ecc534821ab6f1bdc1bc4f7757316dab1658d

services/web/frontend/js/features/ide-react/editor/document-container.ts

@@ -29,7 +29,11 @@ import {
 import { ThreadId } from '../../../../../types/review-panel/review-panel'
 import getMeta from '@/utils/meta'
 import OError from '@overleaf/o-error'
-import { OfflineDocBackup } from '@/features/ide-react/editor/offline-doc-backup'
+import {
+  OfflineDocBackup,
+  OfflineDocBackupRecord,
+} from '@/features/ide-react/editor/offline-doc-backup'
+import { isSplitTestEnabled } from '@/utils/splitTestUtils'
 import {
   HistoryOTShareDoc,
   ShareLatexOTShareDoc,
@@ -526,46 +530,149 @@ export class DocumentContainer extends EventEmitter {
         }
       )
     } else {
-      this.socket.emit(
-        'joinDoc',
+      const record = this.getOfflineBackupForRecovery()
+      if (record) {
+        this.recoverAtBaseline(record, callback)
+      } else {
+        this.joinFreshDoc(callback)
+      }
+    }
+  }
+
+  private joinFreshDoc(callback?: JoinCallback) {
+    this.socket.emit(
+      'joinDoc',
+      this.doc_id,
+      { encodeRanges: true, supportsHistoryOT: true },
+      (error, docLines, version, updates, ranges, type, options) => {
+        if (error) {
+          callback?.(error)
+          return
+        }
+        this.createJoinedDoc(docLines, version, ranges, type, options)
+        callback?.()
+      }
+    )
+  }
+
+  private createJoinedDoc(
+    docLines: string[],
+    version: number,
+    ranges: RangesTracker,
+    type: OTType = 'sharejs-text-ot',
+    options: JoinDocResponseOptions = {}
+  ) {
+    this.joined = true
+    this.canSkipLeaveDoc = options.canSkipLeaveDoc ?? false
+    this.doc = new ShareJsDoc(
+      this.doc_id,
+      docLines,
+      version,
+      this.socket,
+      this.globalEditorWatchdogManager,
+      this.ideEventEmitter,
+      type
+    )
+    if (type === 'sharejs-text-ot') {
+      this.decodeRanges(ranges)
+    }
+    this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
+    this.bindToShareJsDocEvents()
+  }
+
+  private getOfflineBackupForRecovery(): OfflineDocBackupRecord | null {
+    if (!isSplitTestEnabled('intermittent-connection-improvements')) {
+      return null
+    }
+    const record = OfflineDocBackup.read(getMeta('ol-project_id'), this.doc_id)
+    // TODO(35594): tracked-change edits need their track-changes state (user id
+    // + seeds) wired up during recovery to replay as tracked; skip them for now
+    // rather than silently recovering them as untracked edits.
+    if (!record || record.trackChanges) {
+      return null
+    }
+    return record
+  }
+
+  // Recover a doc from its offline backup: rebuild it at the backed-up baseline
+  // with the buffered edits restored, then re-join from that version so the
+  // server streams the ops that landed while we were away. catchUp transforms
+  // and dedupes the buffered edits onto the current version, which we then
+  // send. Any failure falls back to a normal join, so a bad backup can never
+  // block opening the doc.
+  private recoverAtBaseline(
+    record: OfflineDocBackupRecord,
+    callback?: JoinCallback
+  ) {
+    const loadWithoutRecovery = (error: unknown) => {
+      // TODO(35594): surface the "changes can't be restored" modal.
+      debugConsole.error('[recovery] failed, loading without recovery', error)
+      OfflineDocBackup.remove(getMeta('ol-project_id'), record.docId)
+      this.doc?.clearInflightAndPendingOps()
+      this.doc = undefined
+      this.joinFreshDoc(callback)
+    }
+
+    try {
+      this.doc = new ShareJsDoc(
         this.doc_id,
-        {
-          encodeRanges: true,
-          supportsHistoryOT: true,
-        },
-        (
-          error,
-          docLines,
-          version,
-          updates,
-          ranges,
-          type: OTType = 'sharejs-text-ot',
-          options: JoinDocResponseOptions = {}
-        ) => {
-          if (error) {
-            callback?.(error)
-            return
-          }
+        [],
+        record.version,
+        this.socket,
+        this.globalEditorWatchdogManager,
+        this.ideEventEmitter,
+        'sharejs-text-ot'
+      )
+
+      this.doc.restoreFromOfflineBackup(record)
+      this.ranges = new RangesTracker()
+      this.bindToShareJsDocEvents()
+    } catch (error) {
+      loadWithoutRecovery(error)
+      return
+    }
+
+    this.socket.emit(
+      'joinDoc',
+      this.doc_id,
+      record.version,
+      { encodeRanges: true, supportsHistoryOT: true },
+      (
+        error,
+        docLines,
+        version,
+        updates,
+        ranges,
+        type = 'sharejs-text-ot',
+        options: JoinDocResponseOptions = {}
+      ) => {
+        // Versioned join failed (e.g. the baseline aged out of the server's op
+        // window); we can't catch up, so load normally.
+        if (error) {
+          loadWithoutRecovery(error)
+          return
+        }
+        // The backup only holds sharejs-text-ot state; if the doc has since
+        // migrated (e.g. to history-ot) we can't replay onto it.
+        if (type !== 'sharejs-text-ot') {
+          loadWithoutRecovery(new Error(`unexpected doc type: ${type}`))
+          return
+        }
+        try {
           this.joined = true
           this.canSkipLeaveDoc = options.canSkipLeaveDoc ?? false
-          this.doc = new ShareJsDoc(
-            this.doc_id,
-            docLines,
-            version,
-            this.socket,
-            this.globalEditorWatchdogManager,
-            this.ideEventEmitter,
-            type
-          )
-          if (type === 'sharejs-text-ot') {
-            this.decodeRanges(ranges)
-          }
-          this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
-          this.bindToShareJsDocEvents()
-          callback?.()
+          // rethrow errors to catch them here and revert to fresh load
+          this.doc!.catchUp(updates, { rethrow: true })
+          this.decodeRanges(ranges)
+          this.catchUpRanges(ranges?.changes, ranges?.comments)
+          this.doc!.sendRecoveredOps()
+        } catch (err) {
+          loadWithoutRecovery(err)
+          return
         }
-      )
-    }
+        callback?.()
+      }
+    )
   }
 
   private decodeRanges(ranges: RangesTracker) {
@@ -633,7 +740,7 @@ export class DocumentContainer extends EventEmitter {
 
     this.detachDoc(this.doc_id, this)
 
-    this.offlineBackup?.destroy()
+    this.offlineBackup?.disconnect()
 
     this.unBindFromEditorEvents()
     this.unBindFromSocketEvents()

services/web/frontend/js/features/ide-react/editor/offline-doc-backup.ts

@@ -46,6 +46,10 @@ export class OfflineDocBackup {
     )
   }
 
+  static remove(projectId: string, docId: string): void {
+    customSessionStorage.removeItem(OfflineDocBackup.buildKey(projectId, docId))
+  }
+
   // Sweep every user's records. sessionStorage is per-tab and can't be reached
   // from the tab that handles logout for any other tab, so we clear the whole
   // prefix here rather than scoping to the current user.
@@ -152,7 +156,7 @@ export class OfflineDocBackup {
     customSessionStorage.removeItem(this.key)
   }
 
-  destroy() {
+  disconnect() {
     if (!this.enabled) {
       return
     }

services/web/frontend/js/features/ide-react/editor/share-js-doc.ts

@@ -30,6 +30,7 @@ import type {
 } from 'overleaf-editor-core/lib/types'
 import { HistoryOTShareDoc } from '../../../../../types/share-doc'
 import { isSplitTestEnabled } from '@/utils/splitTestUtils'
+import { OfflineDocBackupRecord } from './offline-doc-backup'
 
 const intermittentConnectionImprovementsEnabled = isSplitTestEnabled(
   'intermittent-connection-improvements'
@@ -275,7 +276,7 @@ export class ShareJsDoc extends EventEmitter {
 
   // FIXME: This is the original method. Switch back to this when redis
   // issues are resolved.
-  processUpdateFromServer(message: Message) {
+  processUpdateFromServer(message: Message, { rethrow = false } = {}) {
     try {
       if (this.type === 'history-ot' && message.op != null) {
         const shareDoc = this._doc as HistoryOTShareDoc
@@ -304,6 +305,9 @@ export class ShareJsDoc extends EventEmitter {
     } catch (error) {
       // Version mismatches are thrown as errors
       debugConsole.log(error)
+      if (rethrow) {
+        throw error
+      }
       this.handleError(error)
       return error // return the error for queue handling
     }
@@ -313,11 +317,11 @@ export class ShareJsDoc extends EventEmitter {
     }
   }
 
-  catchUp(updates: Message[]) {
+  catchUp(updates: Message[], { rethrow = false } = {}) {
     return updates.map(update => {
       update.v = this._doc.version
       update.doc = this.doc_id
-      return this.processUpdateFromServer(update)
+      return this.processUpdateFromServer(update, { rethrow })
     })
   }
 
@@ -351,6 +355,50 @@ export class ShareJsDoc extends EventEmitter {
     return this._doc.flush()
   }
 
+  // Rebuild the client state captured in an offline backup: the server-acked
+  // baseline snapshot plus the ops that were buffered on top of it. Recovery
+  // constructs a doc at the baseline version, restores this, then catches the
+  // doc up to the current server version.
+  restoreFromOfflineBackup(backup: OfflineDocBackupRecord) {
+    // The snapshot must be the local view (baseline with the buffered ops
+    // applied), matching ShareJS's invariant that snapshot = acked version +
+    // inflight + pending. Seeding only the baseline would show the server text
+    // without the recovered edits.
+    let snapshot = backup.snapshot
+    if (backup.inflightOp) {
+      snapshot = sharejs.types.text.apply(snapshot, backup.inflightOp)
+    }
+    if (backup.pendingOp) {
+      snapshot = sharejs.types.text.apply(snapshot, backup.pendingOp)
+    }
+    this._doc.snapshot = snapshot
+    this._doc.inflightOp = backup.inflightOp
+    this._doc.pendingOp = backup.pendingOp
+    this._doc.inflightSubmittedIds = [...backup.inflightSubmittedIds]
+  }
+
+  // Push the buffered ops of a recovered doc to the server. A doc rebuilt from
+  // a backup never went through submitOp, so no retry timer was armed: resend a
+  // surviving inflight op explicitly (deduped server-side via
+  // inflightSubmittedIds, and re-arming the timer via connection.send); a lone
+  // pending op just needs a flush.
+  sendRecoveredOps() {
+    if (this._doc.inflightOp?.length === 0) {
+      this._doc.inflightOp = null
+      this._doc.inflightSubmittedIds = []
+    }
+    if (this._doc.inflightOp) {
+      this.connection.send({
+        doc: this.doc_id,
+        op: this._doc.inflightOp,
+        v: this._doc.version,
+        dupIfSource: [...this._doc.inflightSubmittedIds],
+      })
+    } else {
+      this.flushPendingOps()
+    }
+  }
+
   updateConnectionState(state: ShareJsConnectionState) {
     debugConsole.log(`[updateConnectionState] Setting state to ${state}`)
     this.connection.state = state

services/web/test/frontend/features/ide-react/unit/editor/document-container-recovery.test.ts

@@ -0,0 +1,232 @@
+import { expect } from 'chai'
+import sinon from 'sinon'
+import { DocumentContainer } from '@/features/ide-react/editor/document-container'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
+
+const DOC_ID = 'doc-1'
+const PROJECT_ID = 'project-1'
+const USER_ID = 'user-1'
+const KEY = `doc.offline-backup.${USER_ID}.${PROJECT_ID}.${DOC_ID}`
+
+type JoinResponder = (fromVersion: number, cb: (...args: any[]) => void) => void
+
+function writeRecord(overrides: Record<string, any> = {}) {
+  const record = {
+    docId: DOC_ID,
+    projectId: PROJECT_ID,
+    version: 5,
+    snapshot: 'hello',
+    inflightOp: null,
+    pendingOp: [{ i: '!', p: 5 }],
+    trackChanges: false,
+    updatedAt: 1,
+    inflightSubmittedIds: [],
+    ...overrides,
+  }
+  window.sessionStorage.setItem(KEY, JSON.stringify(record))
+}
+
+function makeContainer(joinResponder: JoinResponder) {
+  const sent: any[] = []
+  const socket = {
+    socket: { connected: true },
+    publicId: 'client-1',
+    on: sinon.stub(),
+    removeListener: sinon.stub(),
+    emit: sinon.stub().callsFake((event: string, ...args: any[]) => {
+      if (event === 'joinDoc') {
+        const cb = args[args.length - 1]
+        const fromVersion = typeof args[1] === 'number' ? args[1] : -1
+        joinResponder(fromVersion, cb)
+      } else if (event === 'applyOtUpdate') {
+        sent.push(args[1])
+      }
+    }),
+  }
+  const watchdog = new EditorWatchdogManager({ onTimeoutHandler: () => {} })
+  const ideEmitter = { on: sinon.stub(), off: sinon.stub(), emit: sinon.stub() }
+  const container = new DocumentContainer(
+    DOC_ID,
+    socket as unknown as Socket,
+    watchdog,
+    ideEmitter as unknown as IdeEventEmitter,
+    sinon.stub()
+  )
+  return { container, socket, sent }
+}
+
+function joinDocCalls(socket: { emit: sinon.SinonStub }) {
+  return socket.emit.getCalls().filter(call => call.args[0] === 'joinDoc')
+}
+
+// call.args is [event, docId, fromVersion|options, ...]; a versioned join passes
+// fromVersion (a number) there, a full join passes the options object.
+const isVersioned = (call: sinon.SinonSpyCall) =>
+  typeof call.args[2] === 'number'
+
+describe('DocumentContainer offline recovery', function () {
+  let clock: sinon.SinonFakeTimers
+
+  beforeEach(function () {
+    window.metaAttributesCache = new Map()
+    window.metaAttributesCache.set('ol-user_id', USER_ID)
+    window.metaAttributesCache.set('ol-project_id', PROJECT_ID)
+    window.metaAttributesCache.set('ol-splitTestVariants', {
+      'intermittent-connection-improvements': 'enabled',
+    })
+    window.sessionStorage.clear()
+    clock = sinon.useFakeTimers()
+  })
+
+  afterEach(function () {
+    clock.restore()
+    window.sessionStorage.clear()
+  })
+
+  it('replays the buffered edits onto the current server version', function () {
+    // Offline we appended '!' to 'hello'; meanwhile a collaborator inserted 'X'
+    // at the start, taking the server from v5 to v6.
+    writeRecord()
+    const { container, sent } = makeContainer((fromVersion, cb) => {
+      expect(fromVersion).to.equal(5)
+      cb(
+        null,
+        ['Xhello'],
+        6,
+        [{ op: [{ i: 'X', p: 0 }], meta: { source: 'other' } }],
+        { changes: [], comments: [] },
+        'sharejs-text-ot'
+      )
+    })
+
+    container.join()
+
+    // Our append is transformed past the collaborator's insert and sent.
+    expect(sent).to.have.length(1)
+    expect(sent[0].op).to.deep.equal([{ i: '!', p: 6 }])
+    expect(container.getSnapshot()).to.equal('Xhello!')
+  })
+
+  it('loads normally (no recovery) when the split test is disabled', function () {
+    window.metaAttributesCache.set('ol-splitTestVariants', {})
+    writeRecord()
+    const { container, socket, sent } = makeContainer((fromVersion, cb) => {
+      cb(
+        null,
+        ['hello'],
+        5,
+        [],
+        { changes: [], comments: [] },
+        'sharejs-text-ot'
+      )
+    })
+
+    container.join()
+
+    const calls = joinDocCalls(socket)
+    expect(calls).to.have.length(1)
+    expect(isVersioned(calls[0])).to.equal(false)
+    expect(sent).to.have.length(0)
+    expect(container.getSnapshot()).to.equal('hello')
+  })
+
+  it('skips recovery for tracked-change records', function () {
+    writeRecord({ trackChanges: true })
+    const { container, socket, sent } = makeContainer((fromVersion, cb) => {
+      cb(
+        null,
+        ['hello'],
+        5,
+        [],
+        { changes: [], comments: [] },
+        'sharejs-text-ot'
+      )
+    })
+
+    container.join()
+
+    const calls = joinDocCalls(socket)
+    expect(calls).to.have.length(1)
+    expect(isVersioned(calls[0])).to.equal(false)
+    expect(sent).to.have.length(0)
+  })
+
+  it('falls back to a normal load when the server cannot catch us up', function () {
+    writeRecord()
+    const { container, socket, sent } = makeContainer((fromVersion, cb) => {
+      if (fromVersion >= 0) {
+        cb(new Error('OpRangeNotAvailable')) // baseline aged out
+      } else {
+        cb(
+          null,
+          ['head'],
+          9,
+          [],
+          { changes: [], comments: [] },
+          'sharejs-text-ot'
+        )
+      }
+    })
+
+    container.join()
+
+    const calls = joinDocCalls(socket)
+    expect(calls).to.have.length(2)
+    expect(isVersioned(calls[0])).to.equal(true)
+    expect(isVersioned(calls[1])).to.equal(false)
+    expect(sent).to.have.length(0)
+    expect(container.getSnapshot()).to.equal('head')
+  })
+
+  it('falls back to a normal load when the doc is no longer sharejs-text-ot', function () {
+    writeRecord()
+    const { container, socket, sent } = makeContainer((fromVersion, cb) => {
+      if (fromVersion >= 0) {
+        cb(null, [], 6, [], { changes: [], comments: [] }, 'history-ot')
+      } else {
+        cb(
+          null,
+          ['head'],
+          6,
+          [],
+          { changes: [], comments: [] },
+          'sharejs-text-ot'
+        )
+      }
+    })
+
+    container.join()
+
+    const calls = joinDocCalls(socket)
+    expect(calls).to.have.length(2)
+    expect(sent).to.have.length(0)
+    expect(container.getSnapshot()).to.equal('head')
+  })
+
+  it('falls back to a normal load when the backup cannot be applied', function () {
+    // A pending delete that does not match the baseline text throws during seeding.
+    writeRecord({ pendingOp: [{ d: 'zzz', p: 0 }] })
+    const { container, socket, sent } = makeContainer((fromVersion, cb) => {
+      cb(
+        null,
+        ['head'],
+        5,
+        [],
+        { changes: [], comments: [] },
+        'sharejs-text-ot'
+      )
+    })
+
+    container.join()
+
+    // Seeding throws before the versioned join is emitted, so only the fallback
+    // full join runs.
+    const calls = joinDocCalls(socket)
+    expect(calls).to.have.length(1)
+    expect(isVersioned(calls[0])).to.equal(false)
+    expect(sent).to.have.length(0)
+    expect(container.getSnapshot()).to.equal('head')
+  })
+})

services/web/test/frontend/features/ide-react/unit/editor/offline-doc-backup.test.ts

@@ -101,7 +101,7 @@ describe('OfflineDocBackup', function () {
     doc.trigger('change')
     clock.tick(FLUSH_DELAY)
     expect(readRecord()).to.be.null
-    backup.destroy()
+    backup.disconnect()
   })
 
   it('does not write while online', function () {
@@ -255,7 +255,7 @@ describe('OfflineDocBackup', function () {
     expect(record?.snapshot).to.equal('server text updated')
   })
 
-  it('destroy() removes listeners but preserves the stored record', function () {
+  it('disconnect() removes listeners but preserves the stored record', function () {
     const backup = create()
     goOffline()
     doc.pendingOp = [{ i: 'x', p: 0 }]
@@ -265,10 +265,10 @@ describe('OfflineDocBackup', function () {
 
     // Teardown on the error/timeout path must not delete the backup: it is what
     // recovery reads on reload.
-    backup.destroy()
+    backup.disconnect()
     expect(readRecord()).to.not.be.null
 
-    // No further writes after destroy.
+    // No further writes after disconnect.
     doc.pendingOp = [{ i: 'xy', p: 0 }]
     doc.trigger('change')
     clock.tick(FLUSH_DELAY)
@@ -286,7 +286,7 @@ describe('OfflineDocBackup', function () {
     // Mirror onError(): buffered ops are cleared before teardown runs.
     doc.pendingOp = null
     doc.inflightOp = null
-    backup.destroy()
+    backup.disconnect()
 
     expect(readRecord()).to.not.be.null
   })
@@ -305,6 +305,19 @@ describe('OfflineDocBackup', function () {
     ])
   })
 
+  it('static remove deletes the stored record for the given doc', function () {
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.not.be.null
+
+    OfflineDocBackup.remove(PROJECT_ID, DOC_ID)
+
+    expect(readRecord()).to.be.null
+  })
+
   it('clearAll removes every backup key but leaves unrelated keys', function () {
     window.sessionStorage.setItem(
       `doc.offline-backup.${USER_ID}.${PROJECT_ID}.${DOC_ID}`,

services/web/test/frontend/features/ide-react/unit/editor/share-js-doc-recovery.test.ts

@@ -0,0 +1,222 @@
+import { expect } from 'chai'
+import sinon from 'sinon'
+import { ShareJsDoc } from '@/features/ide-react/editor/share-js-doc'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
+import { ShareJsOperation } from '@/features/ide-react/editor/types/document'
+import { OfflineDocBackupRecord } from '@/features/ide-react/editor/offline-doc-backup'
+
+const DOC_ID = 'doc-1'
+
+function makeDoc(version = 5) {
+  const socket = { emit: sinon.stub(), publicId: 'client-1' }
+  const globalWatchdog = new EditorWatchdogManager({
+    onTimeoutHandler: () => {},
+  })
+  const doc = new ShareJsDoc(
+    DOC_ID,
+    [],
+    version,
+    socket as unknown as Socket,
+    globalWatchdog,
+    { emit() {} } as unknown as IdeEventEmitter,
+    'sharejs-text-ot'
+  )
+  return { doc, socket }
+}
+
+function makeBackup(
+  overrides: Partial<OfflineDocBackupRecord> = {}
+): OfflineDocBackupRecord {
+  return {
+    docId: DOC_ID,
+    projectId: 'project-1',
+    version: 5,
+    snapshot: '',
+    inflightOp: null,
+    pendingOp: null,
+    trackChanges: false,
+    updatedAt: 0,
+    inflightSubmittedIds: [],
+    ...overrides,
+  }
+}
+
+describe('ShareJsDoc offline recovery', function () {
+  let clock: sinon.SinonFakeTimers
+
+  beforeEach(function () {
+    window.metaAttributesCache = new Map()
+    clock = sinon.useFakeTimers()
+  })
+
+  afterEach(function () {
+    clock.restore()
+  })
+
+  describe('restoreFromOfflineBackup', function () {
+    it('rebuilds the local view from the baseline plus the buffered ops', function () {
+      const { doc } = makeDoc(5)
+      doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          inflightOp: [{ i: 'X', p: 0 }] as ShareJsOperation,
+          pendingOp: [{ i: 'Y', p: 3 }] as ShareJsOperation,
+          inflightSubmittedIds: ['old-client'],
+        })
+      )
+
+      // baseline 'ab' -> inflight inserts 'X' at 0 -> 'Xab' -> pending inserts
+      // 'Y' at 3 -> 'XabY'
+      expect(doc.getSnapshot()).to.equal('XabY')
+      expect(doc.getVersion()).to.equal(5)
+      expect(doc.getInflightOp()).to.deep.equal([{ i: 'X', p: 0 }])
+      expect(doc.getPendingOp()).to.deep.equal([{ i: 'Y', p: 3 }])
+      expect(Array.from(doc.getInflightSubmittedIds())).to.deep.equal([
+        'old-client',
+      ])
+    })
+
+    it('keeps the baseline snapshot when there are no buffered ops', function () {
+      const { doc } = makeDoc()
+      doc.restoreFromOfflineBackup(makeBackup({ snapshot: 'ab' }))
+
+      expect(doc.getSnapshot()).to.equal('ab')
+      expect(doc.getInflightOp()).to.equal(null)
+      expect(doc.getPendingOp()).to.equal(null)
+    })
+
+    it('applies a lone inflight or pending op', function () {
+      const inflightOnly = makeDoc()
+      inflightOnly.doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          inflightOp: [{ i: 'X', p: 0 }] as ShareJsOperation,
+        })
+      )
+      expect(inflightOnly.doc.getSnapshot()).to.equal('Xab')
+
+      const pendingOnly = makeDoc()
+      pendingOnly.doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          pendingOp: [{ i: 'Y', p: 2 }] as ShareJsOperation,
+        })
+      )
+      expect(pendingOnly.doc.getSnapshot()).to.equal('abY')
+    })
+  })
+
+  describe('sendRecoveredOps', function () {
+    it('resends a surviving inflight op with dupIfSource', function () {
+      const { doc, socket } = makeDoc(5)
+      doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          inflightOp: [{ i: 'X', p: 0 }] as ShareJsOperation,
+          pendingOp: [{ i: 'Y', p: 3 }] as ShareJsOperation,
+          inflightSubmittedIds: ['old-client'],
+        })
+      )
+      socket.emit.resetHistory()
+
+      doc.sendRecoveredOps()
+
+      expect(socket.emit).to.have.been.calledOnce
+      const [event, docId, update] = socket.emit.getCall(0).args
+      expect(event).to.equal('applyOtUpdate')
+      expect(docId).to.equal(DOC_ID)
+      expect(update.op).to.deep.equal([{ i: 'X', p: 0 }])
+      expect(update.v).to.equal(5)
+      expect(update.dupIfSource).to.deep.equal(['old-client'])
+    })
+
+    it('flushes the pending op when nothing is inflight', function () {
+      const { doc, socket } = makeDoc()
+      doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          pendingOp: [{ i: 'Y', p: 2 }] as ShareJsOperation,
+        })
+      )
+      socket.emit.resetHistory()
+
+      doc.sendRecoveredOps()
+
+      expect(socket.emit).to.have.been.calledOnce
+      const [event, , update] = socket.emit.getCall(0).args
+      expect(event).to.equal('applyOtUpdate')
+      expect(update.op).to.deep.equal([{ i: 'Y', p: 2 }])
+    })
+
+    it('drops an inflight op that cancelled out and flushes the pending op instead', function () {
+      const { doc, socket } = makeDoc(5)
+      // An inflight op that transformed away during catch-up is left as an empty
+      // op list rather than null.
+      doc.restoreFromOfflineBackup(
+        makeBackup({
+          snapshot: 'ab',
+          inflightOp: [] as unknown as ShareJsOperation,
+          pendingOp: [{ i: 'Y', p: 2 }] as ShareJsOperation,
+          inflightSubmittedIds: ['old-client'],
+        })
+      )
+      socket.emit.resetHistory()
+
+      doc.sendRecoveredOps()
+
+      // The empty inflight op is never sent (the server rejects zero-length
+      // ops); the pending op is flushed instead.
+      expect(socket.emit).to.have.been.calledOnce
+      const [event, , update] = socket.emit.getCall(0).args
+      expect(event).to.equal('applyOtUpdate')
+      expect(update.op).to.deep.equal([{ i: 'Y', p: 2 }])
+      // The cancelled op's stale dedup ids are cleared so they don't leak onto
+      // the flushed pending op.
+      expect(Array.from(doc.getInflightSubmittedIds())).to.deep.equal([])
+    })
+
+    it('sends nothing when there are no buffered ops', function () {
+      const { doc, socket } = makeDoc()
+      doc.restoreFromOfflineBackup(makeBackup({ snapshot: 'ab' }))
+      socket.emit.resetHistory()
+
+      doc.sendRecoveredOps()
+
+      expect(socket.emit).to.not.have.been.called
+    })
+  })
+
+  describe('catchUp', function () {
+    // A remote delete that doesn't match the snapshot throws when applied; this
+    // stands in for any replay failure during catch-up. catchUp overwrites `v`,
+    // and `meta` just needs to be present so the op isn't mistaken for an ack.
+    const failingUpdate = {
+      v: 0,
+      op: [{ d: 'ZZ', p: 0 }],
+      meta: {},
+    }
+
+    function makeRestoredDoc() {
+      const { doc } = makeDoc(5)
+      doc.restoreFromOfflineBackup(makeBackup({ snapshot: 'ab' }))
+      return doc
+    }
+
+    it('rethrows a replay failure when asked, so recovery can fall back', function () {
+      const doc = makeRestoredDoc()
+
+      expect(() => doc.catchUp([failingUpdate], { rethrow: true })).to.throw()
+    })
+
+    it('reports a replay failure via the error event and swallows it by default', function () {
+      const doc = makeRestoredDoc()
+      const onError = sinon.stub()
+      doc.on('error', onError)
+
+      expect(() => doc.catchUp([failingUpdate])).to.not.throw()
+      expect(onError).to.have.been.called
+    })
+  })
+})