[web] save doc to sessionStorage while editing offline (#35519)

* save doc to session storage when offline
* scope offline doc backup key to user id
* clear offline doc backups on logout
* handle anonymous users
* include trackChanges

GitOrigin-RevId: 3fe6a1523cb10cf76a1fcf4ee46790f6673bd84f

services/web/frontend/js/features/ide-react/editor/document-container.ts

@@ -29,6 +29,7 @@ import {
 import { ThreadId } from '../../../../../types/review-panel/review-panel'
 import getMeta from '@/utils/meta'
 import OError from '@overleaf/o-error'
+import { OfflineDocBackup } from '@/features/ide-react/editor/offline-doc-backup'
 import {
   HistoryOTShareDoc,
   ShareLatexOTShareDoc,
@@ -104,6 +105,7 @@ export class DocumentContainer extends EventEmitter {
   doc?: ShareJsDoc
   cm6?: EditorFacade
   oldInflightOp?: ShareJsOperation
+  private offlineBackup?: OfflineDocBackup
 
   ranges?: _RangesTracker | RangesTrackerWithResolvedThreadIds
 
@@ -607,6 +609,8 @@ export class DocumentContainer extends EventEmitter {
 
     this.detachDoc(this.doc_id, this)
 
+    this.offlineBackup?.destroy()
+
     this.unBindFromEditorEvents()
     this.unBindFromSocketEvents()
   }
@@ -616,6 +620,11 @@ export class DocumentContainer extends EventEmitter {
       return
     }
 
+    this.offlineBackup = new OfflineDocBackup(
+      this.doc,
+      getMeta('ol-project_id')
+    )
+
     this.doc.on('error', (error: Error, meta: ErrorMetadata) =>
       this.onError(error, meta)
     )

services/web/frontend/js/features/ide-react/editor/offline-doc-backup.ts

@@ -0,0 +1,173 @@
+import customSessionStorage from '@/infrastructure/session-storage'
+import { isSplitTestEnabled } from '@/utils/splitTestUtils'
+import getMeta from '@/utils/meta'
+import { ShareJsDoc } from './share-js-doc'
+import { ShareJsOperation } from './types/document'
+
+const EVENT_NAMESPACE = 'offlineBackup'
+const OFFLINE_BACKUP_INTERVAL_MS = 2000
+const KEY_PREFIX = 'doc.offline-backup.'
+
+type Baseline = {
+  version: number
+  snapshot: string | null
+}
+
+export type OfflineDocBackupRecord = {
+  docId: string
+  projectId: string
+  version: number
+  snapshot: string
+  inflightOp: ShareJsOperation | null
+  pendingOp: ShareJsOperation | null
+  trackChanges: boolean
+  updatedAt: number
+  inflightSubmittedIds: string[]
+}
+
+/**
+ * Periodically persists, per open document, the baseline snapshot (V_start)
+ * plus local edits to sessionStorage while the client is offline
+ */
+export class OfflineDocBackup {
+  private readonly enabled: boolean
+  private readonly key: string
+  private baseline: Baseline | null = null
+  private throttleTimer: number | null = null
+
+  static buildKey(projectId: string, docId: string): string {
+    const userId = getMeta('ol-user_id') ?? 'anonymous'
+    return `${KEY_PREFIX}${userId}.${projectId}.${docId}`
+  }
+
+  static read(projectId: string, docId: string): OfflineDocBackupRecord | null {
+    return customSessionStorage.getItem(
+      OfflineDocBackup.buildKey(projectId, docId)
+    )
+  }
+
+  // Sweep every user's records. sessionStorage is per-tab and can't be reached
+  // from the tab that handles logout for any other tab, so we clear the whole
+  // prefix here rather than scoping to the current user.
+  static clearAll(): void {
+    customSessionStorage.removeByPrefix(KEY_PREFIX)
+  }
+
+  constructor(
+    private readonly doc: ShareJsDoc,
+    private readonly projectId: string
+  ) {
+    this.enabled = isSplitTestEnabled('intermittent-connection-improvements')
+    this.key = OfflineDocBackup.buildKey(projectId, doc.doc_id)
+
+    if (!this.enabled) {
+      return
+    }
+
+    this.refreshBaseline()
+
+    this.onSaved = this.onSaved.bind(this)
+    this.onDocChange = this.onDocChange.bind(this)
+
+    this.doc.on(`saved.${EVENT_NAMESPACE}`, this.onSaved)
+    this.doc.on(`change.${EVENT_NAMESPACE}`, this.onDocChange)
+    this.doc.on(`acknowledge.${EVENT_NAMESPACE}`, this.onDocChange)
+    this.doc.on(
+      `flipped_pending_to_inflight.${EVENT_NAMESPACE}`,
+      this.onDocChange
+    )
+  }
+
+  private isOffline() {
+    return this.doc.connection.state !== 'ok'
+  }
+
+  private refreshBaseline() {
+    // baseline is the clean server state, buffered ops would
+    // bake local edits into V_start and corrupt recovery
+    if (this.doc.hasBufferedOps()) {
+      return
+    }
+    this.baseline = {
+      version: this.doc.getVersion(),
+      snapshot: this.serializeSnapshot(),
+    }
+  }
+
+  private serializeSnapshot(): string | null {
+    // TODO: support history-ot
+    if (this.doc.getType() !== 'sharejs-text-ot') {
+      return null
+    }
+    return this.doc.getSnapshot()
+  }
+
+  private onSaved() {
+    this.refreshBaseline()
+    this.clear()
+  }
+
+  private onDocChange() {
+    // While online, keep the baseline in step with the server so V_start
+    // reflects remote ops applied before we go offline. refreshBaseline no-ops
+    // once there are buffered ops, freezing the baseline at the point we start
+    // diverging offline.
+    if (!this.isOffline()) {
+      this.refreshBaseline()
+      return
+    }
+    if (!this.doc.hasBufferedOps()) {
+      return
+    }
+    if (this.throttleTimer !== null) {
+      return
+    }
+    this.throttleTimer = window.setTimeout(() => {
+      this.throttleTimer = null
+      if (this.isOffline() && this.doc.hasBufferedOps()) {
+        this.write()
+      }
+    }, OFFLINE_BACKUP_INTERVAL_MS)
+  }
+
+  private write() {
+    if (!this.baseline || this.baseline.snapshot === null) {
+      return
+    }
+    const record: OfflineDocBackupRecord = {
+      docId: this.doc.doc_id,
+      projectId: this.projectId,
+      version: this.baseline.version,
+      snapshot: this.baseline.snapshot,
+      inflightOp: this.doc.getInflightOp(),
+      pendingOp: this.doc.getPendingOp(),
+      trackChanges: this.doc.track_changes,
+      updatedAt: Date.now(),
+      inflightSubmittedIds: Array.from(this.doc.getInflightSubmittedIds()),
+    }
+    customSessionStorage.setItem(this.key, record)
+  }
+
+  private clear() {
+    customSessionStorage.removeItem(this.key)
+  }
+
+  destroy() {
+    if (!this.enabled) {
+      return
+    }
+    if (this.throttleTimer !== null) {
+      window.clearTimeout(this.throttleTimer)
+      this.throttleTimer = null
+    }
+    this.doc.off(`saved.${EVENT_NAMESPACE}`)
+    this.doc.off(`change.${EVENT_NAMESPACE}`)
+    this.doc.off(`acknowledge.${EVENT_NAMESPACE}`)
+    this.doc.off(`flipped_pending_to_inflight.${EVENT_NAMESPACE}`)
+
+    // Don't clear the stored record here. This also runs when the doc is torn
+    // down after a fatal timeout / out-of-sync error, which is when we need the
+    // backup to survive so it can be recovered on reload. Only `saved` clears
+    // it, once the server confirms our edits are stored.
+  }
+}

services/web/frontend/js/features/ide-react/editor/share-js-doc.ts

@@ -372,6 +372,10 @@ export class ShareJsDoc extends EventEmitter {
     return this._doc.pendingOp
   }
 
+  getInflightSubmittedIds(): Iterable<string> {
+    return this._doc.inflightSubmittedIds
+  }
+
   getRecentAck() {
     // check if we have received an ack recently (within a factor of two of the single user flush delay)
     return (

services/web/frontend/js/infrastructure/session-storage.ts

@@ -42,11 +42,23 @@ const removeItem = function (key: string) {
   return sessionStorage.removeItem(key)
 }
 
+const removeByPrefix = function (prefix: string) {
+  const keys = []
+  for (let i = 0; i < sessionStorage.length; i++) {
+    const key = sessionStorage.key(i)
+    if (key !== null && key.startsWith(prefix)) {
+      keys.push(key)
+    }
+  }
+  keys.forEach(key => sessionStorage.removeItem(key))
+}
+
 const customSessionStorage = {
   getItem: (key: string) => callSafe(getItem, key),
   setItem: (key: string, value: any) => callSafe(setItem, key, value),
   clear: () => callSafe(clear),
   removeItem: (key: string) => callSafe(removeItem, key),
+  removeByPrefix: (prefix: string) => callSafe(removeByPrefix, prefix),
 }
 
 export default customSessionStorage

services/web/frontend/js/shared/components/navbar/account-menu-items.tsx

@@ -8,6 +8,7 @@ import NavDropdownLinkItem from './nav-dropdown-link-item'
 import { useDsNavStyle } from '@/features/project-list/components/use-is-ds-nav'
 import { SignOut } from '@phosphor-icons/react'
 import ThemeToggle from '@/features/project-list/components/sidebar/theme-toggle'
+import { OfflineDocBackup } from '@/features/ide-react/editor/offline-doc-backup'
 
 export function AccountMenuItems({
   sessionUser,
@@ -61,7 +62,12 @@ export function AccountMenuItems({
           <span>{t('log_out')}</span>
           {dsNavStyle && <SignOut size={16} />}
         </OLDropdownItem>
-        <form id={logOutFormId} method="POST" action="/logout">
+        <form
+          id={logOutFormId}
+          method="POST"
+          action="/logout"
+          onSubmit={() => OfflineDocBackup.clearAll()}
+        >
           <input type="hidden" name="_csrf" value={getMeta('ol-csrfToken')} />
         </form>
       </DropdownListItem>

services/web/test/frontend/features/ide-react/unit/editor/offline-doc-backup.test.ts

@@ -0,0 +1,333 @@
+import { expect } from 'chai'
+import sinon from 'sinon'
+import EventEmitter from '@/utils/EventEmitter'
+import {
+  OfflineDocBackup,
+  OfflineDocBackupRecord,
+} from '@/features/ide-react/editor/offline-doc-backup'
+import { ShareJsOperation } from '@/features/ide-react/editor/types/document'
+import { OTType, ShareJsDoc } from '@/features/ide-react/editor/share-js-doc'
+
+const DOC_ID = 'doc-123'
+const PROJECT_ID = 'project-456'
+const USER_ID = 'user-789'
+const KEY = `doc.offline-backup.${USER_ID}.${PROJECT_ID}.${DOC_ID}`
+const FLUSH_DELAY = 2000
+
+class FakeShareJsDoc extends EventEmitter {
+  doc_id = DOC_ID
+  connection = {
+    state: 'ok' as 'ok' | 'disconnected' | 'stopped',
+    id: 'client-1',
+  }
+  version = 5
+  snapshot = 'server text'
+  inflightOp: ShareJsOperation | null = null
+  pendingOp: ShareJsOperation | null = null
+  track_changes = false
+  otType: OTType = 'sharejs-text-ot'
+  _doc = { inflightSubmittedIds: new Set<string>() }
+
+  getVersion() {
+    return this.version
+  }
+
+  getSnapshot() {
+    return this.snapshot
+  }
+
+  getType() {
+    return this.otType
+  }
+
+  hasBufferedOps() {
+    return this.inflightOp != null || this.pendingOp != null
+  }
+
+  getInflightOp() {
+    return this.inflightOp
+  }
+
+  getPendingOp() {
+    return this.pendingOp
+  }
+
+  getInflightSubmittedIds() {
+    return this._doc.inflightSubmittedIds
+  }
+}
+
+function readRecord(): OfflineDocBackupRecord | null {
+  const raw = window.sessionStorage.getItem(KEY)
+  return raw === null ? null : JSON.parse(raw)
+}
+
+describe('OfflineDocBackup', function () {
+  let clock: sinon.SinonFakeTimers
+  let doc: FakeShareJsDoc
+
+  const enableFlag = (enabled = true) => {
+    window.metaAttributesCache.set('ol-splitTestVariants', {
+      'intermittent-connection-improvements': enabled ? 'enabled' : 'default',
+    })
+  }
+
+  const create = () =>
+    new OfflineDocBackup(doc as unknown as ShareJsDoc, PROJECT_ID)
+
+  beforeEach(function () {
+    window.metaAttributesCache = new Map()
+    window.metaAttributesCache.set('ol-user_id', USER_ID)
+    window.sessionStorage.clear()
+    clock = sinon.useFakeTimers()
+    doc = new FakeShareJsDoc()
+    enableFlag()
+  })
+
+  afterEach(function () {
+    clock.restore()
+    window.sessionStorage.clear()
+  })
+
+  const goOffline = () => {
+    doc.connection.state = 'disconnected'
+  }
+
+  it('does nothing when the split test is disabled', function () {
+    enableFlag(false)
+    const backup = create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.be.null
+    backup.destroy()
+  })
+
+  it('does not write while online', function () {
+    create()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.be.null
+  })
+
+  it('does not write when offline with no buffered ops', function () {
+    create()
+    goOffline()
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.be.null
+  })
+
+  it('writes the baseline plus live ops when offline with buffered ops', function () {
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'hello', p: 0 }]
+    doc._doc.inflightSubmittedIds.add('old-client')
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    const record = readRecord()
+    expect(record?.updatedAt).to.be.a('number')
+    expect(record).to.deep.equal({
+      docId: DOC_ID,
+      projectId: PROJECT_ID,
+      version: 5,
+      snapshot: 'server text',
+      inflightOp: null,
+      pendingOp: [{ i: 'hello', p: 0 }],
+      trackChanges: false,
+      updatedAt: record?.updatedAt,
+      inflightSubmittedIds: ['old-client'],
+    })
+  })
+
+  it('persists the track changes state in the record', function () {
+    create()
+    goOffline()
+    doc.track_changes = true
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    expect(readRecord()?.trackChanges).to.be.true
+  })
+
+  it('captures a non-null inflightOp in the persisted record', function () {
+    create()
+    goOffline()
+    doc.inflightOp = [{ i: 'sent', p: 0 }]
+    doc.pendingOp = [{ i: 'queued', p: 5 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    const record = readRecord()
+    expect(record?.inflightOp).to.deep.equal([{ i: 'sent', p: 0 }])
+    expect(record?.pendingOp).to.deep.equal([{ i: 'queued', p: 5 }])
+  })
+
+  it('keeps the baseline frozen at V_start while editing offline', function () {
+    create()
+    goOffline()
+    // The server version/snapshot are only observed again on `saved`; while
+    // offline they must not leak into the stored baseline even if the doc
+    // reports a newer version.
+    doc.version = 99
+    doc.snapshot = 'diverged offline text'
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    const record = readRecord()
+    expect(record?.version).to.equal(5)
+    expect(record?.snapshot).to.equal('server text')
+  })
+
+  it('refreshes the baseline on a remote change received while online and clean', function () {
+    create()
+    // A collaborator's op advances the server version/snapshot while we're
+    // clean and online. ShareJS does not emit `saved` in this case, so the
+    // change event must keep the baseline current.
+    doc.version = 6
+    doc.snapshot = 'server text with remote edit'
+    doc.trigger('change')
+
+    // Now go offline and make a local edit relative to the refreshed snapshot.
+    goOffline()
+    doc.pendingOp = [{ i: 'mine', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    const record = readRecord()
+    expect(record?.version).to.equal(6)
+    expect(record?.snapshot).to.equal('server text with remote edit')
+    expect(record?.pendingOp).to.deep.equal([{ i: 'mine', p: 0 }])
+  })
+
+  it('throttles writes to one per flush delay', function () {
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'a', p: 0 }]
+    doc.trigger('change')
+    doc.pendingOp = [{ i: 'ab', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY - 1)
+    expect(readRecord()).to.be.null
+    clock.tick(1)
+    // trailing write picks up the latest buffered op
+    expect(readRecord()?.pendingOp).to.deep.equal([{ i: 'ab', p: 0 }])
+  })
+
+  it('does not back up a non-sharejs-text-ot doc', function () {
+    doc.otType = 'history-ot'
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.be.null
+  })
+
+  it('refreshes the baseline and clears the key on saved', function () {
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.not.be.null
+
+    // Server acked everything: new clean baseline, key cleared.
+    doc.version = 6
+    doc.snapshot = 'server text updated'
+    doc.pendingOp = null
+    doc.inflightOp = null
+    doc.trigger('saved')
+    expect(readRecord()).to.be.null
+
+    // A later offline edit should record the refreshed baseline.
+    goOffline()
+    doc.pendingOp = [{ i: 'y', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    const record = readRecord()
+    expect(record?.version).to.equal(6)
+    expect(record?.snapshot).to.equal('server text updated')
+  })
+
+  it('destroy() removes listeners but preserves the stored record', function () {
+    const backup = create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.not.be.null
+
+    // Teardown on the error/timeout path must not delete the backup: it is what
+    // recovery reads on reload.
+    backup.destroy()
+    expect(readRecord()).to.not.be.null
+
+    // No further writes after destroy.
+    doc.pendingOp = [{ i: 'xy', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()?.pendingOp).to.deep.equal([{ i: 'x', p: 0 }])
+  })
+
+  it('preserves the backup across a fatal-timeout teardown', function () {
+    const backup = create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+    expect(readRecord()).to.not.be.null
+
+    // Mirror onError(): buffered ops are cleared before teardown runs.
+    doc.pendingOp = null
+    doc.inflightOp = null
+    backup.destroy()
+
+    expect(readRecord()).to.not.be.null
+  })
+
+  it('static read returns the stored record, or null when absent', function () {
+    expect(OfflineDocBackup.read(PROJECT_ID, DOC_ID)).to.be.null
+
+    create()
+    goOffline()
+    doc.pendingOp = [{ i: 'x', p: 0 }]
+    doc.trigger('change')
+    clock.tick(FLUSH_DELAY)
+
+    expect(OfflineDocBackup.read(PROJECT_ID, DOC_ID)?.pendingOp).to.deep.equal([
+      { i: 'x', p: 0 },
+    ])
+  })
+
+  it('clearAll removes every backup key but leaves unrelated keys', function () {
+    window.sessionStorage.setItem(
+      `doc.offline-backup.${USER_ID}.${PROJECT_ID}.${DOC_ID}`,
+      '{}'
+    )
+    window.sessionStorage.setItem(
+      'doc.offline-backup.other-user.other-project.other-doc',
+      '{}'
+    )
+    window.sessionStorage.setItem('unrelated-key', 'keep me')
+
+    OfflineDocBackup.clearAll()
+
+    expect(
+      window.sessionStorage.getItem(
+        `doc.offline-backup.${USER_ID}.${PROJECT_ID}.${DOC_ID}`
+      )
+    ).to.be.null
+    expect(
+      window.sessionStorage.getItem(
+        'doc.offline-backup.other-user.other-project.other-doc'
+      )
+    ).to.be.null
+    expect(window.sessionStorage.getItem('unrelated-key')).to.equal('keep me')
+  })
+})