[web] analytics for flaky connections (#35916)

* emit connection-restored events * emit post-offline sync success/failure events GitOrigin-RevId: 3fce5f328d726aeb5edf39f14239ef6e5bda2f99
2026-11-14 14:13:43 +01:00 · 2026-09-17 11:44:19 +00:00
parent f4eb0102ea
commit b53bfbe8a4
9 changed files with 1331 additions and 6 deletions
@@ -3,6 +3,7 @@ import { useLayoutEventTracking } from '@/features/ide-react/hooks/use-layout-ev
 import useSocketListeners from '@/features/ide-react/hooks/use-socket-listeners'
 import { useEditingSessionHeartbeat } from '@/features/ide-react/hooks/use-editing-session-heartbeat'
 import { useRegisterUserActivity } from '@/features/ide-react/hooks/use-register-user-activity'
+import useConnectionOutageTracker from '@/features/ide-react/hooks/use-connection-outage-tracker'
 import { useHasLintingError } from '@/features/ide-react/hooks/use-has-linting-error'
 import { Modals } from '@/features/ide-react/components/modals/modals'
 import { GlobalAlertsProvider } from '@/features/ide-react/context/global-alerts-context'
@@ -21,6 +22,7 @@ export default function IdePage() {
  useSocketListeners() // listen for project-related websocket messages
  useEditingSessionHeartbeat() // send a batched event when user is active
  useRegisterUserActivity() // record activity and ensure connection when user is active
+  useConnectionOutageTracker() // track offline episodes and emit connection-restored
  useHasLintingError() // pass editor:lint hasLintingError to the compiler
  useStatusFavicon() // update the favicon based on the compile status
  useThemedPage() // set the page theme based on user settings
@@ -27,6 +27,7 @@ import {
 import { useModalsContext } from '@/features/ide-react/context/modals-context'
 import { IdeEvents } from '@/features/ide-react/create-ide-event-emitter'
 import { OfflineDocBackup } from '@/features/ide-react/editor/offline-doc-backup'
+import { ConnectionOutageTracker } from '@/features/ide-react/editor/connection-outage-tracker'
 import { useTranslation } from 'react-i18next'
 import customLocalStorage from '@/infrastructure/local-storage'
 import useEventListener from '@/shared/hooks/use-event-listener'
@@ -591,6 +592,7 @@ export const EditorManagerProvider: FC<React.PropsWithChildren> = ({
          projectId,
          document.doc_id
        )
+        ConnectionOutageTracker.recordTeardown(projectId)
        if (backupRecord) {
          if (!isConnected) {
            // Still offline, so nothing has been rejected yet: this is the
@@ -0,0 +1,118 @@
+import customSessionStorage from '@/infrastructure/session-storage'
+import getMeta from '@/utils/meta'
+
+const KEY_PREFIX = 'conn.outage.'
+
+export type OutageResolution = 'reconnected' | 'saved' | 'out-of-sync'
+
+export type ConnectionOutageRecord = {
+  projectId: string
+  detectedAt: number
+  oldestUnsavedOpAt: number | null
+  pendingChars: number
+  inflightChars: number
+  updatedAt: number
+  teardownAt?: number
+}
+
+export class ConnectionOutageTracker {
+  static buildKey(projectId: string): string {
+    const userId = getMeta('ol-user_id') ?? 'anonymous'
+    return `${KEY_PREFIX}${userId}.${projectId}`
+  }
+
+  static read(projectId: string): ConnectionOutageRecord | null {
+    return customSessionStorage.getItem(this.buildKey(projectId))
+  }
+
+  static remove(projectId: string): void {
+    customSessionStorage.removeItem(this.buildKey(projectId))
+  }
+
+  private static write(record: ConnectionOutageRecord): void {
+    customSessionStorage.setItem(this.buildKey(record.projectId), record)
+  }
+
+  // Sweep every user's records on logout. sessionStorage is per-tab and can't
+  // be reached from the tab that handles logout for any other tab, so we clear
+  // the whole prefix here rather than scoping to the current user.
+  static clearAll(): void {
+    customSessionStorage.removeByPrefix(KEY_PREFIX)
+  }
+
+  // Begin an outage. No-op if a record already exists, preserving the original
+  // timestamps (epoch ms) across reloads and repeated stalled ticks.
+  static start(projectId: string, oldestUnsavedOpAt: number | null): void {
+    if (this.read(projectId)) {
+      return
+    }
+    this.write({
+      projectId,
+      detectedAt: Date.now(),
+      oldestUnsavedOpAt,
+      pendingChars: 0,
+      inflightChars: 0,
+      updatedAt: Date.now(),
+    })
+  }
+
+  // Snapshot current unsaved edit sizes while offline. Ops flush on reconnect,
+  // so the live sizes are gone by restore time; persist them here.
+  //
+  // oldestUnsavedOpAt is backfilled the first time an op appears mid-outage:
+  // START runs before the user has typed on an idle drop, so the field is null
+  // then, and unsavedDurationMs would otherwise stay zero even once edits pile up.
+  static recordEdits(
+    projectId: string,
+    pendingChars: number,
+    inflightChars: number,
+    oldestUnsavedOpAt: number | null
+  ): void {
+    const record = this.read(projectId)
+    // A torn-down record is final: its edit sizes belong to the teardown moment,
+    // and its updatedAt has to stop moving so the staleness cap can expire it.
+    // The stall never clears after a teardown, so the snapshot interval keeps
+    // running and would otherwise overwrite both.
+    if (!record || record.teardownAt !== undefined) {
+      return
+    }
+    // Each field only ever grows during a genuine stall, because ops cannot
+    // drain without a working connection. A drop therefore means the recovery
+    // flush has begun and the snapshot is measuring that, not the outage: an ack
+    // nulls the inflight op a flush delay before the next pending op is
+    // promoted, so a snapshot landing in that gap reads zero inflight while
+    // pending is still full.
+    record.pendingChars = Math.max(record.pendingChars, pendingChars)
+    record.inflightChars = Math.max(record.inflightChars, inflightChars)
+    if (record.oldestUnsavedOpAt === null && oldestUnsavedOpAt !== null) {
+      record.oldestUnsavedOpAt = oldestUnsavedOpAt
+    }
+    record.updatedAt = Date.now()
+    this.write(record)
+  }
+
+  // Mark an outage as ended by a terminal out-of-sync teardown. Reporting is
+  // left to the hook, which flushes the record as soon as it sees the teardown
+  // and again on the next load, since a teardown that happens while offline
+  // cannot tell that its report was dropped.
+  //
+  // Upserts, because a doc error unrelated to connectivity gets here before any
+  // stall is detected. Those records have no unsaved op timestamp, which is what
+  // separates them from a genuine outage.
+  static recordTeardown(projectId: string): void {
+    const record = this.read(projectId) ?? {
+      projectId,
+      detectedAt: Date.now(),
+      oldestUnsavedOpAt: null,
+      pendingChars: 0,
+      inflightChars: 0,
+      updatedAt: Date.now(),
+    }
+    if (record.teardownAt !== undefined) {
+      return
+    }
+    record.teardownAt = Date.now()
+    record.updatedAt = Date.now()
+    this.write(record)
+  }
+}
@@ -86,7 +86,7 @@ function getOpSize(op: AnyOperation) {
  return 0
 }

-function getShareJsOpSize(shareJsOp: ShareJsOperation) {
+export function getShareJsOpSize(shareJsOp: ShareJsOperation) {
  return shareJsOp.reduce((total, op) => total + getOpSize(op), 0)
 }

@@ -1,6 +1,9 @@
 // Migrated from static methods of Document in Document.js

-import { DocumentContainer } from '@/features/ide-react/editor/document-container'
+import {
+  DocumentContainer,
+  getShareJsOpSize,
+} from '@/features/ide-react/editor/document-container'
 import { debugConsole } from '@/utils/debugging'
 import { Socket } from '@/features/ide-react/connection/types/socket'
 import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
@@ -45,18 +48,37 @@ export class OpenDocuments {
    const docs = this.unsavedDocs()
    let pendingOpsLength = 0
    let inflightOpsLength = 0
+    let pendingChars = 0
+    let inflightChars = 0
    for (const doc of docs) {
      const pendingOp = doc.getPendingOp()
      const inFlightOp = doc.getInflightOp()
      pendingOpsLength += pendingOp?.length || 0
      inflightOpsLength += inFlightOp?.length || 0
+      pendingChars += pendingOp ? getShareJsOpSize(pendingOp) : 0
+      inflightChars += inFlightOp ? getShareJsOpSize(inFlightOp) : 0
    }
    return {
      pendingOpsLength,
      inflightOpsLength,
+      pendingChars,
+      inflightChars,
    }
  }

+  // performance.now() timeline, matching the unsaved-docs alert
+  getOldestUnsavedOpCreatedAt(): number | null {
+    let oldest: number | null = null
+    for (const doc of this.unsavedDocs()) {
+      const createdAt =
+        doc.getInflightOpCreatedAt() ?? doc.getPendingOpCreatedAt()
+      if (createdAt != null && (oldest === null || createdAt < oldest)) {
+        oldest = createdAt
+      }
+    }
+    return oldest
+  }
+
  private createDoc(docId: string) {
    const doc = new DocumentContainer(
      docId,
@@ -0,0 +1,313 @@
+import { useEffect, useRef } from 'react'
+import { sendMB } from '@/infrastructure/event-tracking'
+import { isSplitTestEnabled } from '@/utils/splitTestUtils'
+import { useIsNetworkStalledState } from '@/features/ide-react/hooks/use-is-network-stalled'
+import {
+  ConnectionOutageTracker,
+  type ConnectionOutageRecord,
+  type OutageResolution,
+} from '@/features/ide-react/editor/connection-outage-tracker'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useProjectContext } from '@/shared/context/project-context'
+import type { IdeEvents } from '@/features/ide-react/create-ide-event-emitter'
+import type { OpenDocuments } from '@/features/ide-react/editor/open-documents'
+
+const OUTAGE_THRESHOLD_MS = 30_000
+const EDIT_SNAPSHOT_INTERVAL_MS = 2_000
+const MAX_RELOAD_RESTORE_MS = 24 * 60 * 60 * 1000
+
+const offlineBackupEnabled = isSplitTestEnabled(
+  'intermittent-connection-improvements'
+)
+
+type OutageProps = {
+  detectedAt: number
+  unsavedDurationMs: number
+  outageDurationMs: number
+  pendingChars: number
+  inflightChars: number
+}
+
+// unsavedDurationMs runs from the oldest unacknowledged local op and is 0 when
+// the outage carried no edits at all. outageDurationMs stays separate so that
+// "long outage but no work at risk" and "long outage with unsaved work" are
+// distinguishable in the data.
+//
+// detectedAt identifies the outage: it is set once and frozen for the record's
+// lifetime, so user + page + detectedAt deduplicates a teardown reported both
+// inline and again on the next load. It is a client clock, so comparing it
+// against server-side event timestamps needs skew tolerance, and for a doc
+// error that never stalled it is the teardown time rather than an outage onset
+// (those rows have an outageDurationMs near zero).
+function readOutageProps(
+  record: ConnectionOutageRecord,
+  endedAt = Date.now()
+): OutageProps {
+  return {
+    detectedAt: record.detectedAt,
+    unsavedDurationMs:
+      record.oldestUnsavedOpAt === null
+        ? 0
+        : Math.floor(endedAt - record.oldestUnsavedOpAt),
+    outageDurationMs: Math.floor(endedAt - record.detectedAt),
+    pendingChars: record.pendingChars,
+    inflightChars: record.inflightChars,
+  }
+}
+
+// Op timestamps are on the performance.now() timeline, but the record survives
+// reloads, so convert to epoch ms.
+function getOldestUnsavedOpAt(openDocs: OpenDocuments): number | null {
+  const oldestOpCreatedAt = openDocs.getOldestUnsavedOpCreatedAt()
+  if (oldestOpCreatedAt === null) {
+    return null
+  }
+  return Date.now() - (performance.now() - oldestOpCreatedAt)
+}
+
+// spannedReload is independent of the resolution: a reload during the outage
+// discards the in-memory op queue, so the work only survived if the offline
+// backup held it, however the outage went on to end.
+function emitOutageEnded(
+  record: ConnectionOutageRecord,
+  {
+    resolution,
+    spannedReload,
+  }: { resolution: OutageResolution; spannedReload: boolean }
+) {
+  const props = readOutageProps(record)
+  if (
+    props.outageDurationMs >= OUTAGE_THRESHOLD_MS ||
+    props.unsavedDurationMs >= OUTAGE_THRESHOLD_MS
+  ) {
+    sendMB('connection-restored', {
+      ...props,
+      resolution,
+      spannedReload,
+      offlineBackupEnabled,
+    })
+  }
+  return props
+}
+
+// Reported whatever the duration: sub-threshold teardowns are doc errors
+// unrelated to connectivity, and separating those out needs them in the data.
+function emitTeardown(record: ConnectionOutageRecord, spannedReload: boolean) {
+  const { teardownAt } = record
+  if (teardownAt === undefined) {
+    return null
+  }
+  const props = readOutageProps(record, teardownAt)
+  sendMB('connection-restored', {
+    ...props,
+    resolution: 'out-of-sync' satisfies OutageResolution,
+    spannedReload,
+    offlineBackupEnabled,
+  })
+  return props
+}
+
+export default function useConnectionOutageTracker(): void {
+  const stalled = useIsNetworkStalledState()
+  const { isConnected, connectionState } = useConnectionContext()
+  const websocketDisconnected =
+    !isConnected || connectionState.reconnectAt !== null
+  const { openDocs } = useEditorManagerContext()
+  const { eventEmitter, outOfSync } = useIdeReactContext()
+  const { projectId } = useProjectContext()
+
+  const prevWebsocketDisconnectedRef = useRef(websocketDisconnected)
+  const prevStalledRef = useRef(stalled)
+  // The outage record is removed as soon as the connection is back, but the
+  // offline edits are only acknowledged (or rejected) some time after that, so
+  // keep the last known outage around to attach to the sync outcome.
+  const lastOutageRef = useRef<OutageProps | null>(null)
+  // The socket starts out CLOSED, so every mount looks disconnected for a beat.
+  // Nothing before the first connection is an outage.
+  const hasConnectedRef = useRef(isConnected)
+  // Set when this load picks up an outage that is still running from a previous
+  // one, so the emit that eventually ends it can report that. Cleared with the
+  // record, so a later outage in the same tab starts from false again.
+  const spannedReloadRef = useRef(false)
+  // The teardown record is deliberately not removed once flushed, so it cannot
+  // double as the "already reported on this load" latch.
+  const teardownFlushedRef = useRef(false)
+
+  // RELOAD survival: Handle recovery after page refresh.
+  //
+  // Declared first so it reads the persisted record before START or SNAPSHOT
+  // can touch it.
+  useEffect(() => {
+    const record = ConnectionOutageTracker.read(projectId)
+    if (!record) return
+
+    // Stale record from before MAX_RELOAD_RESTORE_MS, discard without emitting
+    if (Date.now() - record.updatedAt > MAX_RELOAD_RESTORE_MS) {
+      ConnectionOutageTracker.remove(projectId)
+      return
+    }
+
+    // Ended in a terminal teardown on a previous load. That load may have
+    // flushed it inline already, but had no way to tell whether the beacon
+    // landed, so report it again here and let detectedAt collapse the pair.
+    if (record.teardownAt !== undefined) {
+      lastOutageRef.current = emitTeardown(record, true)
+      ConnectionOutageTracker.remove(projectId)
+      return
+    }
+
+    // Already recovered by mount time, emit and clean up
+    if (!stalled) {
+      lastOutageRef.current = emitOutageEnded(record, {
+        resolution: 'reconnected',
+        spannedReload: true,
+      })
+      ConnectionOutageTracker.remove(projectId)
+      return
+    }
+
+    // Still stalled at mount, seed refs to detect upcoming recovery. This is the
+    // only branch that leaves the outage running, so it is the only one whose
+    // eventual emit has to report that the outage spanned a reload.
+    spannedReloadRef.current = true
+    prevWebsocketDisconnectedRef.current = true
+    prevStalledRef.current = true
+  }, []) // eslint-disable-line react-hooks/exhaustive-deps
+
+  // TEARDOWN: flush a terminal teardown as soon as it happens. The teardown
+  // closes the websocket, but sendMB goes over HTTP, so it still gets through.
+  // Deferring to the next load instead would lose the report whenever the modal
+  // does not force a reload, which the treatment arm's modal never does.
+  //
+  // The record is left in place, because sendMB is fire-and-forget: a teardown
+  // that happens while the machine is genuinely offline has no way of knowing
+  // its beacon was dropped, so the next load has to report it again. That makes
+  // delivery at-least-once, and the duplicate collapses on detectedAt.
+  useEffect(() => {
+    if (!outOfSync || teardownFlushedRef.current) {
+      return
+    }
+    const record = ConnectionOutageTracker.read(projectId)
+    if (!record || record.teardownAt === undefined) {
+      return
+    }
+    teardownFlushedRef.current = true
+    lastOutageRef.current = emitTeardown(record, spannedReloadRef.current)
+  }, [outOfSync, projectId])
+
+  useEffect(() => {
+    if (isConnected) {
+      hasConnectedRef.current = true
+    }
+  }, [isConnected])
+
+  // START: Initialize tracking when stalled.
+  //
+  // Keep websocketDisconnected in the deps: stalled ORs it in, so a drop while
+  // an earlier stall is unresolved leaves stalled true and this would not re-run,
+  // losing the second outage.
+  useEffect(() => {
+    // Bail before reading the docs: that poll mutates per-doc state shared with
+    // the unsaved-docs poll.
+    if (
+      !hasConnectedRef.current ||
+      !stalled ||
+      ConnectionOutageTracker.read(projectId)
+    ) {
+      return
+    }
+    ConnectionOutageTracker.start(projectId, getOldestUnsavedOpAt(openDocs))
+  }, [stalled, websocketDisconnected, isConnected, projectId, openDocs])
+
+  // SNAPSHOT: Record edits while offline
+  useEffect(() => {
+    if (!stalled) return
+
+    const snapshot = () => {
+      const { pendingChars, inflightChars } = openDocs.getUnsavedOpsSize()
+      // No docs are open yet on the first tick after a reload, which reads as
+      // zero unsaved work. Ops cannot drain while stalled, so a zero here is
+      // never news and must not overwrite what a previous load recorded.
+      if (pendingChars === 0 && inflightChars === 0) {
+        return
+      }
+      ConnectionOutageTracker.recordEdits(
+        projectId,
+        pendingChars,
+        inflightChars,
+        getOldestUnsavedOpAt(openDocs)
+      )
+    }
+
+    snapshot()
+    const id = window.setInterval(snapshot, EDIT_SNAPSHOT_INTERVAL_MS)
+
+    return () => {
+      window.clearInterval(id)
+    }
+  }, [stalled, projectId, openDocs])
+
+  // FIRE (a): Emit on websocket reconnect. The edits may not have been
+  // acknowledged yet, hence 'reconnected' rather than 'saved'.
+  useEffect(() => {
+    if (
+      prevWebsocketDisconnectedRef.current === true &&
+      websocketDisconnected === false
+    ) {
+      const record = ConnectionOutageTracker.read(projectId)
+      if (record && record.teardownAt === undefined) {
+        lastOutageRef.current = emitOutageEnded(record, {
+          resolution: 'reconnected',
+          spannedReload: spannedReloadRef.current,
+        })
+        ConnectionOutageTracker.remove(projectId)
+        spannedReloadRef.current = false
+      }
+    }
+    prevWebsocketDisconnectedRef.current = websocketDisconnected
+  }, [websocketDisconnected, projectId])
+
+  // FIRE (b): Emit when the stall clears without the socket having dropped,
+  // which means the ops were finally acknowledged on the existing connection.
+  useEffect(() => {
+    if (prevStalledRef.current === true && stalled === false) {
+      const record = ConnectionOutageTracker.read(projectId)
+      if (record && record.teardownAt === undefined) {
+        lastOutageRef.current = emitOutageEnded(record, {
+          resolution: 'saved',
+          spannedReload: spannedReloadRef.current,
+        })
+        ConnectionOutageTracker.remove(projectId)
+        spannedReloadRef.current = false
+      }
+    }
+    prevStalledRef.current = stalled
+  }, [stalled, projectId])
+
+  // SYNC OUTCOME: Emit whether the offline edits made it back to the server
+  useEffect(() => {
+    const onSynced = (
+      event: CustomEvent<IdeEvents['ide:offlineChangesSynced']>
+    ) => {
+      const [{ docId }] = event.detail
+      sendMB('post-offline-sync-succeeded', { docId })
+    }
+
+    const onFailed = (
+      event: CustomEvent<IdeEvents['ide:unableToSyncOfflineChanges']>
+    ) => {
+      const [{ docId }] = event.detail
+      sendMB('post-offline-sync-failed', { docId })
+    }
+
+    eventEmitter.on('ide:offlineChangesSynced', onSynced)
+    eventEmitter.on('ide:unableToSyncOfflineChanges', onFailed)
+
+    return () => {
+      eventEmitter.off('ide:offlineChangesSynced', onSynced)
+      eventEmitter.off('ide:unableToSyncOfflineChanges', onFailed)
+    }
+  }, [eventEmitter, projectId])
+}
@@ -3,12 +3,17 @@ import { useUnsavedDocsContext } from '@/features/ide-react/context/unsaved-docs
 import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
 import { useConnectionContext } from '../context/connection-context'

-export default function useIsNetworkStalled(): boolean {
-  const enabled = useFeatureFlag('intermittent-connection-improvements')
+export function useIsNetworkStalledState(): boolean {
  const { isSavingStalled } = useUnsavedDocsContext()
  const { outOfSync } = useIdeReactContext()
  const { isConnected, connectionState } = useConnectionContext()
  const websocketDisconnected =
    !isConnected || connectionState.reconnectAt !== null
-  return enabled && (isSavingStalled || outOfSync || websocketDisconnected)
+  return isSavingStalled || outOfSync || websocketDisconnected
+}
+
+export default function useIsNetworkStalled(): boolean {
+  const enabled = useFeatureFlag('intermittent-connection-improvements')
+  const stalled = useIsNetworkStalledState()
+  return enabled && stalled
 }
@@ -9,6 +9,7 @@ import { useDsNavStyle } from '@/features/project-list/components/use-is-ds-nav'
 import { SignOut } from '@phosphor-icons/react'
 import ThemeToggle from '@/features/project-list/components/sidebar/theme-toggle'
 import { OfflineDocBackup } from '@/features/ide-react/editor/offline-doc-backup'
+import { ConnectionOutageTracker } from '@/features/ide-react/editor/connection-outage-tracker'

 export function AccountMenuItems({
  sessionUser,
@@ -66,7 +67,10 @@ export function AccountMenuItems({
          id={logOutFormId}
          method="POST"
          action="/logout"
-          onSubmit={() => OfflineDocBackup.clearAll()}
+          onSubmit={() => {
+            OfflineDocBackup.clearAll()
+            ConnectionOutageTracker.clearAll()
+          }}
        >
          <input type="hidden" name="_csrf" value={getMeta('ol-csrfToken')} />
        </form>