Merge pull request #25492 from overleaf/em-paginate-changes-1

Paginate history changes endpoint

GitOrigin-RevId: 2b48044d64244404efcd2e090b682c1f571a5567

services/web/app/src/Features/History/HistoryController.js

@@ -465,9 +465,37 @@ async function getLatestHistory(req, res, next) {
 
 async function getChanges(req, res, next) {
   const projectId = req.params.project_id
-  const since = req.query.since
-  const changes = await HistoryManager.promises.getChanges(projectId, { since })
-  res.json(changes)
+  let since = req.query.since
+  // TODO: Transition flag; remove after a while
+  const paginated = req.query.paginated === 'true'
+
+  if (paginated) {
+    const changes = await HistoryManager.promises.getChanges(projectId, {
+      since,
+    })
+    return res.json(changes)
+  } else {
+    // TODO: Remove this code path after a while
+    let hasMore = true
+    const allChanges = []
+    while (hasMore) {
+      const response = await HistoryManager.promises.getChanges(projectId, {
+        since,
+      })
+
+      let changes
+      if (Array.isArray(response)) {
+        changes = response
+        hasMore = false
+      } else {
+        changes = response.changes
+        hasMore = response.hasMore
+        since += changes.length
+      }
+      allChanges.push(...changes)
+    }
+    return res.json(allChanges)
+  }
 }
 
 function isPrematureClose(err) {

services/web/frontend/js/infrastructure/project-snapshot.ts

@@ -124,9 +124,15 @@ export class ProjectSnapshot {
    */
   private async loadChanges() {
     await flushHistory(this.projectId)
-    const changes = await fetchLatestChanges(this.projectId, this.version)
-    this.snapshot.applyAll(changes)
-    this.version += changes.length
+    let hasMore = true
+    while (hasMore) {
+      const response = await fetchLatestChanges(this.projectId, this.version)
+      const changes = response.changes
+      this.snapshot.applyAll(changes)
+      this.version += changes.length
+      hasMore = response.hasMore
+    }
+
     await this.loadDocs()
   }
 
@@ -181,14 +187,43 @@ async function fetchLatestChunk(projectId: string): Promise<Chunk> {
   return Chunk.fromRaw(response.chunk)
 }
 
+type FetchLatestChangesResponse = {
+  changes: Change[]
+  hasMore: boolean
+}
+
+type FetchLatestChangesApiResponse =
+  | RawChange[]
+  | {
+      changes: RawChange[]
+      hasMore: boolean
+    }
+
 async function fetchLatestChanges(
   projectId: string,
   version: number
-): Promise<Change[]> {
-  const response = await getJSON<RawChange[]>(
-    `/project/${projectId}/changes?since=${version}`
+): Promise<FetchLatestChangesResponse> {
+  // TODO: The paginated flag is a transition flag. It can be removed after this
+  // code has been deployed for a few weeks.
+  const response = await getJSON<FetchLatestChangesApiResponse>(
+    `/project/${projectId}/changes?since=${version}&paginated=true`
   )
-  return response.map(Change.fromRaw).filter(change => change != null)
+
+  let changes, hasMore
+  if (Array.isArray(response)) {
+    // deprecated response format is a simple array of changes
+    // TODO: Remove this branch after the transition
+    changes = response
+    hasMore = false
+  } else {
+    changes = response.changes
+    hasMore = response.hasMore
+  }
+
+  return {
+    changes: changes.map(Change.fromRaw).filter(change => change != null),
+    hasMore,
+  }
 }
 
 async function fetchBlob(projectId: string, hash: string): Promise<string> {

services/web/test/frontend/infrastructure/project-snapshot.test.ts

@@ -119,10 +119,14 @@ describe('ProjectSnapshot', function () {
   }
 
   function mockChanges() {
-    fetchMock.getOnce(`/project/${projectId}/changes?since=1`, changes, {
-      name: 'changes-1',
-    })
-    fetchMock.get(`/project/${projectId}/changes?since=2`, [], {
+    fetchMock.getOnce(
+      `/project/${projectId}/changes?since=1&paginated=true`,
+      changes,
+      {
+        name: 'changes-1',
+      }
+    )
+    fetchMock.get(`/project/${projectId}/changes?since=2&paginated=true`, [], {
       name: 'changes-2',
     })
   }