[history-v1] add endpoint for downloading latest zip (#33181)

* [history-v1] add endpoint for downloading latest zip * [web] address review feedback * [web] tests: do not overwrite db.projects.overleaf, extend it * [web] set includeReferer flag from downloading zip GitOrigin-RevId: e63e549f004230086f82eccf03b43fd62bde6071
2026-05-23 17:19:37 +02:00 · 2026-05-13 09:35:03 +02:00
parent b1931d0b3b
commit 7c50dc9990
13 changed files with 319 additions and 63 deletions
--- a/services/history-v1/api/controllers/projects.js
+++ b/services/history-v1/api/controllers/projects.js
@@ -244,6 +244,29 @@ async function getChanges(req, res, next) {
  }
 }

+async function getLatestZip(req, res, next) {
+  const { params } = parseReq(req, schemas.getLatestZip)
+  const projectId = params.project_id
+  const blobStore = new BlobStore(projectId)
+
+  let snapshot
+  try {
+    const chunk = await chunkStore.loadLatest(projectId)
+    snapshot = chunk.getSnapshot()
+    snapshot.applyAll(chunk.getChanges())
+
+    res.setHeader('X-History-Version', chunk.getEndVersion())
+  } catch (err) {
+    if (err instanceof Chunk.NotFoundError) {
+      return render.notFound(res)
+    } else {
+      throw err
+    }
+  }
+
+  await streamZip(snapshot, blobStore, res)
+}
+
 async function getZip(req, res, next) {
  const { params } = parseReq(req, schemas.getZip)
  const projectId = params.project_id
@@ -261,6 +284,10 @@ async function getZip(req, res, next) {
    }
  }

+  await streamZip(snapshot, blobStore, res)
+}
+
+async function streamZip(snapshot, blobStore, res) {
  await withTmpDir('get-zip-', async tmpDir => {
    const tmpFilename = Path.join(tmpDir, 'project.zip')
    const archive = new ProjectArchive(snapshot)
@@ -578,6 +605,7 @@ module.exports = {
  getHistory: expressify(getHistory),
  getHistoryBefore: expressify(getHistoryBefore),
  getChanges: expressify(getChanges),
+  getLatestZip: expressify(getLatestZip),
  getZip: expressify(getZip),
  createZip: expressify(createZip),
  deleteProject: expressify(deleteProject),
--- a/services/history-v1/api/routes/projects.js
+++ b/services/history-v1/api/routes/projects.js
@@ -104,6 +104,12 @@ router.get(
  projectsController.getHistoryBefore
 )

+router.get(
+  '/projects/:project_id/latest/zip',
+  handleTokenAuth,
+  projectsController.getLatestZip
+)
+
 router.get(
  '/projects/:project_id/version/:version/zip',
  handleTokenAuth,
--- a/services/history-v1/api/schema.js
+++ b/services/history-v1/api/schema.js
@@ -201,6 +201,12 @@ const schemas = {
    }),
  }),

+  getLatestZip: z.object({
+    params: z.object({
+      project_id: z.string(),
+    }),
+  }),
+
  getZip: z.object({
    params: z.object({
      project_id: z.string(),
--- a/services/history-v1/test/acceptance/js/api/projects.test.js
+++ b/services/history-v1/test/acceptance/js/api/projects.test.js
@@ -30,6 +30,7 @@ const {
  AddFileOperation,
  EditFileOperation,
  TextOperation,
+  Operation,
 } = require('overleaf-editor-core')
 const testProjects = require('./support/test_projects')
 const { ObjectId } = require('mongodb')
@@ -107,6 +108,79 @@ describe('project controller', function () {
    })
  })

+  describe('getLatestZip', function () {
+    it('returns a zip of the latest snapshot', async function () {
+      const projectId = fixtures.docs.uninitializedProject.id
+
+      const uploadResponse = await fetch(
+        testServer.url(
+          `/api/projects/${projectId}/blobs/${testFiles.HELLO_TXT_HASH}`
+        ),
+        {
+          method: 'PUT',
+          body: fs.createReadStream(testFiles.path('hello.txt')),
+          headers: { Authorization: testServer.basicAuthHeader },
+        }
+      )
+      expect(uploadResponse.ok).to.be.true
+
+      const snapshot = new Snapshot()
+      snapshot.addFile('hello.txt', File.fromHash(testFiles.HELLO_TXT_HASH))
+      const importResponse =
+        await testServer.basicAuthClient.apis.ProjectImport.importSnapshot1({
+          project_id: projectId,
+          snapshot: snapshot.toRaw(),
+        })
+      expect(importResponse.obj.projectId).to.equal(projectId)
+
+      const downloadClient =
+        await testServer.createClientForDownloadZip(projectId)
+      const zipResponse = await downloadClient.apis.Project.getLatestZip({
+        project_id: projectId,
+      })
+      expect(zipResponse.status).to.equal(HTTPStatus.OK)
+      expect(zipResponse.headers['x-history-version']).to.equal('0')
+      expect(zipResponse.headers['content-type']).to.equal(
+        'application/octet-stream'
+      )
+      expect(zipResponse.headers['content-disposition']).to.equal(
+        'attachment; filename=project.zip'
+      )
+
+      const testFile = File.fromHash(testFiles.HELLO_TXT_HASH)
+      const testChange = new Change(
+        [Operation.addFile('main.tex', testFile)],
+        new Date()
+      )
+      const importchangesResponse =
+        await testServer.basicAuthClient.apis.ProjectImport.importChanges1({
+          project_id: projectId,
+          end_version: 0,
+          changes: [testChange.toRaw()],
+        })
+      expect(importchangesResponse.status).to.equal(HTTPStatus.CREATED)
+      expect(importchangesResponse.obj).to.deep.equal({ resyncNeeded: false })
+
+      const zipResponse2 = await downloadClient.apis.Project.getLatestZip({
+        project_id: projectId,
+      })
+      expect(zipResponse2.status).to.equal(HTTPStatus.OK)
+      expect(zipResponse2.headers['x-history-version']).to.equal('1')
+    })
+
+    it('returns 404 for an unknown project', async function () {
+      const unknownProjectId = new ObjectId().toString()
+      const downloadClient =
+        await testServer.createClientForDownloadZip(unknownProjectId)
+      await expectHttpError(
+        downloadClient.apis.Project.getLatestZip({
+          project_id: unknownProjectId,
+        }),
+        HTTPStatus.NOT_FOUND
+      )
+    })
+  })
+
  describe('blob stats', function () {
    let populatedPostgresProjectId,
      populatedMongoProjectId,
--- a/services/history-v1/test/acceptance/js/api/support/http_client.js
+++ b/services/history-v1/test/acceptance/js/api/support/http_client.js
@@ -391,6 +391,8 @@ function createHttpClient(baseUrl, options = {}) {
            `/api/projects/:project_id/timestamp/:timestamp/history`,
            params
          ),
+        getLatestZip: params =>
+          makeRequest('GET', `/api/projects/:project_id/latest/zip`, params),
        getZip: params =>
          makeRequest(
            'GET',
--- a/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs
+++ b/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs
@@ -10,6 +10,7 @@ import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs'
 import Validation from '../../infrastructure/Validation.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import { pipeline } from 'node:stream/promises'
+import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'

 const { z, zz, parseReq } = Validation

@@ -142,7 +143,7 @@ export default {
      }
      ProjectGetter.getProject(
        projectId,
-        { name: true },
+        { name: true, 'overleaf.history.id': true },
        function (error, project) {
          if (error) {
            return next(error)
@@ -153,14 +154,33 @@ export default {
            userId,
            req.ip
          )
-          ProjectZipStreamManager.createZipStreamForProject(
-            projectId,
-            function (error, stream) {
+          SplitTestHandler.featureFlagEnabled(
+            req,
+            res,
+            'zip-from-history',
+            { includeReferer: true },
+            function (error, enabled) {
              if (error) {
                return next(error)
              }
-              prepareZipAttachment(res, `${getSafeProjectName(project)}.zip`)
-              stream.pipe(res)
+              ProjectZipStreamManager.createZipStreamForProject(
+                projectId,
+                enabled,
+                project.overleaf.history.id,
+                function (error, stream, historyVersion) {
+                  if (error) {
+                    return next(error)
+                  }
+                  prepareZipAttachment(
+                    res,
+                    `${getSafeProjectName(project)}.zip`
+                  )
+                  if (historyVersion != null) {
+                    res.setHeader('X-History-Version', historyVersion)
+                  }
+                  stream.pipe(res)
+                }
+              )
            }
          )
        }
@@ -187,17 +207,29 @@ export default {
            req.ip
          )
        }
-        ProjectZipStreamManager.createZipStreamForMultipleProjects(
-          projectIds,
-          function (error, stream) {
+        SplitTestHandler.featureFlagEnabled(
+          req,
+          res,
+          'zip-from-history',
+          { includeReferer: true },
+          function (error, enabled) {
            if (error) {
              return next(error)
            }
-            prepareZipAttachment(
-              res,
-              `Overleaf Projects (${projectIds.length} items).zip`
+            ProjectZipStreamManager.createZipStreamForMultipleProjects(
+              projectIds,
+              enabled,
+              function (error, stream) {
+                if (error) {
+                  return next(error)
+                }
+                prepareZipAttachment(
+                  res,
+                  `Overleaf Projects (${projectIds.length} items).zip`
+                )
+                stream.pipe(res)
+              }
            )
-            stream.pipe(res)
          }
        )
      }
--- a/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs
+++ b/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs
@@ -4,10 +4,11 @@ import logger from '@overleaf/logger'
 import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import HistoryManager from '../History/HistoryManager.mjs'
+import Metrics from '@overleaf/metrics'
 let ProjectZipStreamManager

 export default ProjectZipStreamManager = {
-  createZipStreamForMultipleProjects(projectIds, callback) {
+  createZipStreamForMultipleProjects(projectIds, zipFromHistory, callback) {
    // We'll build up a zip file that contains multiple zip files
    const archive = archiver('zip')
    archive.on('error', err =>
@@ -19,38 +20,44 @@ export default ProjectZipStreamManager = {
    callback(null, archive)

    const jobs = projectIds.map(projectId => cb => {
-      ProjectGetter.getProject(projectId, { name: true }, (error, project) => {
-        if (error) {
-          return cb(error)
-        }
-        if (!project) {
-          logger.debug(
-            { projectId },
-            'cannot append project to zip stream: project not found'
-          )
-          return cb()
-        }
-        logger.debug(
-          { projectId, name: project.name },
-          'appending project to zip stream'
-        )
-        ProjectZipStreamManager.createZipStreamForProject(
-          projectId,
-          (error, stream) => {
-            if (error) {
-              return cb(error)
-            }
-            archive.append(stream, { name: `${project.name}.zip` })
-            stream.on('end', () => {
-              logger.debug(
-                { projectId, name: project.name },
-                'zip stream ended'
-              )
-              cb()
-            })
+      ProjectGetter.getProject(
+        projectId,
+        { name: true, 'overleaf.history.id': true },
+        (error, project) => {
+          if (error) {
+            return cb(error)
          }
-        )
-      })
+          if (!project) {
+            logger.debug(
+              { projectId },
+              'cannot append project to zip stream: project not found'
+            )
+            return cb()
+          }
+          logger.debug(
+            { projectId, name: project.name },
+            'appending project to zip stream'
+          )
+          ProjectZipStreamManager.createZipStreamForProject(
+            projectId,
+            zipFromHistory,
+            project.overleaf.history.id,
+            (error, stream) => {
+              if (error) {
+                return cb(error)
+              }
+              archive.append(stream, { name: `${project.name}.zip` })
+              stream.on('end', () => {
+                logger.debug(
+                  { projectId, name: project.name },
+                  'zip stream ended'
+                )
+                cb()
+              })
+            }
+          )
+        }
+      )
    })

    async.series(jobs, () => {
@@ -62,7 +69,16 @@ export default ProjectZipStreamManager = {
    })
  },

-  createZipStreamForProject(projectId, callback) {
+  createZipStreamForProject(projectId, zipFromHistory, historyId, callback) {
+    Metrics.inc('project_zip_download', 1, {
+      method: zipFromHistory ? 'history-v1' : 'web',
+    })
+    if (zipFromHistory) {
+      return HistoryManager.flushProject(projectId, error => {
+        if (error) return callback(error)
+        HistoryManager.getLatestZipWithHistoryId(historyId, callback)
+      })
+    }
    const archive = archiver('zip')
    // return stream immediately before we start adding things to it
    archive.on('error', err =>
--- a/services/web/app/src/Features/History/HistoryManager.mjs
+++ b/services/web/app/src/Features/History/HistoryManager.mjs
@@ -1,4 +1,4 @@
-import { callbackify } from 'node:util'
+import { callbackify, callbackifyMultiResult } from '@overleaf/promise-utils'
 import {
  fetchJson,
  fetchNothing,
@@ -303,6 +303,22 @@ async function getLatestHistoryWithHistoryId(historyId) {
  )
 }

+/**
+ * Get the latest chunk from history using already resolved historyId
+ *
+ * @param {string} historyId
+ */
+async function getLatestZipWithHistoryId(historyId) {
+  const { response, stream } = await fetchStreamWithResponse(
+    `${HISTORY_V1_URL}/projects/${historyId}/latest/zip`,
+    {
+      basicAuth: HISTORY_V1_BASIC_AUTH,
+      signal: AbortSignal.timeout(10 * 60 * 1000),
+    }
+  )
+  return { stream, historyVersion: response.headers.get('X-History-Version') }
+}
+
 async function ensureNoResyncPending(projectId) {
  const { resyncPending } = await fetchJson(
    `${settings.apis.project_history.url}/project/${projectId}/resync-pending`
@@ -475,6 +491,10 @@ export default {
  requestBlob: callbackify(requestBlob),
  requestBlobWithProjectId: callbackify(requestBlobWithProjectId),
  getLatestHistory: callbackify(getLatestHistory),
+  getLatestZipWithHistoryId: callbackifyMultiResult(getLatestZipWithHistoryId, [
+    'stream',
+    'historyVersion',
+  ]),
  getChanges: callbackify(getChanges),
  promises: {
    initializeProject,
@@ -491,6 +511,7 @@ export default {
    requestBlob,
    requestBlobWithProjectId,
    getLatestHistory,
+    getLatestZipWithHistoryId,
    getChanges,
    getChangesWithHistoryId,
    getProjectBlobStats,
--- a/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs
+++ b/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs
@@ -333,6 +333,30 @@ async function getOneTimeAssignment(splitTestName) {
  }
 }

+/**
+ * Checks if a feature flag is enabled for a specific user
+ *
+ * Retrieves the feature flag assignment for a user and determines if the assigned variant is 'enabled'
+ *
+ * @param req the request
+ * @param res the Express response object
+ * @param {string} splitTestName - The unique name of the feature flag
+ * @param {Object} options
+ * @param {boolean} options.includeReferer For ajax requests and downloads include the split test overrides of the page
+ * @returns {Promise<boolean>} True if the user's assigned variant is 'enabled', false otherwise
+ */
+async function featureFlagEnabled(
+  req,
+  res,
+  splitTestName,
+  { includeReferer = false } = { includeReferer: false }
+) {
+  const { variant } = await getAssignment(req, res, splitTestName, {
+    includeReferer,
+  })
+  return variant === 'enabled'
+}
+
 /**
 * Checks if a feature flag is enabled for a specific user
 *
@@ -997,6 +1021,7 @@ export default {
  getPercentile,
  getAssignment: callbackify(getAssignment),
  getAssignmentForUser: callbackify(getAssignmentForUser),
+  featureFlagEnabled: callbackify(featureFlagEnabled),
  featureFlagEnabledForUser: callbackify(featureFlagEnabledForUser),
  getOneTimeAssignment: callbackify(getOneTimeAssignment),
  getActiveAssignmentsForUser: callbackify(getActiveAssignmentsForUser),
@@ -1006,6 +1031,7 @@ export default {
  promises: {
    getAssignment,
    getAssignmentForUser,
+    featureFlagEnabled,
    featureFlagEnabledForUser,
    getOneTimeAssignment,
    getActiveAssignmentsForUser,
--- a/services/web/scripts/e2e_test_setup.mjs
+++ b/services/web/scripts/e2e_test_setup.mjs
@@ -163,6 +163,26 @@ export async function provisionSplitTests(merge = false, extraSplitTests = []) {
      'utf-8'
    )
  )
+  // Add WIP split test, we can update the JSON blob once this is in production
+  SPLIT_TESTS.push({
+    name: 'zip-from-history',
+    versions: [
+      {
+        versionNumber: 1,
+        createdAt: '2026-02-25T14:55:31.260Z',
+        active: true,
+        analyticsEnabled: false,
+        phase: 'release',
+        variants: [
+          {
+            name: 'enabled',
+            rolloutPercent: 0,
+            rolloutStripes: [],
+          },
+        ],
+      },
+    ],
+  })
  console.log(`> Importing ${SPLIT_TESTS.length} split-tests from production.`)
  if (merge) {
    await SplitTestManager.mergeSplitTests(SPLIT_TESTS, false)
--- a/services/web/test/acceptance/src/TokenAccessTests.mjs
+++ b/services/web/test/acceptance/src/TokenAccessTests.mjs
@@ -1112,11 +1112,7 @@ describe('TokenAccess', function () {
            this.owner.makePrivate(this.projectId, () => {
              db.projects.updateOne(
                { _id: project._id },
-                {
-                  $set: {
-                    overleaf: { id: 1234 },
-                  },
-                },
+                { $set: { 'overleaf.id': 1234 } },
                err => {
                  expect(err).not.to.exist
                  done()
--- a/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs
+++ b/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs
@@ -73,6 +73,14 @@ describe('ProjectDownloadsController', function () {
        }),
      })
    )
+    vi.doMock(
+      '../../../../app/src/Features/SplitTests/SplitTestHandler.mjs',
+      () => ({
+        default: (ctx.SplitTestHandler = {
+          featureFlagEnabled: sinon.stub().yields(null, false),
+        }),
+      })
+    )

    vi.doMock('@overleaf/settings', () => ({
      default: (ctx.Settings = {
@@ -92,7 +100,7 @@ describe('ProjectDownloadsController', function () {
      ctx.stream = { pipe: sinon.stub() }
      ctx.ProjectZipStreamManager.createZipStreamForProject = sinon
        .stub()
-        .callsArgWith(1, null, ctx.stream)
+        .yields(null, ctx.stream)
      ctx.req.params = { Project_id: ctx.project_id }
      ctx.req.ip = '192.168.1.1'
      ctx.req.session = {
@@ -102,9 +110,10 @@ describe('ProjectDownloadsController', function () {
        },
      }
      ctx.project_name = 'project name with accênts and % special characters'
-      ctx.ProjectGetter.getProject = sinon
-        .stub()
-        .callsArgWith(2, null, { name: ctx.project_name })
+      ctx.ProjectGetter.getProject = sinon.stub().callsArgWith(2, null, {
+        name: ctx.project_name,
+        overleaf: { history: { id: 123 } },
+      })
      ctx.DocumentUpdaterHandler.flushProjectToMongo = sinon
        .stub()
        .callsArgWith(1)
@@ -138,7 +147,7 @@ describe('ProjectDownloadsController', function () {

    it("should look up the project's name", function (ctx) {
      return ctx.ProjectGetter.getProject
-        .calledWith(ctx.project_id, { name: true })
+        .calledWith(ctx.project_id, { name: true, 'overleaf.history.id': true })
        .should.equal(true)
    })

@@ -172,7 +181,7 @@ describe('ProjectDownloadsController', function () {
      ctx.stream = { pipe: sinon.stub() }
      ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects = sinon
        .stub()
-        .callsArgWith(1, null, ctx.stream)
+        .yields(null, ctx.stream)
      ctx.project_ids = ['project-1', 'project-2']
      ctx.req.query = { project_ids: ctx.project_ids.join(',') }
      ctx.req.ip = '192.168.1.1'
--- a/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs
+++ b/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs
@@ -46,7 +46,9 @@ describe('ProjectZipStreamManager', function () {
    vi.doMock(
      '../../../../app/src/Features/History/HistoryManager.mjs',
      () => ({
-        default: (ctx.HistoryManager = {}),
+        default: (ctx.HistoryManager = {
+          flushProject: sinon.stub().yields(null),
+        }),
      })
    )

@@ -81,6 +83,8 @@ describe('ProjectZipStreamManager', function () {

          ctx.ProjectZipStreamManager.createZipStreamForProject = (
            projectId,
+            zipFromHistory,
+            historyId,
            callback
          ) => {
            callback(null, ctx.zip_streams[projectId])
@@ -92,12 +96,16 @@ describe('ProjectZipStreamManager', function () {
          sinon.spy(ctx.ProjectZipStreamManager, 'createZipStreamForProject')

          ctx.ProjectGetter.getProject = (projectId, fields, callback) => {
-            return callback(null, { name: ctx.project_names[projectId] })
+            return callback(null, {
+              name: ctx.project_names[projectId],
+              overleaf: { history: { id: 123 } },
+            })
          }
          sinon.spy(ctx.ProjectGetter, 'getProject')

          ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects(
            ctx.project_ids,
+            false,
            (...args) => {
              return ctx.callback(...Array.from(args || []))
            }
@@ -131,7 +139,7 @@ describe('ProjectZipStreamManager', function () {
      it('should get the names of each project', function (ctx) {
        return Array.from(ctx.project_ids).map(projectId =>
          ctx.ProjectGetter.getProject
-            .calledWith(projectId, { name: true })
+            .calledWith(projectId, { name: true, 'overleaf.history.id': true })
            .should.equal(true)
        )
      })
@@ -160,6 +168,8 @@ describe('ProjectZipStreamManager', function () {

          ctx.ProjectZipStreamManager.createZipStreamForProject = (
            projectId,
+            zipFromHistory,
+            historyId,
            callback
          ) => {
            callback(null, ctx.zip_streams[projectId])
@@ -171,12 +181,16 @@ describe('ProjectZipStreamManager', function () {

          ctx.ProjectGetter.getProject = (projectId, fields, callback) => {
            const name = ctx.project_names[projectId]
-            callback(null, name ? { name } : undefined)
+            callback(
+              null,
+              name ? { name, overleaf: { history: { id: 123 } } } : undefined
+            )
          }
          sinon.spy(ctx.ProjectGetter, 'getProject')

          ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects(
            ctx.project_ids,
+            false,
            ctx.callback
          )

@@ -200,7 +214,7 @@ describe('ProjectZipStreamManager', function () {
      it('should get the names of each project', function (ctx) {
        ctx.project_ids.map(projectId =>
          ctx.ProjectGetter.getProject
-            .calledWith(projectId, { name: true })
+            .calledWith(projectId, { name: true, 'overleaf.history.id': true })
            .should.equal(true)
        )
      })
@@ -237,6 +251,8 @@ describe('ProjectZipStreamManager', function () {
        ctx.archive.finalize = sinon.stub()
        return ctx.ProjectZipStreamManager.createZipStreamForProject(
          ctx.project_id,
+          false,
+          123,
          ctx.callback
        )
      })
@@ -285,6 +301,8 @@ describe('ProjectZipStreamManager', function () {
        ctx.archive.finalize = sinon.stub()
        ctx.ProjectZipStreamManager.createZipStreamForProject(
          ctx.project_id,
+          false,
+          123,
          ctx.callback
        )
      })
@@ -317,6 +335,8 @@ describe('ProjectZipStreamManager', function () {
        ctx.archive.finalize = sinon.stub()
        return ctx.ProjectZipStreamManager.createZipStreamForProject(
          ctx.project_id,
+          false,
+          123,
          ctx.callback
        )
      })