diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js index 9f2005d2a8..040970faea 100644 --- a/services/history-v1/api/controllers/projects.js +++ b/services/history-v1/api/controllers/projects.js @@ -244,6 +244,29 @@ async function getChanges(req, res, next) { } } +async function getLatestZip(req, res, next) { + const { params } = parseReq(req, schemas.getLatestZip) + const projectId = params.project_id + const blobStore = new BlobStore(projectId) + + let snapshot + try { + const chunk = await chunkStore.loadLatest(projectId) + snapshot = chunk.getSnapshot() + snapshot.applyAll(chunk.getChanges()) + + res.setHeader('X-History-Version', chunk.getEndVersion()) + } catch (err) { + if (err instanceof Chunk.NotFoundError) { + return render.notFound(res) + } else { + throw err + } + } + + await streamZip(snapshot, blobStore, res) +} + async function getZip(req, res, next) { const { params } = parseReq(req, schemas.getZip) const projectId = params.project_id @@ -261,6 +284,10 @@ async function getZip(req, res, next) { } } + await streamZip(snapshot, blobStore, res) +} + +async function streamZip(snapshot, blobStore, res) { await withTmpDir('get-zip-', async tmpDir => { const tmpFilename = Path.join(tmpDir, 'project.zip') const archive = new ProjectArchive(snapshot) @@ -578,6 +605,7 @@ module.exports = { getHistory: expressify(getHistory), getHistoryBefore: expressify(getHistoryBefore), getChanges: expressify(getChanges), + getLatestZip: expressify(getLatestZip), getZip: expressify(getZip), createZip: expressify(createZip), deleteProject: expressify(deleteProject), diff --git a/services/history-v1/api/routes/projects.js b/services/history-v1/api/routes/projects.js index 9f81b4c54e..85e91156ac 100644 --- a/services/history-v1/api/routes/projects.js +++ b/services/history-v1/api/routes/projects.js @@ -104,6 +104,12 @@ router.get( projectsController.getHistoryBefore ) +router.get( + '/projects/:project_id/latest/zip', + handleTokenAuth, + projectsController.getLatestZip +) + router.get( '/projects/:project_id/version/:version/zip', handleTokenAuth, diff --git a/services/history-v1/api/schema.js b/services/history-v1/api/schema.js index e8f826834e..797a68c96f 100644 --- a/services/history-v1/api/schema.js +++ b/services/history-v1/api/schema.js @@ -201,6 +201,12 @@ const schemas = { }), }), + getLatestZip: z.object({ + params: z.object({ + project_id: z.string(), + }), + }), + getZip: z.object({ params: z.object({ project_id: z.string(), diff --git a/services/history-v1/test/acceptance/js/api/projects.test.js b/services/history-v1/test/acceptance/js/api/projects.test.js index 578e6af94d..659a70ece3 100644 --- a/services/history-v1/test/acceptance/js/api/projects.test.js +++ b/services/history-v1/test/acceptance/js/api/projects.test.js @@ -30,6 +30,7 @@ const { AddFileOperation, EditFileOperation, TextOperation, + Operation, } = require('overleaf-editor-core') const testProjects = require('./support/test_projects') const { ObjectId } = require('mongodb') @@ -107,6 +108,79 @@ describe('project controller', function () { }) }) + describe('getLatestZip', function () { + it('returns a zip of the latest snapshot', async function () { + const projectId = fixtures.docs.uninitializedProject.id + + const uploadResponse = await fetch( + testServer.url( + `/api/projects/${projectId}/blobs/${testFiles.HELLO_TXT_HASH}` + ), + { + method: 'PUT', + body: fs.createReadStream(testFiles.path('hello.txt')), + headers: { Authorization: testServer.basicAuthHeader }, + } + ) + expect(uploadResponse.ok).to.be.true + + const snapshot = new Snapshot() + snapshot.addFile('hello.txt', File.fromHash(testFiles.HELLO_TXT_HASH)) + const importResponse = + await testServer.basicAuthClient.apis.ProjectImport.importSnapshot1({ + project_id: projectId, + snapshot: snapshot.toRaw(), + }) + expect(importResponse.obj.projectId).to.equal(projectId) + + const downloadClient = + await testServer.createClientForDownloadZip(projectId) + const zipResponse = await downloadClient.apis.Project.getLatestZip({ + project_id: projectId, + }) + expect(zipResponse.status).to.equal(HTTPStatus.OK) + expect(zipResponse.headers['x-history-version']).to.equal('0') + expect(zipResponse.headers['content-type']).to.equal( + 'application/octet-stream' + ) + expect(zipResponse.headers['content-disposition']).to.equal( + 'attachment; filename=project.zip' + ) + + const testFile = File.fromHash(testFiles.HELLO_TXT_HASH) + const testChange = new Change( + [Operation.addFile('main.tex', testFile)], + new Date() + ) + const importchangesResponse = + await testServer.basicAuthClient.apis.ProjectImport.importChanges1({ + project_id: projectId, + end_version: 0, + changes: [testChange.toRaw()], + }) + expect(importchangesResponse.status).to.equal(HTTPStatus.CREATED) + expect(importchangesResponse.obj).to.deep.equal({ resyncNeeded: false }) + + const zipResponse2 = await downloadClient.apis.Project.getLatestZip({ + project_id: projectId, + }) + expect(zipResponse2.status).to.equal(HTTPStatus.OK) + expect(zipResponse2.headers['x-history-version']).to.equal('1') + }) + + it('returns 404 for an unknown project', async function () { + const unknownProjectId = new ObjectId().toString() + const downloadClient = + await testServer.createClientForDownloadZip(unknownProjectId) + await expectHttpError( + downloadClient.apis.Project.getLatestZip({ + project_id: unknownProjectId, + }), + HTTPStatus.NOT_FOUND + ) + }) + }) + describe('blob stats', function () { let populatedPostgresProjectId, populatedMongoProjectId, diff --git a/services/history-v1/test/acceptance/js/api/support/http_client.js b/services/history-v1/test/acceptance/js/api/support/http_client.js index 166cb7c0af..7b8ed2f7df 100644 --- a/services/history-v1/test/acceptance/js/api/support/http_client.js +++ b/services/history-v1/test/acceptance/js/api/support/http_client.js @@ -391,6 +391,8 @@ function createHttpClient(baseUrl, options = {}) { `/api/projects/:project_id/timestamp/:timestamp/history`, params ), + getLatestZip: params => + makeRequest('GET', `/api/projects/:project_id/latest/zip`, params), getZip: params => makeRequest( 'GET', diff --git a/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs b/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs index f8a964e45e..7f240e8b5e 100644 --- a/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs +++ b/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs @@ -10,6 +10,7 @@ import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs' import Validation from '../../infrastructure/Validation.mjs' import { expressify } from '@overleaf/promise-utils' import { pipeline } from 'node:stream/promises' +import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' const { z, zz, parseReq } = Validation @@ -142,7 +143,7 @@ export default { } ProjectGetter.getProject( projectId, - { name: true }, + { name: true, 'overleaf.history.id': true }, function (error, project) { if (error) { return next(error) @@ -153,14 +154,33 @@ export default { userId, req.ip ) - ProjectZipStreamManager.createZipStreamForProject( - projectId, - function (error, stream) { + SplitTestHandler.featureFlagEnabled( + req, + res, + 'zip-from-history', + { includeReferer: true }, + function (error, enabled) { if (error) { return next(error) } - prepareZipAttachment(res, `${getSafeProjectName(project)}.zip`) - stream.pipe(res) + ProjectZipStreamManager.createZipStreamForProject( + projectId, + enabled, + project.overleaf.history.id, + function (error, stream, historyVersion) { + if (error) { + return next(error) + } + prepareZipAttachment( + res, + `${getSafeProjectName(project)}.zip` + ) + if (historyVersion != null) { + res.setHeader('X-History-Version', historyVersion) + } + stream.pipe(res) + } + ) } ) } @@ -187,17 +207,29 @@ export default { req.ip ) } - ProjectZipStreamManager.createZipStreamForMultipleProjects( - projectIds, - function (error, stream) { + SplitTestHandler.featureFlagEnabled( + req, + res, + 'zip-from-history', + { includeReferer: true }, + function (error, enabled) { if (error) { return next(error) } - prepareZipAttachment( - res, - `Overleaf Projects (${projectIds.length} items).zip` + ProjectZipStreamManager.createZipStreamForMultipleProjects( + projectIds, + enabled, + function (error, stream) { + if (error) { + return next(error) + } + prepareZipAttachment( + res, + `Overleaf Projects (${projectIds.length} items).zip` + ) + stream.pipe(res) + } ) - stream.pipe(res) } ) } diff --git a/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs b/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs index 1c5eb6b43a..bd0c1dc5a7 100644 --- a/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs +++ b/services/web/app/src/Features/Downloads/ProjectZipStreamManager.mjs @@ -4,10 +4,11 @@ import logger from '@overleaf/logger' import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs' import ProjectGetter from '../Project/ProjectGetter.mjs' import HistoryManager from '../History/HistoryManager.mjs' +import Metrics from '@overleaf/metrics' let ProjectZipStreamManager export default ProjectZipStreamManager = { - createZipStreamForMultipleProjects(projectIds, callback) { + createZipStreamForMultipleProjects(projectIds, zipFromHistory, callback) { // We'll build up a zip file that contains multiple zip files const archive = archiver('zip') archive.on('error', err => @@ -19,38 +20,44 @@ export default ProjectZipStreamManager = { callback(null, archive) const jobs = projectIds.map(projectId => cb => { - ProjectGetter.getProject(projectId, { name: true }, (error, project) => { - if (error) { - return cb(error) - } - if (!project) { - logger.debug( - { projectId }, - 'cannot append project to zip stream: project not found' - ) - return cb() - } - logger.debug( - { projectId, name: project.name }, - 'appending project to zip stream' - ) - ProjectZipStreamManager.createZipStreamForProject( - projectId, - (error, stream) => { - if (error) { - return cb(error) - } - archive.append(stream, { name: `${project.name}.zip` }) - stream.on('end', () => { - logger.debug( - { projectId, name: project.name }, - 'zip stream ended' - ) - cb() - }) + ProjectGetter.getProject( + projectId, + { name: true, 'overleaf.history.id': true }, + (error, project) => { + if (error) { + return cb(error) } - ) - }) + if (!project) { + logger.debug( + { projectId }, + 'cannot append project to zip stream: project not found' + ) + return cb() + } + logger.debug( + { projectId, name: project.name }, + 'appending project to zip stream' + ) + ProjectZipStreamManager.createZipStreamForProject( + projectId, + zipFromHistory, + project.overleaf.history.id, + (error, stream) => { + if (error) { + return cb(error) + } + archive.append(stream, { name: `${project.name}.zip` }) + stream.on('end', () => { + logger.debug( + { projectId, name: project.name }, + 'zip stream ended' + ) + cb() + }) + } + ) + } + ) }) async.series(jobs, () => { @@ -62,7 +69,16 @@ export default ProjectZipStreamManager = { }) }, - createZipStreamForProject(projectId, callback) { + createZipStreamForProject(projectId, zipFromHistory, historyId, callback) { + Metrics.inc('project_zip_download', 1, { + method: zipFromHistory ? 'history-v1' : 'web', + }) + if (zipFromHistory) { + return HistoryManager.flushProject(projectId, error => { + if (error) return callback(error) + HistoryManager.getLatestZipWithHistoryId(historyId, callback) + }) + } const archive = archiver('zip') // return stream immediately before we start adding things to it archive.on('error', err => diff --git a/services/web/app/src/Features/History/HistoryManager.mjs b/services/web/app/src/Features/History/HistoryManager.mjs index 36d4897c59..b6ea680de0 100644 --- a/services/web/app/src/Features/History/HistoryManager.mjs +++ b/services/web/app/src/Features/History/HistoryManager.mjs @@ -1,4 +1,4 @@ -import { callbackify } from 'node:util' +import { callbackify, callbackifyMultiResult } from '@overleaf/promise-utils' import { fetchJson, fetchNothing, @@ -303,6 +303,22 @@ async function getLatestHistoryWithHistoryId(historyId) { ) } +/** + * Get the latest chunk from history using already resolved historyId + * + * @param {string} historyId + */ +async function getLatestZipWithHistoryId(historyId) { + const { response, stream } = await fetchStreamWithResponse( + `${HISTORY_V1_URL}/projects/${historyId}/latest/zip`, + { + basicAuth: HISTORY_V1_BASIC_AUTH, + signal: AbortSignal.timeout(10 * 60 * 1000), + } + ) + return { stream, historyVersion: response.headers.get('X-History-Version') } +} + async function ensureNoResyncPending(projectId) { const { resyncPending } = await fetchJson( `${settings.apis.project_history.url}/project/${projectId}/resync-pending` @@ -475,6 +491,10 @@ export default { requestBlob: callbackify(requestBlob), requestBlobWithProjectId: callbackify(requestBlobWithProjectId), getLatestHistory: callbackify(getLatestHistory), + getLatestZipWithHistoryId: callbackifyMultiResult(getLatestZipWithHistoryId, [ + 'stream', + 'historyVersion', + ]), getChanges: callbackify(getChanges), promises: { initializeProject, @@ -491,6 +511,7 @@ export default { requestBlob, requestBlobWithProjectId, getLatestHistory, + getLatestZipWithHistoryId, getChanges, getChangesWithHistoryId, getProjectBlobStats, diff --git a/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs b/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs index baf8db6b49..351f0d9ef8 100644 --- a/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs +++ b/services/web/app/src/Features/SplitTests/SplitTestHandler.mjs @@ -333,6 +333,30 @@ async function getOneTimeAssignment(splitTestName) { } } +/** + * Checks if a feature flag is enabled for a specific user + * + * Retrieves the feature flag assignment for a user and determines if the assigned variant is 'enabled' + * + * @param req the request + * @param res the Express response object + * @param {string} splitTestName - The unique name of the feature flag + * @param {Object} options + * @param {boolean} options.includeReferer For ajax requests and downloads include the split test overrides of the page + * @returns {Promise} True if the user's assigned variant is 'enabled', false otherwise + */ +async function featureFlagEnabled( + req, + res, + splitTestName, + { includeReferer = false } = { includeReferer: false } +) { + const { variant } = await getAssignment(req, res, splitTestName, { + includeReferer, + }) + return variant === 'enabled' +} + /** * Checks if a feature flag is enabled for a specific user * @@ -997,6 +1021,7 @@ export default { getPercentile, getAssignment: callbackify(getAssignment), getAssignmentForUser: callbackify(getAssignmentForUser), + featureFlagEnabled: callbackify(featureFlagEnabled), featureFlagEnabledForUser: callbackify(featureFlagEnabledForUser), getOneTimeAssignment: callbackify(getOneTimeAssignment), getActiveAssignmentsForUser: callbackify(getActiveAssignmentsForUser), @@ -1006,6 +1031,7 @@ export default { promises: { getAssignment, getAssignmentForUser, + featureFlagEnabled, featureFlagEnabledForUser, getOneTimeAssignment, getActiveAssignmentsForUser, diff --git a/services/web/scripts/e2e_test_setup.mjs b/services/web/scripts/e2e_test_setup.mjs index 7d56b90f20..a0f918db7e 100644 --- a/services/web/scripts/e2e_test_setup.mjs +++ b/services/web/scripts/e2e_test_setup.mjs @@ -163,6 +163,26 @@ export async function provisionSplitTests(merge = false, extraSplitTests = []) { 'utf-8' ) ) + // Add WIP split test, we can update the JSON blob once this is in production + SPLIT_TESTS.push({ + name: 'zip-from-history', + versions: [ + { + versionNumber: 1, + createdAt: '2026-02-25T14:55:31.260Z', + active: true, + analyticsEnabled: false, + phase: 'release', + variants: [ + { + name: 'enabled', + rolloutPercent: 0, + rolloutStripes: [], + }, + ], + }, + ], + }) console.log(`> Importing ${SPLIT_TESTS.length} split-tests from production.`) if (merge) { await SplitTestManager.mergeSplitTests(SPLIT_TESTS, false) diff --git a/services/web/test/acceptance/src/TokenAccessTests.mjs b/services/web/test/acceptance/src/TokenAccessTests.mjs index a3f4c5c9a9..eb66c9ef2e 100644 --- a/services/web/test/acceptance/src/TokenAccessTests.mjs +++ b/services/web/test/acceptance/src/TokenAccessTests.mjs @@ -1112,11 +1112,7 @@ describe('TokenAccess', function () { this.owner.makePrivate(this.projectId, () => { db.projects.updateOne( { _id: project._id }, - { - $set: { - overleaf: { id: 1234 }, - }, - }, + { $set: { 'overleaf.id': 1234 } }, err => { expect(err).not.to.exist done() diff --git a/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs b/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs index fc663acf80..6dd2359539 100644 --- a/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs +++ b/services/web/test/unit/src/Downloads/ProjectDownloadsController.test.mjs @@ -73,6 +73,14 @@ describe('ProjectDownloadsController', function () { }), }) ) + vi.doMock( + '../../../../app/src/Features/SplitTests/SplitTestHandler.mjs', + () => ({ + default: (ctx.SplitTestHandler = { + featureFlagEnabled: sinon.stub().yields(null, false), + }), + }) + ) vi.doMock('@overleaf/settings', () => ({ default: (ctx.Settings = { @@ -92,7 +100,7 @@ describe('ProjectDownloadsController', function () { ctx.stream = { pipe: sinon.stub() } ctx.ProjectZipStreamManager.createZipStreamForProject = sinon .stub() - .callsArgWith(1, null, ctx.stream) + .yields(null, ctx.stream) ctx.req.params = { Project_id: ctx.project_id } ctx.req.ip = '192.168.1.1' ctx.req.session = { @@ -102,9 +110,10 @@ describe('ProjectDownloadsController', function () { }, } ctx.project_name = 'project name with accĂȘnts and % special characters' - ctx.ProjectGetter.getProject = sinon - .stub() - .callsArgWith(2, null, { name: ctx.project_name }) + ctx.ProjectGetter.getProject = sinon.stub().callsArgWith(2, null, { + name: ctx.project_name, + overleaf: { history: { id: 123 } }, + }) ctx.DocumentUpdaterHandler.flushProjectToMongo = sinon .stub() .callsArgWith(1) @@ -138,7 +147,7 @@ describe('ProjectDownloadsController', function () { it("should look up the project's name", function (ctx) { return ctx.ProjectGetter.getProject - .calledWith(ctx.project_id, { name: true }) + .calledWith(ctx.project_id, { name: true, 'overleaf.history.id': true }) .should.equal(true) }) @@ -172,7 +181,7 @@ describe('ProjectDownloadsController', function () { ctx.stream = { pipe: sinon.stub() } ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects = sinon .stub() - .callsArgWith(1, null, ctx.stream) + .yields(null, ctx.stream) ctx.project_ids = ['project-1', 'project-2'] ctx.req.query = { project_ids: ctx.project_ids.join(',') } ctx.req.ip = '192.168.1.1' diff --git a/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs b/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs index 61f8a4ab77..da788f91b7 100644 --- a/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs +++ b/services/web/test/unit/src/Downloads/ProjectZipStreamManager.test.mjs @@ -46,7 +46,9 @@ describe('ProjectZipStreamManager', function () { vi.doMock( '../../../../app/src/Features/History/HistoryManager.mjs', () => ({ - default: (ctx.HistoryManager = {}), + default: (ctx.HistoryManager = { + flushProject: sinon.stub().yields(null), + }), }) ) @@ -81,6 +83,8 @@ describe('ProjectZipStreamManager', function () { ctx.ProjectZipStreamManager.createZipStreamForProject = ( projectId, + zipFromHistory, + historyId, callback ) => { callback(null, ctx.zip_streams[projectId]) @@ -92,12 +96,16 @@ describe('ProjectZipStreamManager', function () { sinon.spy(ctx.ProjectZipStreamManager, 'createZipStreamForProject') ctx.ProjectGetter.getProject = (projectId, fields, callback) => { - return callback(null, { name: ctx.project_names[projectId] }) + return callback(null, { + name: ctx.project_names[projectId], + overleaf: { history: { id: 123 } }, + }) } sinon.spy(ctx.ProjectGetter, 'getProject') ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects( ctx.project_ids, + false, (...args) => { return ctx.callback(...Array.from(args || [])) } @@ -131,7 +139,7 @@ describe('ProjectZipStreamManager', function () { it('should get the names of each project', function (ctx) { return Array.from(ctx.project_ids).map(projectId => ctx.ProjectGetter.getProject - .calledWith(projectId, { name: true }) + .calledWith(projectId, { name: true, 'overleaf.history.id': true }) .should.equal(true) ) }) @@ -160,6 +168,8 @@ describe('ProjectZipStreamManager', function () { ctx.ProjectZipStreamManager.createZipStreamForProject = ( projectId, + zipFromHistory, + historyId, callback ) => { callback(null, ctx.zip_streams[projectId]) @@ -171,12 +181,16 @@ describe('ProjectZipStreamManager', function () { ctx.ProjectGetter.getProject = (projectId, fields, callback) => { const name = ctx.project_names[projectId] - callback(null, name ? { name } : undefined) + callback( + null, + name ? { name, overleaf: { history: { id: 123 } } } : undefined + ) } sinon.spy(ctx.ProjectGetter, 'getProject') ctx.ProjectZipStreamManager.createZipStreamForMultipleProjects( ctx.project_ids, + false, ctx.callback ) @@ -200,7 +214,7 @@ describe('ProjectZipStreamManager', function () { it('should get the names of each project', function (ctx) { ctx.project_ids.map(projectId => ctx.ProjectGetter.getProject - .calledWith(projectId, { name: true }) + .calledWith(projectId, { name: true, 'overleaf.history.id': true }) .should.equal(true) ) }) @@ -237,6 +251,8 @@ describe('ProjectZipStreamManager', function () { ctx.archive.finalize = sinon.stub() return ctx.ProjectZipStreamManager.createZipStreamForProject( ctx.project_id, + false, + 123, ctx.callback ) }) @@ -285,6 +301,8 @@ describe('ProjectZipStreamManager', function () { ctx.archive.finalize = sinon.stub() ctx.ProjectZipStreamManager.createZipStreamForProject( ctx.project_id, + false, + 123, ctx.callback ) }) @@ -317,6 +335,8 @@ describe('ProjectZipStreamManager', function () { ctx.archive.finalize = sinon.stub() return ctx.ProjectZipStreamManager.createZipStreamForProject( ctx.project_id, + false, + 123, ctx.callback ) })