From be5a7b56c8bdd6becafac12afc1cdaa8dbaa6bfa Mon Sep 17 00:00:00 2001 From: Davinder Singh Date: Thu, 23 Apr 2026 11:44:38 +0100 Subject: [PATCH] [WEB + CLSI] Download as docx file feature (#32851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * using CLSI logic for fetching the project contents and skip the .zip export * Use unique conversion directory for project-to-docx export to avoid corrupting the shared compile directory when a compile runs concurrently * Remove X-Accel-Buffering header — not needed as CLSI does not run behind nginx * moving log before sending the data * Return CLSI stream directly instead of buffering to disk on web Previously convertProjectToDocx wrote the CLSI response to a temp file on disk, then the controller read it back to stream to the client. Now the stream is returned directly and piped to the response, avoiding unnecessary disk I/O on the web server. * Use href redirect for docx export instead of fetching blob into memory * making functions and files more generic so they can be used in future for other documents exports as well * adding export-docx split test * adding unit tests * adding cypress E2E test * format:fix * renaming the route to download from convert * adding new icon for export docx button * format:fix * remove unused showExportDocumentErrorToast export and adding guard against invalid Content-Length header from CLSI * format:fix * refactor(clsi): move promisify(parse) into RequestParser * refactor: generic conversion endpoint with type as route param * refactor: use type→extension map for validated conversion types * refactor(clsi): remove --standalone flag and fix rejection test * fixing the href in cypress test * renaming function * adding type to Metrics.inc * fix: rename exportProjectDocument, add WithLock wrapper and metrics type label * format:fix * fix: hide docx export from anonymous users and add WithLock wrapper * format fix * remove redundant Content-Length validation from DocumentConversionManager * format:fix * removing trailing icon GitOrigin-RevId: e9764fefac2c4b625d23be9e942ea4a8b283c70d --- services/clsi/app.js | 5 + services/clsi/app/js/ConversionController.js | 57 ++++++ services/clsi/app/js/ConversionManager.js | 69 +++++++ services/clsi/app/js/RequestParser.js | 3 +- .../test/unit/js/ConversionController.test.js | 177 ++++++++++++++++++ .../test/unit/js/ConversionManager.test.js | 85 +++++++++ .../app/src/Features/Compile/ClsiManager.mjs | 16 ++ .../Downloads/ProjectDownloadsController.mjs | 42 +++++ .../Features/Project/ProjectController.mjs | 1 + .../Uploads/DocumentConversionManager.mjs | 37 +++- services/web/app/src/router.mjs | 16 ++ .../web/frontend/extracted-translations.json | 2 + .../ide-react/components/global-toasts.tsx | 2 + .../components/toolbar/download-project.tsx | 42 +++++ .../toolbar/export-document-toasts.tsx | 22 +++ .../ide-react/components/toolbar/menu-bar.tsx | 2 +- .../components/toolbar/project-title.tsx | 7 +- services/web/locales/en.json | 2 + .../ProjectDownloadsController.test.mjs | 135 ++++++++++++- .../DocumentConversionManager.test.mjs | 91 ++++++--- 20 files changed, 772 insertions(+), 41 deletions(-) create mode 100644 services/web/frontend/js/features/ide-react/components/toolbar/export-document-toasts.tsx diff --git a/services/clsi/app.js b/services/clsi/app.js index 40210426d3..b8d79aaccf 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -130,6 +130,11 @@ app.post( FileUploadMiddleware.multerMiddleware, ConversionController.convertDocxToLaTeX ) +app.post( + '/project/:project_id/user/:user_id/download/project-to-document', + bodyParser.json({ limit: Settings.compileSizeLimit }), + ConversionController.convertProjectToDocument +) if (process.env.NODE_ENV === 'development' && global.__coverage__) { app.get('/coverage', (req, res) => { diff --git a/services/clsi/app/js/ConversionController.js b/services/clsi/app/js/ConversionController.js index afdce93d0d..eb7f9a79cc 100644 --- a/services/clsi/app/js/ConversionController.js +++ b/services/clsi/app/js/ConversionController.js @@ -3,10 +3,14 @@ import { expressify } from '@overleaf/promise-utils' import fs from 'node:fs/promises' import fsSync from 'node:fs' import ConversionManager from './ConversionManager.js' +import ResourceWriter from './ResourceWriter.js' +import RequestParser from './RequestParser.js' import { pipeline } from 'node:stream/promises' import Settings from '@overleaf/settings' import Path from 'node:path' +const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']]) + async function convertDocxToLaTeX(req, res) { const { path } = req.file if (!Settings.enablePandocConversions) { @@ -41,6 +45,59 @@ async function convertDocxToLaTeX(req, res) { } } +async function convertProjectToDocument(req, res) { + if (!Settings.enablePandocConversions) { + return res.sendStatus(404) + } + + const type = req.query.type + const extension = SUPPORTED_CONVERSION_TYPES.get(type) + if (!extension) { + return res.sendStatus(400) + } + + const request = await RequestParser.promises.parse(req.body) + request.project_id = req.params.project_id + request.user_id = req.params.user_id + request.metricsOpts = {} + + const conversionId = crypto.randomUUID() + const conversionDir = Path.join(Settings.path.compilesDir, conversionId) + + logger.debug( + { + projectId: request.project_id, + userId: request.user_id, + rootResourcePath: request.rootResourcePath, + type, + }, + 'syncing resources for project-to-document conversion' + ) + + try { + await ResourceWriter.promises.syncResourcesToDisk(request, conversionDir) + + const documentPath = + await ConversionManager.promises.convertLaTeXToDocumentInDirWithLock( + conversionId, + conversionDir, + request.rootResourcePath, + type, + extension + ) + + const documentStat = await fs.stat(documentPath) + res.setHeader('Content-Length', documentStat.size) + res.attachment(`output.${extension}`) + res.setHeader('X-Content-Type-Options', 'nosniff') + const readStream = fsSync.createReadStream(documentPath) + await pipeline(readStream, res) + } finally { + await fs.rm(conversionDir, { recursive: true, force: true }).catch(() => {}) + } +} + export default { convertDocxToLaTeX: expressify(convertDocxToLaTeX), + convertProjectToDocument: expressify(convertProjectToDocument), } diff --git a/services/clsi/app/js/ConversionManager.js b/services/clsi/app/js/ConversionManager.js index e68583d465..8527064f4c 100644 --- a/services/clsi/app/js/ConversionManager.js +++ b/services/clsi/app/js/ConversionManager.js @@ -94,8 +94,77 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) { return Path.join(conversionDir, outputName) } +async function convertLaTeXToDocumentInDirWithLock( + conversionId, + compileDir, + rootDocPath, + type, + extension +) { + const lock = LockManager.acquire(compileDir) + try { + return await convertLaTeXToDocumentInDir( + conversionId, + compileDir, + rootDocPath, + type, + extension + ) + } finally { + lock.release() + } +} + +async function convertLaTeXToDocumentInDir( + conversionId, + compileDir, + rootDocPath = 'main.tex', + type, + extension +) { + const outputName = crypto.randomUUID() + '.' + extension + const timeoutMs = Settings.conversionTimeoutSeconds * 1000 + + logger.debug( + { compileDir, rootDocPath, type }, + 'running pandoc latex-to-document in compile dir' + ) + + const { exitCode, stdout, stderr } = await CommandRunner.promises.run( + conversionId, + [ + 'pandoc', + rootDocPath, + '--output', + outputName, + '--from', + 'latex', + '--to', + type, + '--resource-path=.', + ], + compileDir, + Settings.pandocImage, + timeoutMs, + {}, + 'conversions' + ) + + if (exitCode !== 0) { + throw new OError('pandoc latex-to-document conversion failed', { + type, + exitCode, + stdout, + stderr, + }) + } + + return Path.join(compileDir, outputName) +} + export default { promises: { convertDocxToLaTeXWithLock, + convertLaTeXToDocumentInDirWithLock, }, } diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js index 5168d72e83..12fac93e86 100644 --- a/services/clsi/app/js/RequestParser.js +++ b/services/clsi/app/js/RequestParser.js @@ -1,3 +1,4 @@ +import { promisify } from 'node:util' import settings from '@overleaf/settings' import OutputCacheManager from './OutputCacheManager.js' @@ -285,4 +286,4 @@ function _checkPath(path) { return path } -export default { parse, MAX_TIMEOUT } +export default { parse, MAX_TIMEOUT, promises: { parse: promisify(parse) } } diff --git a/services/clsi/test/unit/js/ConversionController.test.js b/services/clsi/test/unit/js/ConversionController.test.js index e9c5e77c9a..e4f0a0c7f5 100644 --- a/services/clsi/test/unit/js/ConversionController.test.js +++ b/services/clsi/test/unit/js/ConversionController.test.js @@ -13,12 +13,29 @@ describe('ConversionController', function () { ctx.conversionDir = '/path/to/conversion/result' ctx.zipPath = '/path/to/conversion/result/output.zip' ctx.zipStat = { size: 1234 } + ctx.documentPath = '/compiles/output-uuid/output-uuid.docx' + ctx.documentStat = { size: 5678 } ctx.Settings = { enablePandocConversions: true, + path: { compilesDir: '/compiles' }, } + ctx.parsedRequest = { rootResourcePath: 'main.tex' } ctx.ConversionManager = { promises: { convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath), + convertLaTeXToDocumentInDirWithLock: sinon + .stub() + .resolves(ctx.documentPath), + }, + } + ctx.ResourceWriter = { + promises: { + syncResourcesToDisk: sinon.stub().resolves(), + }, + } + ctx.RequestParser = { + promises: { + parse: sinon.stub().resolves(ctx.parsedRequest), }, } @@ -54,6 +71,14 @@ describe('ConversionController', function () { default: ctx.ConversionManager, })) + vi.doMock('../../../app/js/ResourceWriter', () => ({ + default: ctx.ResourceWriter, + })) + + vi.doMock('../../../app/js/RequestParser', () => ({ + default: ctx.RequestParser, + })) + ctx.res = new PassThrough() ctx.res.attachment = sinon.stub() ctx.res.setHeader = sinon.stub() @@ -155,4 +180,156 @@ describe('ConversionController', function () { }) }) }) + + describe('convertProjectToDocument', function () { + beforeEach(function (ctx) { + ctx.req = { + body: {}, + params: { project_id: 'test-project-id', user_id: 'test-user-id' }, + query: { type: 'docx' }, + } + ctx.fs.stat.resolves(ctx.documentStat) + }) + + describe('when conversions are disabled', function () { + beforeEach(async function (ctx) { + ctx.Settings.enablePandocConversions = false + ctx.res.sendStatus = sinon.stub() + + await ctx.ConversionController.convertProjectToDocument( + ctx.req, + ctx.res, + sinon.stub() + ) + }) + + it('should return 404', function (ctx) { + sinon.assert.calledWith(ctx.res.sendStatus, 404) + }) + + it('should not sync resources or call the conversion manager', function (ctx) { + sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk) + sinon.assert.notCalled( + ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock + ) + }) + }) + + describe('when an unsupported type is requested', function () { + beforeEach(async function (ctx) { + ctx.req.query = { type: 'unsupported' } + ctx.res.sendStatus = sinon.stub() + + await ctx.ConversionController.convertProjectToDocument( + ctx.req, + ctx.res, + sinon.stub() + ) + }) + + it('should return 400', function (ctx) { + sinon.assert.calledWith(ctx.res.sendStatus, 400) + }) + + it('should not sync resources or call the conversion manager', function (ctx) { + sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk) + sinon.assert.notCalled( + ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock + ) + }) + }) + + const uuidDirPattern = + /^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + + describe('successfully', function () { + beforeEach(async function (ctx) { + await ctx.ConversionController.convertProjectToDocument( + ctx.req, + ctx.res, + sinon.stub() + ) + }) + + it('should sync resources to a unique conversion directory', function (ctx) { + sinon.assert.calledWith( + ctx.ResourceWriter.promises.syncResourcesToDisk, + sinon.match({ rootResourcePath: 'main.tex' }), + sinon.match(uuidDirPattern) + ) + }) + + it('should call convertLaTeXToDocumentInDirWithLock with docx type and extension', function (ctx) { + sinon.assert.calledWith( + ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock, + sinon.match( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + ), + sinon.match(uuidDirPattern), + 'main.tex', + 'docx', + 'docx' + ) + }) + + it('should set the Content-Length header from the document stat', function (ctx) { + sinon.assert.calledWith( + ctx.res.setHeader, + 'Content-Length', + ctx.documentStat.size + ) + }) + + it('should set the attachment filename', function (ctx) { + sinon.assert.calledWith(ctx.res.attachment, 'output.docx') + }) + + it('should set X-Content-Type-Options header', function (ctx) { + sinon.assert.calledWith( + ctx.res.setHeader, + 'X-Content-Type-Options', + 'nosniff' + ) + }) + + it('should stream the document to the response', function (ctx) { + sinon.assert.calledWith(ctx.fsSync.createReadStream, ctx.documentPath) + sinon.assert.calledWith(ctx.pipeline, ctx.readStream, ctx.res) + }) + + it('should clean up the conversion directory', function (ctx) { + sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), { + recursive: true, + force: true, + }) + }) + }) + + describe('when conversion fails', function () { + beforeEach(async function (ctx) { + ctx.next = sinon.stub() + ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock.rejects( + new Error('mock conversion error') + ) + + await ctx.ConversionController.convertProjectToDocument( + ctx.req, + ctx.res, + ctx.next + ) + }) + + it('should pass the error to next', function (ctx) { + sinon.assert.calledOnce(ctx.next) + expect(ctx.next.firstCall.args[0]).to.be.instanceOf(Error) + }) + + it('should still clean up the conversion directory', function (ctx) { + sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), { + recursive: true, + force: true, + }) + }) + }) + }) }) diff --git a/services/clsi/test/unit/js/ConversionManager.test.js b/services/clsi/test/unit/js/ConversionManager.test.js index b8288eba5e..24999fccde 100644 --- a/services/clsi/test/unit/js/ConversionManager.test.js +++ b/services/clsi/test/unit/js/ConversionManager.test.js @@ -250,4 +250,89 @@ describe('ConversionManager', function () { }) }) }) + + describe('convertLaTeXToDocumentInDirWithLock', function () { + describe('successfully', function () { + beforeEach(async function (ctx) { + ctx.compileDir = '/compiles/test-compile-dir' + ctx.rootDocPath = 'main.tex' + ctx.type = 'docx' + ctx.extension = 'docx' + + ctx.result = + await ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock( + ctx.conversionId, + ctx.compileDir, + ctx.rootDocPath, + ctx.type, + ctx.extension + ) + }) + + it('should acquire a lock on the compile dir', function (ctx) { + sinon.assert.calledWith(ctx.LockManager.acquire, ctx.compileDir) + }) + + it('should release the lock', function (ctx) { + sinon.assert.called(ctx.lock.release) + }) + + it('should run pandoc with correct arguments', function (ctx) { + expect(ctx.CommandRunner.promises.run.callCount).toBe(1) + expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([ + ctx.conversionId, + [ + 'pandoc', + ctx.rootDocPath, + '--output', + `output-uuid.${ctx.extension}`, + '--from', + 'latex', + '--to', + ctx.type, + '--resource-path=.', + ], + ctx.compileDir, + ctx.Settings.pandocImage, + 60_000, + {}, + 'conversions', + ]) + }) + + it('should convert conversion timeout to milliseconds', function (ctx) { + expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000) + }) + + it('should return path to the output document', function (ctx) { + expect(ctx.result).toBe( + Path.join(ctx.compileDir, `output-uuid.${ctx.extension}`) + ) + }) + }) + + describe('when pandoc fails (non-zero exit code)', function () { + it('should reject with an error and release the lock', async function (ctx) { + ctx.compileDir = '/compiles/test-compile-dir' + + ctx.CommandRunner.promises.run.resolves({ + stdout: 'mock-stdout', + stderr: 'mock-stderr', + exitCode: 1, + }) + + await expect( + ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock( + ctx.conversionId, + ctx.compileDir, + 'main.tex', + 'docx', + 'docx' + ) + ).to.be.rejectedWith('pandoc latex-to-document conversion failed') + + sinon.assert.called(ctx.lock.release) + }) + }) + }) }) diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index 3e8e9c35e7..97f4a93c5a 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1171,6 +1171,21 @@ function _finaliseRequest(projectId, options, project, docs, files) { } } +async function buildDocumentConversionRequest(projectId) { + const project = await ProjectGetter.promises.getProject(projectId, { + compiler: 1, + imageName: 1, + 'overleaf.history.id': 1, + rootDoc_id: 1, + rootFolder: 1, + }) + if (project == null) { + throw new Errors.NotFoundError(`project does not exist: ${projectId}`) + } + const projectStateHash = ClsiStateManager.computeHash(project, {}) + return _buildRequestFromMongo(projectId, {}, project, projectStateHash) +} + async function wordCount(projectId, userId, file, limits, clsiserverid) { const { compileBackendClass, compileGroup } = limits const req = await _buildRequest(projectId, userId, limits) @@ -1297,5 +1312,6 @@ export default { getOutputFileStream, wordCount, syncTeX, + buildDocumentConversionRequest, }, } diff --git a/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs b/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs index b51aad3b3a..520b5a712a 100644 --- a/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs +++ b/services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs @@ -5,13 +5,55 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj import { prepareZipAttachment } from '../../infrastructure/Response.mjs' import SessionManager from '../Authentication/SessionManager.mjs' import ProjectAuditLogHandler from '../Project/ProjectAuditLogHandler.mjs' +import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs' +import { expressify } from '@overleaf/promise-utils' +import { pipeline } from 'node:stream/promises' + +const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']]) // Keep in sync with the logic for PDF files in CompileController function getSafeProjectName(project) { return project.name.replace(/[^\p{L}\p{Nd}]/gu, '_') } +async function exportProjectConversion(req, res) { + const type = req.params.type + const extension = SUPPORTED_CONVERSION_TYPES.get(type) + if (!extension) { + return res.sendStatus(400) + } + const userId = SessionManager.getLoggedInUserId(req.session) + const projectId = req.params.Project_id + Metrics.inc('document-exports', 1, { type }) + + const project = await ProjectGetter.promises.getProject(projectId, { + name: true, + }) + + const { stream, contentLength } = + await DocumentConversionManager.promises.convertProjectToDocument( + projectId, + userId, + type + ) + + const safeFileName = getSafeProjectName(project) + res.setHeader('Content-Length', contentLength) + res.attachment(`${safeFileName}.${extension}`) + res.setHeader('X-Content-Type-Options', 'nosniff') + res.setHeader('X-Accel-Buffering', 'no') + ProjectAuditLogHandler.addEntryInBackground( + projectId, + `project-exported-${type}`, + userId, + req.ip + ) + await pipeline(stream, res) +} + export default { + exportProjectConversion: expressify(exportProjectConversion), + downloadProject(req, res, next) { const userId = SessionManager.getLoggedInUserId(req.session) const projectId = req.params.Project_id diff --git a/services/web/app/src/Features/Project/ProjectController.mjs b/services/web/app/src/Features/Project/ProjectController.mjs index 1762236089..50b3500556 100644 --- a/services/web/app/src/Features/Project/ProjectController.mjs +++ b/services/web/app/src/Features/Project/ProjectController.mjs @@ -480,6 +480,7 @@ const _ProjectController = { 'wf-fake-non-english-suggestions', 'editor-tabs', 'overleaf-code', + 'export-docx', ].filter(Boolean) const getUserValues = async userId => diff --git a/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs b/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs index dddaaa49df..a4b9fe6887 100644 --- a/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs +++ b/services/web/app/src/Features/Uploads/DocumentConversionManager.mjs @@ -1,5 +1,6 @@ import Settings from '@overleaf/settings' import CompileManager from '../Compile/CompileManager.mjs' +import ClsiManager from '../Compile/ClsiManager.mjs' import fs from 'node:fs' import fsPromises from 'node:fs/promises' import logger from '@overleaf/logger' @@ -38,14 +39,7 @@ async function convertDocxToLaTeXZipArchive(path, userId) { signal: abortController.signal, }) - const contentLengthHeader = response.headers.get('Content-Length') - if (contentLengthHeader == null) { - logger.warn( - 'CLSI did not provide Content-Length header for converted document' - ) - throw new OError('CLSI response missing Content-Length header') - } - const contentLength = parseInt(contentLengthHeader, 10) + const contentLength = parseInt(response.headers.get('Content-Length'), 10) if (contentLength > Settings.maxUploadSize) { abortController.abort() stream.destroy() @@ -77,8 +71,35 @@ async function convertDocxToLaTeXZipArchive(path, userId) { return outputPath } +async function convertProjectToDocument(projectId, userId, type) { + const limits = await CompileManager.promises._getUserCompileLimits(userId) + const clsiRequest = + await ClsiManager.promises.buildDocumentConversionRequest(projectId) + + const clsiUrl = new URL(Settings.apis.clsi.url) + clsiUrl.pathname = `/project/${projectId}/user/${userId}/download/project-to-document` + clsiUrl.searchParams.set('type', type) + clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass) + clsiUrl.searchParams.set('compileGroup', limits.compileGroup) + + logger.debug( + { clsiUrl: clsiUrl.toString(), projectId, userId, type }, + 'sending project to CLSI for document conversion' + ) + + const { stream, response } = await fetchStreamWithResponse(clsiUrl, { + method: 'POST', + json: clsiRequest, + }) + + const contentLength = parseInt(response.headers.get('Content-Length'), 10) + + return { stream, contentLength } +} + export default { promises: { convertDocxToLaTeXZipArchive, + convertProjectToDocument, }, } diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index a8b940afba..0e37ff3a2d 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -193,6 +193,10 @@ const rateLimiters = { points: 10, duration: 60, }), + documentExport: new RateLimiter('document-export', { + points: 5, + duration: 60, + }), } async function initialize(webRouter, privateApiRouter, publicApiRouter) { @@ -764,6 +768,18 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { ExportsController.exportDownload ) + if (Settings.enablePandocConversions) { + webRouter.get( + '/project/:Project_id/download/conversion/:type', + AuthenticationController.requireLogin(), + RateLimiterMiddleware.rateLimit(rateLimiters.documentExport, { + params: ['Project_id'], + }), + AuthorizationMiddleware.ensureUserCanReadProject, + ProjectDownloadsController.exportProjectConversion + ) + } + webRouter.get( '/Project/:Project_id/download/zip', RateLimiterMiddleware.rateLimit(rateLimiters.zipDownload, { diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index dc3683ac72..ca1a2a873a 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -618,7 +618,9 @@ "expires": "", "expires_in_days": "", "expires_on": "", + "export_as_docx": "", "export_csv": "", + "export_document_error": "", "export_project_to_github": "", "failed": "", "failed_to_consent_to_workbench_terms": "", diff --git a/services/web/frontend/js/features/ide-react/components/global-toasts.tsx b/services/web/frontend/js/features/ide-react/components/global-toasts.tsx index 7788d4c0b3..e2faf57d83 100644 --- a/services/web/frontend/js/features/ide-react/components/global-toasts.tsx +++ b/services/web/frontend/js/features/ide-react/components/global-toasts.tsx @@ -7,6 +7,7 @@ import importOverleafModules from '../../../../macros/import-overleaf-module.mac import { OLToastContainer } from '@/shared/components/ol/ol-toast-container' import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts' import importDocxFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-docx-feedback-toast' +import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts' const moduleGeneratorsImport = importOverleafModules('toastGenerators') as { import: { default: GlobalToastGeneratorEntry[] } @@ -29,6 +30,7 @@ const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [ ...moduleGenerators.flat(), ...clipboardToastGenerators, ...importDocxFeedbackToastGenerators, + ...exportDocumentToastGenerators, ] const GENERATOR_MAP: Map = new Map( GENERATOR_LIST.map(({ key, generator }) => [key, generator]) diff --git a/services/web/frontend/js/features/ide-react/components/toolbar/download-project.tsx b/services/web/frontend/js/features/ide-react/components/toolbar/download-project.tsx index 16cb17774e..390480430a 100644 --- a/services/web/frontend/js/features/ide-react/components/toolbar/download-project.tsx +++ b/services/web/frontend/js/features/ide-react/components/toolbar/download-project.tsx @@ -7,6 +7,8 @@ import { useProjectContext } from '@/shared/context/project-context' import { useCallback } from 'react' import { useTranslation } from 'react-i18next' import { useEditorAnalytics } from '@/shared/hooks/use-editor-analytics' +import getMeta from '@/utils/meta' +import { useFeatureFlag } from '@/shared/context/split-test-context' export const DownloadProjectZip = () => { const { t } = useTranslation() @@ -100,3 +102,43 @@ export const DownloadProjectPDF = () => { return button } } + +export const ExportProjectDocx = () => { + const { t } = useTranslation() + const { projectId } = useProjectContext() + const exportDocxEnabled = useFeatureFlag('export-docx') + const enablePandocConversions = + getMeta('ol-ExposedSettings')?.enablePandocConversions + const anonymous = getMeta('ol-anonymous') + + const showExportDocx = + exportDocxEnabled && enablePandocConversions && !anonymous + + useCommandProvider( + () => + showExportDocx + ? [ + { + id: 'export-as-docx', + href: `/project/${projectId}/download/conversion/docx`, + label: t('export_as_docx'), + }, + ] + : [], + [t, showExportDocx, projectId] + ) + + if (!showExportDocx) { + return null + } + + return ( + + {t('export_as_docx')} + + ) +} diff --git a/services/web/frontend/js/features/ide-react/components/toolbar/export-document-toasts.tsx b/services/web/frontend/js/features/ide-react/components/toolbar/export-document-toasts.tsx new file mode 100644 index 0000000000..eb037fd662 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/toolbar/export-document-toasts.tsx @@ -0,0 +1,22 @@ +import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts' +import { useTranslation } from 'react-i18next' + +const ExportDocumentErrorToast = () => { + const { t } = useTranslation() + return {t('export_document_error')} +} + +const generators: GlobalToastGeneratorEntry[] = [ + { + key: 'export-document:error', + generator: () => ({ + content: , + type: 'error', + autoHide: true, + delay: 5000, + isDismissible: true, + }), + }, +] + +export default generators diff --git a/services/web/frontend/js/features/ide-react/components/toolbar/menu-bar.tsx b/services/web/frontend/js/features/ide-react/components/toolbar/menu-bar.tsx index 6a146b6ad6..9a25786f08 100644 --- a/services/web/frontend/js/features/ide-react/components/toolbar/menu-bar.tsx +++ b/services/web/frontend/js/features/ide-react/components/toolbar/menu-bar.tsx @@ -87,7 +87,7 @@ export const ToolbarMenuBar = () => { { id: 'submit', children: ['submit-project', 'manage-template'] }, { id: 'file-download', - children: ['download-as-source-zip', 'download-pdf'], + children: ['download-as-source-zip', 'download-pdf', 'export-as-docx'], }, { id: 'settings', diff --git a/services/web/frontend/js/features/ide-react/components/toolbar/project-title.tsx b/services/web/frontend/js/features/ide-react/components/toolbar/project-title.tsx index 030ba47569..50fa058041 100644 --- a/services/web/frontend/js/features/ide-react/components/toolbar/project-title.tsx +++ b/services/web/frontend/js/features/ide-react/components/toolbar/project-title.tsx @@ -10,7 +10,11 @@ import { useTranslation } from 'react-i18next' import importOverleafModules from '../../../../../macros/import-overleaf-module.macro' import { useEditorContext } from '@/shared/context/editor-context' import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' -import { DownloadProjectPDF, DownloadProjectZip } from './download-project' +import { + DownloadProjectPDF, + DownloadProjectZip, + ExportProjectDocx, +} from './download-project' import { useCallback, useState } from 'react' import OLDropdownMenuItem from '@/shared/components/ol/ol-dropdown-menu-item' import EditableLabel from './editable-label' @@ -76,6 +80,7 @@ export const ToolbarProjectTitle = () => { )} + ({ - default: (ctx.ProjectGetter = {}), + default: (ctx.ProjectGetter = { + promises: { + getProject: sinon.stub(), + }, + }), })) vi.doMock( @@ -47,6 +51,32 @@ describe('ProjectDownloadsController', function () { }) ) + vi.doMock( + '../../../../app/src/Features/Authentication/SessionManager.mjs', + () => ({ + default: (ctx.SessionManager = { + getLoggedInUserId: sinon + .stub() + .callsFake(session => session?.user?._id), + }), + }) + ) + + vi.doMock( + '../../../../app/src/Features/Uploads/DocumentConversionManager.mjs', + () => ({ + default: (ctx.DocumentConversionManager = { + promises: { + convertProjectToDocument: sinon.stub(), + }, + }), + }) + ) + + vi.doMock('node:stream/promises', () => ({ + pipeline: (ctx.pipeline = sinon.stub().resolves()), + })) + ctx.ProjectDownloadsController = (await import(modulePath)).default }) @@ -208,4 +238,107 @@ describe('ProjectDownloadsController', function () { } }) }) + + describe('exportProjectConversion', function () { + describe('when an unsupported type is requested', function () { + beforeEach(async function (ctx) { + ctx.req.params = { Project_id: 'test-project-id', type: 'unsupported' } + ctx.req.session = { user: { _id: 'test-user-id' } } + + await ctx.ProjectDownloadsController.exportProjectConversion( + ctx.req, + ctx.res, + ctx.next + ) + }) + + it('should return 400', function (ctx) { + expect(ctx.res.statusCode).to.equal(400) + }) + + it('should not call the conversion manager', function (ctx) { + sinon.assert.notCalled( + ctx.DocumentConversionManager.promises.convertProjectToDocument + ) + }) + }) + + describe('with a supported type', function () { + beforeEach(async function (ctx) { + ctx.projectId = 'test-project-id' + ctx.userId = 'test-user-id' + ctx.projectName = 'My Test Project' + ctx.exportStream = { pipe: sinon.stub() } + ctx.contentLength = 9876 + + ctx.req.params = { Project_id: ctx.projectId, type: 'docx' } + ctx.req.session = { user: { _id: ctx.userId } } + ctx.req.ip = '192.168.1.1' + + ctx.res.attachment = sinon.stub().returns(ctx.res) + + ctx.SessionManager.getLoggedInUserId.returns(ctx.userId) + ctx.ProjectGetter.promises.getProject.resolves({ + name: ctx.projectName, + }) + ctx.DocumentConversionManager.promises.convertProjectToDocument.resolves( + { + stream: ctx.exportStream, + contentLength: ctx.contentLength, + } + ) + + await ctx.ProjectDownloadsController.exportProjectConversion( + ctx.req, + ctx.res, + ctx.next + ) + }) + + it('should call convertProjectToDocument with the docx type', function (ctx) { + sinon.assert.calledWith( + ctx.DocumentConversionManager.promises.convertProjectToDocument, + ctx.projectId, + ctx.userId, + 'docx' + ) + }) + + it('should set the Content-Length header', function (ctx) { + expect(ctx.res.headers['Content-Length']).to.equal(ctx.contentLength) + }) + + it('should set the attachment filename with safe project name', function (ctx) { + sinon.assert.calledWith(ctx.res.attachment, 'My_Test_Project.docx') + }) + + it('should set the X-Content-Type-Options header', function (ctx) { + expect(ctx.res.headers['X-Content-Type-Options']).to.equal('nosniff') + }) + + it('should set the X-Accel-Buffering header', function (ctx) { + expect(ctx.res.headers['X-Accel-Buffering']).to.equal('no') + }) + + it('should add an audit log entry', function (ctx) { + sinon.assert.calledWith( + ctx.ProjectAuditLogHandler.addEntryInBackground, + ctx.projectId, + 'project-exported-docx', + ctx.userId, + ctx.req.ip + ) + }) + + it('should record the action via Metrics', function (ctx) { + ctx.Metrics.inc + .calledWith('document-exports', 1, { type: 'docx' }) + .should.equal(true) + }) + + it('should stream the document to the response', function (ctx) { + sinon.assert.calledWith(ctx.pipeline, ctx.exportStream, ctx.res) + }) + }) + }) }) diff --git a/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs b/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs index 8de7ca8806..827dbd6d31 100644 --- a/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs +++ b/services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs @@ -73,6 +73,18 @@ describe('DocumentConversionManager', function () { }) ) + ctx.ClsiManager = { + promises: { + buildDocumentConversionRequest: sinon + .stub() + .resolves({ some: 'clsi-request' }), + }, + } + + vi.doMock('../../../../app/src/Features/Compile/ClsiManager.mjs', () => ({ + default: ctx.ClsiManager, + })) + ctx.DocumentConversionManager = (await import(MODULE_PATH)).default }) @@ -215,43 +227,64 @@ describe('DocumentConversionManager', function () { ) }) }) + }) - describe('when the Content-Length header is missing', function () { + describe('convertProjectToDocument', function () { + beforeEach(function (ctx) { + ctx.projectId = 'test-project-id' + ctx.userId = 'test-user-id' + ctx.type = 'docx' + ctx.mockStream = { destroy: sinon.stub() } + ctx.response = { + headers: { get: sinon.stub().returns(null) }, + } + ctx.response.headers.get.withArgs('Content-Length').returns('50') + ctx.fetchUtils.fetchStreamWithResponse.resolves({ + stream: ctx.mockStream, + response: ctx.response, + }) + }) + + describe('successfully', function () { beforeEach(async function (ctx) { - ctx.path = '/path/to/input.docx' - ctx.userId = 'test-user-id' - ctx.response = { - headers: { - get: sinon.stub().returns(null), - }, - } - - ctx.fetchUtils.fetchStreamWithResponse.resolves({ - stream: 'mocked-fetch-stream', - response: ctx.response, - }) - - await expect( - ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive( - ctx.path, - ctx.userId + ctx.result = + await ctx.DocumentConversionManager.promises.convertProjectToDocument( + ctx.projectId, + ctx.userId, + ctx.type ) - ).to.be.rejectedWith('document conversion failed') }) - it('should not write the archive to disk', function (ctx) { - sinon.assert.notCalled(ctx.fs.createWriteStream) - sinon.assert.notCalled(ctx.nodeStream.pipeline) - }) - - it('should attempt to clean up the output path', function (ctx) { + it('should build the CLSI document conversion request', function (ctx) { sinon.assert.calledWith( - ctx.fsPromises.unlink, - sinon.match( - /\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/ - ) + ctx.ClsiManager.promises.buildDocumentConversionRequest, + ctx.projectId ) }) + + it('should call CLSI with the correct URL', function (ctx) { + const expectedUrl = new URL(ctx.Settings.apis.clsi.url) + expectedUrl.pathname = `/project/${ctx.projectId}/user/${ctx.userId}/download/project-to-document` + expectedUrl.searchParams.set('type', ctx.type) + expectedUrl.searchParams.set( + 'compileBackendClass', + 'test-backend-class' + ) + expectedUrl.searchParams.set('compileGroup', 'test-compile-group') + + sinon.assert.calledWith( + ctx.fetchUtils.fetchStreamWithResponse, + sinon.match(url => url.toString() === expectedUrl.toString()), + { method: 'POST', json: { some: 'clsi-request' } } + ) + }) + + it('should return the stream and content length', function (ctx) { + expect(ctx.result).to.deep.equal({ + stream: ctx.mockStream, + contentLength: 50, + }) + }) }) }) })