[WEB + CLSI] Download as docx file feature (#32851)

* using CLSI logic for fetching the project contents and skip the .zip export

* Use unique conversion directory for project-to-docx export to avoid corrupting the shared compile
  directory when a compile runs concurrently

* Remove X-Accel-Buffering header — not needed as CLSI does not run behind nginx

* moving log before sending the data

* Return CLSI stream directly instead of buffering to disk on web

  Previously convertProjectToDocx wrote the CLSI response to a temp file
  on disk, then the controller read it back to stream to the client.
  Now the stream is returned directly and piped to the response,
  avoiding unnecessary disk I/O on the web server.

* Use href redirect for docx export instead of fetching blob into memory

* making functions and files more generic so they can be used in future for other documents exports as well

* adding export-docx split test

* adding unit tests

* adding cypress E2E test

* format:fix

* renaming the route to download from convert

* adding new icon for export docx button

* format:fix

* remove unused showExportDocumentErrorToast export and adding guard against invalid Content-Length header from CLSI

* format:fix

* refactor(clsi): move promisify(parse) into RequestParser

* refactor: generic conversion endpoint with type as route
  param

* refactor: use type→extension map for validated conversion types

* refactor(clsi): remove --standalone flag and fix rejection test

* fixing the href in cypress test

* renaming function

* adding type to Metrics.inc

* fix: rename exportProjectDocument, add WithLock wrapper and metrics type label

* format:fix

* fix: hide docx export from anonymous users and add WithLock wrapper

* format fix

* remove redundant Content-Length validation from DocumentConversionManager

* format:fix

* removing trailing icon

GitOrigin-RevId: e9764fefac2c4b625d23be9e942ea4a8b283c70d

services/clsi/app.js

@@ -130,6 +130,11 @@ app.post(
   FileUploadMiddleware.multerMiddleware,
   ConversionController.convertDocxToLaTeX
 )
+app.post(
+  '/project/:project_id/user/:user_id/download/project-to-document',
+  bodyParser.json({ limit: Settings.compileSizeLimit }),
+  ConversionController.convertProjectToDocument
+)
 
 if (process.env.NODE_ENV === 'development' && global.__coverage__) {
   app.get('/coverage', (req, res) => {

services/clsi/app/js/ConversionController.js

@@ -3,10 +3,14 @@ import { expressify } from '@overleaf/promise-utils'
 import fs from 'node:fs/promises'
 import fsSync from 'node:fs'
 import ConversionManager from './ConversionManager.js'
+import ResourceWriter from './ResourceWriter.js'
+import RequestParser from './RequestParser.js'
 import { pipeline } from 'node:stream/promises'
 import Settings from '@overleaf/settings'
 import Path from 'node:path'
 
+const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']])
+
 async function convertDocxToLaTeX(req, res) {
   const { path } = req.file
   if (!Settings.enablePandocConversions) {
@@ -41,6 +45,59 @@ async function convertDocxToLaTeX(req, res) {
   }
 }
 
+async function convertProjectToDocument(req, res) {
+  if (!Settings.enablePandocConversions) {
+    return res.sendStatus(404)
+  }
+
+  const type = req.query.type
+  const extension = SUPPORTED_CONVERSION_TYPES.get(type)
+  if (!extension) {
+    return res.sendStatus(400)
+  }
+
+  const request = await RequestParser.promises.parse(req.body)
+  request.project_id = req.params.project_id
+  request.user_id = req.params.user_id
+  request.metricsOpts = {}
+
+  const conversionId = crypto.randomUUID()
+  const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
+
+  logger.debug(
+    {
+      projectId: request.project_id,
+      userId: request.user_id,
+      rootResourcePath: request.rootResourcePath,
+      type,
+    },
+    'syncing resources for project-to-document conversion'
+  )
+
+  try {
+    await ResourceWriter.promises.syncResourcesToDisk(request, conversionDir)
+
+    const documentPath =
+      await ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
+        conversionId,
+        conversionDir,
+        request.rootResourcePath,
+        type,
+        extension
+      )
+
+    const documentStat = await fs.stat(documentPath)
+    res.setHeader('Content-Length', documentStat.size)
+    res.attachment(`output.${extension}`)
+    res.setHeader('X-Content-Type-Options', 'nosniff')
+    const readStream = fsSync.createReadStream(documentPath)
+    await pipeline(readStream, res)
+  } finally {
+    await fs.rm(conversionDir, { recursive: true, force: true }).catch(() => {})
+  }
+}
+
 export default {
   convertDocxToLaTeX: expressify(convertDocxToLaTeX),
+  convertProjectToDocument: expressify(convertProjectToDocument),
 }

services/clsi/app/js/ConversionManager.js

@@ -94,8 +94,77 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
   return Path.join(conversionDir, outputName)
 }
 
+async function convertLaTeXToDocumentInDirWithLock(
+  conversionId,
+  compileDir,
+  rootDocPath,
+  type,
+  extension
+) {
+  const lock = LockManager.acquire(compileDir)
+  try {
+    return await convertLaTeXToDocumentInDir(
+      conversionId,
+      compileDir,
+      rootDocPath,
+      type,
+      extension
+    )
+  } finally {
+    lock.release()
+  }
+}
+
+async function convertLaTeXToDocumentInDir(
+  conversionId,
+  compileDir,
+  rootDocPath = 'main.tex',
+  type,
+  extension
+) {
+  const outputName = crypto.randomUUID() + '.' + extension
+  const timeoutMs = Settings.conversionTimeoutSeconds * 1000
+
+  logger.debug(
+    { compileDir, rootDocPath, type },
+    'running pandoc latex-to-document in compile dir'
+  )
+
+  const { exitCode, stdout, stderr } = await CommandRunner.promises.run(
+    conversionId,
+    [
+      'pandoc',
+      rootDocPath,
+      '--output',
+      outputName,
+      '--from',
+      'latex',
+      '--to',
+      type,
+      '--resource-path=.',
+    ],
+    compileDir,
+    Settings.pandocImage,
+    timeoutMs,
+    {},
+    'conversions'
+  )
+
+  if (exitCode !== 0) {
+    throw new OError('pandoc latex-to-document conversion failed', {
+      type,
+      exitCode,
+      stdout,
+      stderr,
+    })
+  }
+
+  return Path.join(compileDir, outputName)
+}
+
 export default {
   promises: {
     convertDocxToLaTeXWithLock,
+    convertLaTeXToDocumentInDirWithLock,
   },
 }

services/clsi/app/js/RequestParser.js

@@ -1,3 +1,4 @@
+import { promisify } from 'node:util'
 import settings from '@overleaf/settings'
 import OutputCacheManager from './OutputCacheManager.js'
 
@@ -285,4 +286,4 @@ function _checkPath(path) {
   return path
 }
 
-export default { parse, MAX_TIMEOUT }
+export default { parse, MAX_TIMEOUT, promises: { parse: promisify(parse) } }

services/clsi/test/unit/js/ConversionController.test.js

@@ -13,12 +13,29 @@ describe('ConversionController', function () {
     ctx.conversionDir = '/path/to/conversion/result'
     ctx.zipPath = '/path/to/conversion/result/output.zip'
     ctx.zipStat = { size: 1234 }
+    ctx.documentPath = '/compiles/output-uuid/output-uuid.docx'
+    ctx.documentStat = { size: 5678 }
     ctx.Settings = {
       enablePandocConversions: true,
+      path: { compilesDir: '/compiles' },
     }
+    ctx.parsedRequest = { rootResourcePath: 'main.tex' }
     ctx.ConversionManager = {
       promises: {
         convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
+        convertLaTeXToDocumentInDirWithLock: sinon
+          .stub()
+          .resolves(ctx.documentPath),
+      },
+    }
+    ctx.ResourceWriter = {
+      promises: {
+        syncResourcesToDisk: sinon.stub().resolves(),
+      },
+    }
+    ctx.RequestParser = {
+      promises: {
+        parse: sinon.stub().resolves(ctx.parsedRequest),
       },
     }
 
@@ -54,6 +71,14 @@ describe('ConversionController', function () {
       default: ctx.ConversionManager,
     }))
 
+    vi.doMock('../../../app/js/ResourceWriter', () => ({
+      default: ctx.ResourceWriter,
+    }))
+
+    vi.doMock('../../../app/js/RequestParser', () => ({
+      default: ctx.RequestParser,
+    }))
+
     ctx.res = new PassThrough()
     ctx.res.attachment = sinon.stub()
     ctx.res.setHeader = sinon.stub()
@@ -155,4 +180,156 @@ describe('ConversionController', function () {
       })
     })
   })
+
+  describe('convertProjectToDocument', function () {
+    beforeEach(function (ctx) {
+      ctx.req = {
+        body: {},
+        params: { project_id: 'test-project-id', user_id: 'test-user-id' },
+        query: { type: 'docx' },
+      }
+      ctx.fs.stat.resolves(ctx.documentStat)
+    })
+
+    describe('when conversions are disabled', function () {
+      beforeEach(async function (ctx) {
+        ctx.Settings.enablePandocConversions = false
+        ctx.res.sendStatus = sinon.stub()
+
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          sinon.stub()
+        )
+      })
+
+      it('should return 404', function (ctx) {
+        sinon.assert.calledWith(ctx.res.sendStatus, 404)
+      })
+
+      it('should not sync resources or call the conversion manager', function (ctx) {
+        sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk)
+        sinon.assert.notCalled(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock
+        )
+      })
+    })
+
+    describe('when an unsupported type is requested', function () {
+      beforeEach(async function (ctx) {
+        ctx.req.query = { type: 'unsupported' }
+        ctx.res.sendStatus = sinon.stub()
+
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          sinon.stub()
+        )
+      })
+
+      it('should return 400', function (ctx) {
+        sinon.assert.calledWith(ctx.res.sendStatus, 400)
+      })
+
+      it('should not sync resources or call the conversion manager', function (ctx) {
+        sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk)
+        sinon.assert.notCalled(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock
+        )
+      })
+    })
+
+    const uuidDirPattern =
+      /^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+
+    describe('successfully', function () {
+      beforeEach(async function (ctx) {
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          sinon.stub()
+        )
+      })
+
+      it('should sync resources to a unique conversion directory', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ResourceWriter.promises.syncResourcesToDisk,
+          sinon.match({ rootResourcePath: 'main.tex' }),
+          sinon.match(uuidDirPattern)
+        )
+      })
+
+      it('should call convertLaTeXToDocumentInDirWithLock with docx type and extension', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
+          sinon.match(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          ),
+          sinon.match(uuidDirPattern),
+          'main.tex',
+          'docx',
+          'docx'
+        )
+      })
+
+      it('should set the Content-Length header from the document stat', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.res.setHeader,
+          'Content-Length',
+          ctx.documentStat.size
+        )
+      })
+
+      it('should set the attachment filename', function (ctx) {
+        sinon.assert.calledWith(ctx.res.attachment, 'output.docx')
+      })
+
+      it('should set X-Content-Type-Options header', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.res.setHeader,
+          'X-Content-Type-Options',
+          'nosniff'
+        )
+      })
+
+      it('should stream the document to the response', function (ctx) {
+        sinon.assert.calledWith(ctx.fsSync.createReadStream, ctx.documentPath)
+        sinon.assert.calledWith(ctx.pipeline, ctx.readStream, ctx.res)
+      })
+
+      it('should clean up the conversion directory', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), {
+          recursive: true,
+          force: true,
+        })
+      })
+    })
+
+    describe('when conversion fails', function () {
+      beforeEach(async function (ctx) {
+        ctx.next = sinon.stub()
+        ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock.rejects(
+          new Error('mock conversion error')
+        )
+
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          ctx.next
+        )
+      })
+
+      it('should pass the error to next', function (ctx) {
+        sinon.assert.calledOnce(ctx.next)
+        expect(ctx.next.firstCall.args[0]).to.be.instanceOf(Error)
+      })
+
+      it('should still clean up the conversion directory', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), {
+          recursive: true,
+          force: true,
+        })
+      })
+    })
+  })
 })

services/clsi/test/unit/js/ConversionManager.test.js

@@ -250,4 +250,89 @@ describe('ConversionManager', function () {
       })
     })
   })
+
+  describe('convertLaTeXToDocumentInDirWithLock', function () {
+    describe('successfully', function () {
+      beforeEach(async function (ctx) {
+        ctx.compileDir = '/compiles/test-compile-dir'
+        ctx.rootDocPath = 'main.tex'
+        ctx.type = 'docx'
+        ctx.extension = 'docx'
+
+        ctx.result =
+          await ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
+            ctx.conversionId,
+            ctx.compileDir,
+            ctx.rootDocPath,
+            ctx.type,
+            ctx.extension
+          )
+      })
+
+      it('should acquire a lock on the compile dir', function (ctx) {
+        sinon.assert.calledWith(ctx.LockManager.acquire, ctx.compileDir)
+      })
+
+      it('should release the lock', function (ctx) {
+        sinon.assert.called(ctx.lock.release)
+      })
+
+      it('should run pandoc with correct arguments', function (ctx) {
+        expect(ctx.CommandRunner.promises.run.callCount).toBe(1)
+        expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
+          ctx.conversionId,
+          [
+            'pandoc',
+            ctx.rootDocPath,
+            '--output',
+            `output-uuid.${ctx.extension}`,
+            '--from',
+            'latex',
+            '--to',
+            ctx.type,
+            '--resource-path=.',
+          ],
+          ctx.compileDir,
+          ctx.Settings.pandocImage,
+          60_000,
+          {},
+          'conversions',
+        ])
+      })
+
+      it('should convert conversion timeout to milliseconds', function (ctx) {
+        expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
+      })
+
+      it('should return path to the output document', function (ctx) {
+        expect(ctx.result).toBe(
+          Path.join(ctx.compileDir, `output-uuid.${ctx.extension}`)
+        )
+      })
+    })
+
+    describe('when pandoc fails (non-zero exit code)', function () {
+      it('should reject with an error and release the lock', async function (ctx) {
+        ctx.compileDir = '/compiles/test-compile-dir'
+
+        ctx.CommandRunner.promises.run.resolves({
+          stdout: 'mock-stdout',
+          stderr: 'mock-stderr',
+          exitCode: 1,
+        })
+
+        await expect(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
+            ctx.conversionId,
+            ctx.compileDir,
+            'main.tex',
+            'docx',
+            'docx'
+          )
+        ).to.be.rejectedWith('pandoc latex-to-document conversion failed')
+
+        sinon.assert.called(ctx.lock.release)
+      })
+    })
+  })
 })