[web]+clsi] Allow docx import via pandoc (#32004)

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 246b3290ec04867f71545b1a7c5d95d0f68379ff

services/clsi/.gitignore

@@ -1,3 +1,4 @@
 compiles
 output
 cache
+uploads

services/clsi/Makefile

@@ -154,6 +154,8 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
+	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/clsi/app.js

@@ -20,6 +20,8 @@ import bodyParser from 'body-parser'
 import net from 'node:net'
 import os from 'node:os'
 import OError from '@overleaf/o-error'
+import ConversionController from './app/js/ConversionController.js'
+import FileUploadMiddleware from './app/js/FileUploadMiddleware.js'
 logger.initialize('clsi')
 logger.logger.serializers.clsiRequest = LoggerSerializers.clsiRequest
 
@@ -122,6 +124,13 @@ app.get(
   OutputController.createOutputZip
 )
 
+// Conversion endpoints
+app.post(
+  '/convert/docx-to-latex',
+  FileUploadMiddleware.multerMiddleware,
+  ConversionController.convertDocxToLaTeX
+)
+
 if (process.env.NODE_ENV === 'development' && global.__coverage__) {
   app.get('/coverage', (req, res) => {
     const coverage = {}

services/clsi/app/js/ConversionController.js

@@ -0,0 +1,46 @@
+import logger from '@overleaf/logger'
+import { expressify } from '@overleaf/promise-utils'
+import fs from 'node:fs/promises'
+import fsSync from 'node:fs'
+import ConversionManager from './ConversionManager.js'
+import { pipeline } from 'node:stream/promises'
+import Settings from '@overleaf/settings'
+import Path from 'node:path'
+
+async function convertDocxToLaTeX(req, res) {
+  const { path } = req.file
+  if (!Settings.enablePandocConversions) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(404)
+  }
+  logger.debug({ path }, 'received file for conversion')
+  const conversionId = crypto.randomUUID()
+  let zipPath
+  try {
+    zipPath = await ConversionManager.promises.convertDocxToLaTeXWithLock(
+      conversionId,
+      path
+    )
+  } finally {
+    await fs.unlink(path).catch(() => {})
+  }
+
+  try {
+    const zipStat = await fs.stat(zipPath)
+
+    res.setHeader('Content-Length', zipStat.size)
+    res.attachment('conversion.zip')
+    res.setHeader('X-Content-Type-Options', 'nosniff')
+
+    const readStream = fsSync.createReadStream(zipPath)
+    await pipeline(readStream, res)
+  } finally {
+    await fs
+      .rm(Path.dirname(zipPath), { recursive: true, force: true })
+      .catch(() => {})
+  }
+}
+
+export default {
+  convertDocxToLaTeX: expressify(convertDocxToLaTeX),
+}

services/clsi/app/js/ConversionManager.js

@@ -0,0 +1,101 @@
+import logger from '@overleaf/logger'
+import Settings from '@overleaf/settings'
+import fs from 'node:fs/promises'
+import Path from 'node:path'
+import CommandRunner from './CommandRunner.js'
+import LockManager from './LockManager.js'
+import OError from '@overleaf/o-error'
+
+async function convertDocxToLaTeXWithLock(conversionId, inputPath) {
+  const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
+  const lock = LockManager.acquire(conversionDir)
+  try {
+    return await convertDocxToLaTeX(conversionId, conversionDir, inputPath)
+  } finally {
+    lock.release()
+  }
+}
+
+async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
+  await fs.mkdir(conversionDir, { recursive: true })
+  const newSourcePath = Path.join(conversionDir, 'input.docx')
+  await fs.copyFile(inputPath, newSourcePath)
+  const outputName = crypto.randomUUID() + '.zip'
+
+  try {
+    const {
+      stdout: stdoutPandoc,
+      stderr: stderrPandoc,
+      exitCode: exitCodePandoc,
+    } = await CommandRunner.promises.run(
+      conversionId,
+      [
+        'pandoc',
+        'input.docx',
+        '--output',
+        'main.tex',
+        '--extract-media=.',
+        '--from',
+        'docx+citations',
+        '--to',
+        'latex',
+        '--citeproc',
+        '--standalone',
+      ],
+      conversionDir,
+      Settings.pandocImage,
+      Settings.conversionTimeoutSeconds * 1000,
+      {},
+      'conversions'
+    )
+    if (exitCodePandoc !== 0) {
+      throw new OError('Non-zero exit code from pandoc', {
+        exitCode: exitCodePandoc,
+        stderr: stderrPandoc,
+      })
+    }
+    logger.debug(
+      { stdout: stdoutPandoc, stderr: stderrPandoc, exitCode: exitCodePandoc },
+      'conversion command completed'
+    )
+
+    // Clean up the source document to leave only the conversion result
+    await fs.unlink(newSourcePath).catch(() => {})
+
+    const {
+      stdout: stdoutZip,
+      stderr: stderrZip,
+      exitCode: exitCodeZip,
+    } = await CommandRunner.promises.run(
+      conversionId,
+      ['zip', '-r', outputName, '.'],
+      conversionDir,
+      Settings.pandocImage,
+      Settings.conversionTimeoutSeconds * 1000,
+      {},
+      'conversions'
+    )
+    if (exitCodeZip !== 0) {
+      throw new OError('Non-zero exit code from pandoc', {
+        exitCode: exitCodeZip,
+        stderr: stderrZip,
+      })
+    }
+    logger.debug(
+      { stdout: stdoutZip, stderr: stderrZip, exitCode: exitCodeZip },
+      'conversion output compressed'
+    )
+  } catch (error) {
+    // Clean up the conversion directory on error to avoid leaving failed conversions around
+    await fs.rm(conversionDir, { force: true, recursive: true }).catch(() => {})
+    throw new OError('pandoc conversion failed').withCause(error)
+  }
+
+  return Path.join(conversionDir, outputName)
+}
+
+export default {
+  promises: {
+    convertDocxToLaTeXWithLock,
+  },
+}

services/clsi/app/js/FileUploadMiddleware.js

@@ -0,0 +1,31 @@
+import multer from 'multer'
+import Settings from '@overleaf/settings'
+import logger from '@overleaf/logger'
+
+const upload = multer({
+  dest: Settings.path.uploadFolder,
+  limits: {
+    fileSize: Settings.maxUploadSize,
+    parts: 2,
+  },
+})
+
+function multerMiddleware(req, res, next) {
+  return upload.single('qqfile')(req, res, function (err) {
+    if (err instanceof multer.MulterError && err.code === 'LIMIT_FILE_SIZE') {
+      return res.status(422).json({ success: false, error: 'file_too_large' })
+    }
+    if (err) return next(err)
+    if (!req.file?.path) {
+      logger.info({ req }, 'missing req.file.path on upload')
+      return res
+        .status(400)
+        .json({ success: false, error: 'invalid_upload_request' })
+    }
+    next()
+  })
+}
+
+export default {
+  multerMiddleware,
+}

services/clsi/app/js/LocalCommandRunner.js

@@ -82,7 +82,7 @@ export default CommandRunner = {
         err.code = code
         return callback(err)
       } else {
-        return callback(null, { stdout })
+        return callback(null, { stdout, exitCode: code })
       }
     })
 

services/clsi/buildscript.txt

@@ -1,7 +1,7 @@
 clsi
 --data-dirs=cache,compiles,output
 --dependencies=
---env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES="quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1",TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output
+--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES="quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9",TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.13.0

services/clsi/config/settings.defaults.cjs

@@ -20,11 +20,19 @@ module.exports = {
       process.env.CLSI_OUTPUT_PATH || Path.resolve(__dirname, '../output'),
     clsiCacheDir:
       process.env.CLSI_CACHE_PATH || Path.resolve(__dirname, '../cache'),
+    uploadFolder:
+      process.env.CLSI_UPLOAD_PATH || Path.resolve(__dirname, '../uploads'),
     synctexBaseDir(projectId) {
       return Path.join(this.compilesDir, projectId)
     },
   },
 
+  conversionTimeoutSeconds:
+    parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
+  pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
+  enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
+  maxUploadSize: 50 * 1024 * 1024,
+
   internal: {
     clsi: {
       port: 3013,
@@ -152,6 +160,7 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
       wordcount: { 'HostConfig.AutoRemove': true },
       synctex: { 'HostConfig.AutoRemove': true },
       'synctex-output': { 'HostConfig.AutoRemove': true },
+      conversions: { 'HostConfig.AutoRemove': true },
     }
     module.exports.clsi.docker.compileGroupConfig = Object.assign(
       defaultCompileGroupConfig,

services/clsi/docker-compose.ci.yml

@@ -31,13 +31,14 @@ services:
       ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file"
       ENABLE_PDF_CACHING: "true"
       PDF_CACHING_ENABLE_WORKER_POOL: "true"
-      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1"
+      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9"
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: "tex"
       SANDBOXED_COMPILES: "true"
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
+      ENABLE_PANDOC_CONVERSIONS: true
     volumes:
       - ./reports:/overleaf/services/clsi/reports
       - ./compiles:/overleaf/services/clsi/compiles

services/clsi/docker-compose.yml

@@ -45,13 +45,14 @@ services:
       ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file"
       ENABLE_PDF_CACHING: "true"
       PDF_CACHING_ENABLE_WORKER_POOL: "true"
-      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1"
+      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9"
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: "tex"
       SANDBOXED_COMPILES: "true"
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
+      ENABLE_PANDOC_CONVERSIONS: true
     depends_on:
       clsi-nginx:
         condition: service_started

services/clsi/entrypoint.sh

@@ -9,5 +9,6 @@ usermod -aG dockeronhost node
 mkdir -p /overleaf/services/clsi/cache && chown node:node /overleaf/services/clsi/cache
 mkdir -p /overleaf/services/clsi/compiles && chown node:node /overleaf/services/clsi/compiles
 mkdir -p /overleaf/services/clsi/output && chown node:node /overleaf/services/clsi/output
+mkdir -p /overleaf/services/clsi/uploads && chown node:node /overleaf/services/clsi/uploads
 
 exec runuser -u node -- "$@"

services/clsi/package.json

@@ -30,6 +30,7 @@
     "dockerode": "^4.0.9",
     "express": "4.22.1",
     "lodash": "^4.17.21",
+    "multer": "2.1.1",
     "overleaf-editor-core": "*",
     "p-limit": "^3.1.0",
     "request": "2.88.2",
@@ -41,6 +42,7 @@
     "@istanbuljs/esm-loader-hook": "^0.3.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
+    "form-data": "^4.0.5",
     "mocha": "^11.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
@@ -51,6 +53,7 @@
     "sinon-chai": "^3.7.0",
     "timekeeper": "2.2.0",
     "typescript": "^5.0.4",
-    "vitest": "^4.0.0"
+    "vitest": "^4.0.0",
+    "yauzl": "^2.10.0"
   }
 }

services/clsi/test/acceptance/js/ConversionTests.js

@@ -0,0 +1,83 @@
+import Client from './helpers/Client.js'
+import ClsiApp from './helpers/ClsiApp.js'
+import Path from 'node:path'
+import fs from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import yauzl from 'yauzl'
+import { expect } from 'chai'
+
+describe('Conversions', function () {
+  describe('docx conversion', function () {
+    before(async function () {
+      await ClsiApp.ensureRunning()
+      try {
+        this.body = await Client.compile(this.project_id, this.request)
+      } catch (error) {
+        this.error = error
+      }
+    })
+
+    it('should convert file to docx', async function () {
+      const sourcePath = Path.join(
+        import.meta.dirname,
+        '../fixtures/conversion-source.docx'
+      )
+      const outputStream = fs.createWriteStream(
+        '/tmp/clsi_acceptance_tests_' + crypto.randomUUID() + '.zip'
+      )
+      const stream = await Client.convertDocx(sourcePath)
+      await pipeline(stream, outputStream)
+
+      await new Promise((resolve, reject) => {
+        yauzl.open(outputStream.path, { lazyEntries: true }, (err, zipfile) => {
+          if (err) {
+            return reject(err)
+          }
+          zipfile.on('error', reject)
+          zipfile.on('end', resolve)
+          zipfile.readEntry()
+          zipfile.on('entry', entry => {
+            if (entry.fileName === 'main.tex') {
+              zipfile.openReadStream(entry, (err, readStream) => {
+                if (err) {
+                  return reject(err)
+                }
+                let data = ''
+                readStream.on('data', chunk => {
+                  data += chunk.toString()
+                })
+                readStream.on('end', () => {
+                  try {
+                    expect(data).to.include('\\begin{document}')
+                    expect(data).to.include(
+                      '\\[x = \\frac{- b \\pm \\sqrt{b^{2} - 4ac}}{2a}\\]'
+                    )
+                    zipfile.readEntry()
+                  } catch (err) {
+                    reject(err)
+                  }
+                })
+              })
+            } else if (entry.fileName === 'media/') {
+              // Skip the media directory entry
+              zipfile.readEntry()
+            } else if (entry.fileName.startsWith('media/')) {
+              expect(entry.fileName).to.equal('media/image1.png')
+              zipfile.readEntry()
+            } else {
+              reject(new Error('Unexpected file in zip: ' + entry.fileName))
+            }
+          })
+        })
+      })
+    })
+
+    it('should fail when file is not a docx', async function () {
+      const sourcePath = Path.join(
+        import.meta.dirname,
+        '../fixtures/minimal.pdf'
+      )
+      await expect(Client.convertDocx(sourcePath)).to.eventually.be.rejected
+    })
+  })
+})

services/clsi/test/acceptance/js/helpers/Client.js

@@ -1,8 +1,14 @@
 import express from 'express'
-import { fetchJson, fetchNothing, fetchString } from '@overleaf/fetch-utils'
+import {
+  fetchJson,
+  fetchNothing,
+  fetchStream,
+  fetchString,
+} from '@overleaf/fetch-utils'
 import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
 import Settings from '@overleaf/settings'
+import FormData from 'form-data'
 
 const host = Settings.apis.clsi.url
 
@@ -24,6 +30,15 @@ function compile(projectId, data) {
   })
 }
 
+async function convertDocx(path) {
+  const formData = new FormData()
+  formData.append('qqfile', fs.createReadStream(path))
+  return await fetchStream(`${host}/convert/docx-to-latex`, {
+    method: 'POST',
+    body: formData,
+  })
+}
+
 async function stopCompile(projectId) {
   return await fetchNothing(`${host}/project/${projectId}/compile/stop`, {
     method: 'POST',
@@ -187,6 +202,7 @@ function smokeTest() {
 export default {
   randomId,
   compile,
+  convertDocx,
   stopCompile,
   clearCache,
   getOutputFile,

services/clsi/test/acceptance/js/helpers/ClsiApp.js

@@ -1,5 +1,6 @@
 import app from '../../../../app.js'
 import Settings from '@overleaf/settings'
+import testLogRecorder from '@overleaf/logger/test-log-recorder.js'
 
 function startApp() {
   return new Promise((resolve, reject) => {
@@ -26,6 +27,10 @@ async function ensureRunning() {
   await appStartedPromise
 }
 
+if (process.env.CI === 'true') {
+  beforeEach('record error logs in junit', testLogRecorder)
+}
+
 export default {
   ensureRunning,
 }

services/clsi/test/unit/js/ConversionController.test.js

@@ -0,0 +1,158 @@
+import sinon from 'sinon'
+import { vi, describe, it, beforeEach, expect } from 'vitest'
+import Path from 'node:path'
+import { PassThrough } from 'node:stream'
+
+const MODULE_PATH = Path.join(
+  import.meta.dirname,
+  '../../../app/js/ConversionController'
+)
+
+describe('ConversionController', function () {
+  beforeEach(async function (ctx) {
+    ctx.conversionDir = '/path/to/conversion/result'
+    ctx.zipPath = '/path/to/conversion/result/output.zip'
+    ctx.zipStat = { size: 1234 }
+    ctx.Settings = {
+      enablePandocConversions: true,
+    }
+    ctx.ConversionManager = {
+      promises: {
+        convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
+      },
+    }
+
+    ctx.fs = {
+      stat: sinon.stub().resolves(ctx.zipStat),
+      unlink: sinon.stub().resolves(),
+      rm: sinon.stub().resolves(),
+    }
+
+    ctx.readStream = new PassThrough()
+    ctx.fsSync = {
+      createReadStream: sinon.stub().returns(ctx.readStream),
+    }
+    ctx.pipeline = sinon.stub().resolves()
+
+    vi.doMock('node:fs/promises', () => ({
+      default: ctx.fs,
+    }))
+
+    vi.doMock('node:fs', () => ({
+      default: ctx.fsSync,
+    }))
+
+    vi.doMock('node:stream/promises', () => ({
+      pipeline: ctx.pipeline,
+    }))
+
+    vi.doMock('@overleaf/settings', () => ({
+      default: ctx.Settings,
+    }))
+
+    vi.doMock('../../../app/js/ConversionManager', () => ({
+      default: ctx.ConversionManager,
+    }))
+
+    ctx.res = new PassThrough()
+    ctx.res.attachment = sinon.stub()
+    ctx.res.setHeader = sinon.stub()
+
+    ctx.ConversionController = (await import(MODULE_PATH)).default
+  })
+
+  describe('convertDocxToLaTeX', function () {
+    describe('when conversions are disabled', function () {
+      beforeEach(async function (ctx) {
+        ctx.Settings.enablePandocConversions = false
+        ctx.req = {
+          file: { path: '/path/to/uploaded/file.docx' },
+        }
+        ctx.res.sendStatus = sinon.stub()
+
+        await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
+      })
+
+      it('should remove the uploaded file', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.unlink, ctx.req.file.path)
+      })
+
+      it('should return 404', function (ctx) {
+        sinon.assert.calledWith(ctx.res.sendStatus, 404)
+      })
+
+      it('should not call the conversion manager', function (ctx) {
+        sinon.assert.notCalled(
+          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock
+        )
+      })
+    })
+
+    describe('successfully', function () {
+      beforeEach(async function (ctx) {
+        ctx.req = {
+          file: { path: '/path/to/uploaded/file.docx' },
+        }
+
+        await ctx.ConversionController.convertDocxToLaTeX(ctx.req, ctx.res)
+      })
+
+      it('should call the conversion manager with the uploaded file path', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock,
+          sinon.match(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
+          ),
+          ctx.req.file.path
+        )
+      })
+
+      it('should look up the generated zip file size', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.stat, ctx.zipPath)
+      })
+
+      it('should set the response headers for a zip file download', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.res.setHeader,
+          'Content-Length',
+          ctx.zipStat.size
+        )
+        sinon.assert.calledWith(ctx.res.attachment, 'conversion.zip')
+        sinon.assert.calledWith(
+          ctx.res.setHeader,
+          'X-Content-Type-Options',
+          'nosniff'
+        )
+      })
+
+      it('should stream the generated zip file to the response', function (ctx) {
+        sinon.assert.calledWith(ctx.fsSync.createReadStream, ctx.zipPath)
+        sinon.assert.calledWith(ctx.pipeline, ctx.readStream, ctx.res)
+      })
+
+      it('should clean up the generated zip file', function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir)
+      })
+    })
+
+    describe('unsuccessfully', function () {
+      describe('on streaming error', function () {
+        it('should propagate the error and still clean up', async function (ctx) {
+          ctx.pipeline.rejects(new Error('mock stream error'))
+
+          const res = new PassThrough()
+          res.attachment = sinon.stub()
+          res.setHeader = sinon.stub()
+
+          const req = { file: { path: '/path/to/uploaded/file.docx' } }
+
+          await expect(
+            ctx.ConversionController.convertDocxToLaTeX(req, res)
+          ).to.be.rejectedWith('mock stream error')
+
+          sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir)
+        })
+      })
+    })
+  })
+})

services/clsi/test/unit/js/ConversionManager.test.js

@@ -0,0 +1,253 @@
+import Path from 'node:path'
+import sinon from 'sinon'
+import { vi, describe, beforeEach, afterEach, it, expect } from 'vitest'
+const MODULE_PATH = Path.join(
+  import.meta.dirname,
+  '../../../app/js/ConversionManager'
+)
+
+describe('ConversionManager', function () {
+  beforeEach(async function (ctx) {
+    ctx.CommandRunner = {
+      promises: {
+        run: sinon.stub().resolves({ stdout: '', stderr: '', exitCode: 0 }),
+      },
+    }
+
+    ctx.lock = {
+      release: sinon.stub(),
+    }
+
+    ctx.LockManager = {
+      acquire: sinon.stub().returns(ctx.lock),
+    }
+
+    ctx.Settings = {
+      pandocImage: 'mock-pandoc-image',
+      conversionTimeoutSeconds: 60,
+      path: { compilesDir: '/compiles' },
+    }
+
+    ctx.fs = {
+      mkdir: sinon.stub().resolves(),
+      copyFile: sinon.stub().resolves(),
+      rm: sinon.stub().resolves(),
+      unlink: sinon.stub().resolves(),
+    }
+
+    ctx.conversionId = 'test-conversion-id'
+    ctx.inputPath = '/path/to/input.docx'
+    ctx.conversionDir = '/compiles/test-conversion-id'
+    ctx.outputPath = '/compiles/test-conversion-id/output-uuid.zip'
+
+    ctx.uuidStub = sinon
+      .stub(globalThis.crypto, 'randomUUID')
+      .returns('output-uuid')
+
+    vi.doMock('../../../app/js/LockManager', () => ({
+      default: ctx.LockManager,
+    }))
+
+    vi.doMock('@overleaf/settings', () => ({
+      default: ctx.Settings,
+    }))
+
+    vi.doMock('../../../app/js/CommandRunner', () => ({
+      default: ctx.CommandRunner,
+    }))
+
+    vi.doMock('node:fs/promises', () => ({ default: ctx.fs }))
+
+    ctx.ConversionManager = (await import(MODULE_PATH)).default
+  })
+
+  afterEach(function (ctx) {
+    ctx.uuidStub.restore()
+  })
+
+  describe('convertDocxToLaTeXWithLock', function () {
+    describe('general behavior', function () {
+      beforeEach(async function (ctx) {
+        ctx.result =
+          await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            ctx.conversionId,
+            ctx.inputPath
+          )
+      })
+
+      it('should acquire a lock', async function (ctx) {
+        sinon.assert.calledWith(ctx.LockManager.acquire, ctx.conversionDir)
+      })
+
+      it('should copy the input file to the conversion directory', async function (ctx) {
+        sinon.assert.calledWith(ctx.fs.mkdir, ctx.conversionDir, {
+          recursive: true,
+        })
+        sinon.assert.calledWith(
+          ctx.fs.copyFile,
+          ctx.inputPath,
+          Path.join(ctx.conversionDir, 'input.docx')
+        )
+      })
+
+      it('should convert conversion timeout to milliseconds', async function (ctx) {
+        expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
+        expect(ctx.CommandRunner.promises.run.secondCall.args[4]).toBe(60_000)
+      })
+
+      it('should run pandoc followed by zip in the conversion directory', function (ctx) {
+        expect(ctx.CommandRunner.promises.run.callCount).toBe(2)
+        expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
+          ctx.conversionId,
+          [
+            'pandoc',
+            'input.docx',
+            '--output',
+            'main.tex',
+            '--extract-media=.',
+            '--from',
+            'docx+citations',
+            '--to',
+            'latex',
+            '--citeproc',
+            '--standalone',
+          ],
+          ctx.conversionDir,
+          ctx.Settings.pandocImage,
+          60_000,
+          {},
+          'conversions',
+        ])
+        expect(ctx.CommandRunner.promises.run.secondCall.args).toEqual([
+          ctx.conversionId,
+          ['zip', '-r', 'output-uuid.zip', '.'],
+          ctx.conversionDir,
+          ctx.Settings.pandocImage,
+          60_000,
+          {},
+          'conversions',
+        ])
+      })
+    })
+
+    describe('successful conversion', function () {
+      beforeEach(async function (ctx) {
+        ctx.CommandRunner.promises.run.resolves({
+          stdout: 'mock-stdout',
+          stderr: 'mock-stderr',
+          exitCode: 0,
+        })
+
+        ctx.result =
+          await ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            ctx.conversionId,
+            ctx.inputPath
+          )
+      })
+
+      it('should remove the source document after conversion', async function (ctx) {
+        sinon.assert.calledWith(
+          ctx.fs.unlink,
+          Path.join(ctx.conversionDir, 'input.docx')
+        )
+      })
+
+      it('should return the conversion directory', function (ctx) {
+        expect(ctx.result).toBe(ctx.outputPath)
+      })
+
+      it('should release the lock', function (ctx) {
+        sinon.assert.called(ctx.lock.release)
+      })
+    })
+
+    describe('unsuccessful conversion (exitcode)', function () {
+      beforeEach(async function (ctx) {
+        ctx.CommandRunner.promises.run.resolves({
+          stdout: 'mock-stdout',
+          stderr: 'mock-stderr',
+          exitCode: 63,
+        })
+
+        await expect(
+          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            ctx.conversionId,
+            ctx.inputPath
+          )
+        ).to.be.rejectedWith('pandoc conversion failed')
+      })
+
+      it('should remove the entire conversion directory', async function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+          force: true,
+          recursive: true,
+        })
+      })
+
+      it('should release the lock', function (ctx) {
+        sinon.assert.called(ctx.lock.release)
+      })
+    })
+
+    describe('unsuccessful compression (exitcode)', function () {
+      beforeEach(async function (ctx) {
+        ctx.CommandRunner.promises.run
+          .onFirstCall()
+          .resolves({
+            stdout: 'mock-pandoc-stdout',
+            stderr: 'mock-pandoc-stderr',
+            exitCode: 0,
+          })
+          .onSecondCall()
+          .resolves({
+            stdout: 'mock-zip-stdout',
+            stderr: 'mock-zip-stderr',
+            exitCode: 12,
+          })
+
+        await expect(
+          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            ctx.conversionId,
+            ctx.inputPath
+          )
+        ).to.be.rejectedWith('pandoc conversion failed')
+      })
+
+      it('should remove the entire conversion directory', async function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+          force: true,
+          recursive: true,
+        })
+      })
+
+      it('should release the lock', function (ctx) {
+        sinon.assert.called(ctx.lock.release)
+      })
+    })
+
+    describe('unsuccessful conversion (throws)', function () {
+      beforeEach(async function (ctx) {
+        ctx.CommandRunner.promises.run.rejects(
+          new Error('mock conversion error')
+        )
+        await expect(
+          ctx.ConversionManager.promises.convertDocxToLaTeXWithLock(
+            ctx.conversionId,
+            ctx.inputPath
+          )
+        ).to.be.rejectedWith('pandoc conversion failed')
+      })
+
+      it('should remove the entire conversion directory', async function (ctx) {
+        sinon.assert.calledWith(ctx.fs.rm, ctx.conversionDir, {
+          force: true,
+          recursive: true,
+        })
+      })
+
+      it('should release the lock', function (ctx) {
+        sinon.assert.called(ctx.lock.release)
+      })
+    })
+  })
+})

services/clsi/test/unit/js/DockerRunner.test.js

@@ -123,7 +123,7 @@ describe('DockerRunner', () => {
         await new Promise((resolve, reject) => {
           ctx.DockerRunner._runAndWaitForContainer = sinon
             .stub()
-            .callsArgWith(3, null, (ctx.output = 'mock-output'))
+            .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
           return ctx.DockerRunner.run(
             ctx.project_id,
             ctx.command,
@@ -168,7 +168,7 @@ describe('DockerRunner', () => {
         ctx.directory = '/var/lib/overleaf/data/compiles/xyz'
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         return ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -199,7 +199,7 @@ describe('DockerRunner', () => {
         ctx.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id'
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -230,7 +230,7 @@ describe('DockerRunner', () => {
         ctx.directory = '/var/lib/overleaf/data/compile/xyz'
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -261,7 +261,7 @@ describe('DockerRunner', () => {
         ctx.directory = '/var/lib/overleaf/data/compile/xyz'
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -290,7 +290,7 @@ describe('DockerRunner', () => {
     describe('when the run throws an error', () => {
       beforeEach(ctx => {
         let firstTime = true
-        ctx.output = 'mock-output'
+        ctx.output = { stdout: 'mock-output' }
         ctx.DockerRunner._runAndWaitForContainer = (
           options,
           volumes,
@@ -342,7 +342,7 @@ describe('DockerRunner', () => {
       beforeEach(ctx => {
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -372,7 +372,7 @@ describe('DockerRunner', () => {
         ctx.Settings.texliveImageNameOveride = 'overrideimage.com/something'
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -399,7 +399,7 @@ describe('DockerRunner', () => {
         ]
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
       })
 
       describe('with a valid image', () => {
@@ -477,7 +477,7 @@ describe('DockerRunner', () => {
         }
         ctx.DockerRunner._runAndWaitForContainer = sinon
           .stub()
-          .callsArgWith(3, null, (ctx.output = 'mock-output'))
+          .callsArgWith(3, null, (ctx.output = { stdout: 'mock-output' }))
         ctx.DockerRunner.run(
           ctx.project_id,
           ctx.command,
@@ -520,7 +520,7 @@ describe('DockerRunner', () => {
         attachStreamHandler,
         callback
       ) => {
-        attachStreamHandler(null, (ctx.output = 'mock-output'))
+        attachStreamHandler(null, (ctx.output = { stdout: 'mock-output' }))
         callback(null, (ctx.containerId = 'container-id'))
       }
       sinon.spy(ctx.DockerRunner, 'startContainer')