From d99ba08d0194163e0342048b7caa8c8e90669654 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 9 Apr 2025 13:50:37 +0100 Subject: [PATCH] [clsi] run SyncTeX in specific output dir rather than compile dir (#24690) * [clsi] drop support for docker-in-docker * [clsi] run SyncTeX in specific output dir rather than compile dir * [clsi] store output.synctex.gz outside of tar-ball in clsi-cache * [clsi] add documentation for rewriting of docker bind-mounts * [server-pro] update env vars for sandboxed compiles in sample config GitOrigin-RevId: 8debd7102ac612544961f237aa4ff1c530aa3da3 --- develop/docker-compose.yml | 1 + docker-compose.yml | 4 +- services/clsi/README.md | 1 + services/clsi/app/js/CLSICacheHandler.js | 111 ++++++----- services/clsi/app/js/CompileManager.js | 102 +++++----- services/clsi/app/js/DockerRunner.js | 103 ++++------ services/clsi/app/js/LocalCommandRunner.js | 4 + services/clsi/app/js/OutputCacheManager.js | 8 + services/clsi/buildscript.txt | 2 +- services/clsi/config/settings.defaults.js | 6 +- services/clsi/docker-compose.ci.yml | 1 + services/clsi/docker-compose.yml | 1 + .../test/unit/js/CompileControllerTests.js | 2 +- .../clsi/test/unit/js/CompileManagerTests.js | 3 +- .../clsi/test/unit/js/DockerRunnerTests.js | 179 ++++++++++-------- .../src/Features/Compile/ClsiCacheHandler.js | 1 + 16 files changed, 286 insertions(+), 243 deletions(-) diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index d0dc8ec6da..e37999f71d 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -29,6 +29,7 @@ services: - DOCKER_RUNNER=true - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full - COMPILES_HOST_DIR=${PWD}/compiles + - OUTPUT_HOST_DIR=${PWD}/output user: root volumes: - ${PWD}/compiles:/overleaf/services/clsi/compiles diff --git a/docker-compose.yml b/docker-compose.yml index 08d6db6fe7..a99eb7e0a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,9 @@ services: SANDBOXED_COMPILES: 'true' SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true' ### Bind-mount source for /var/lib/overleaf/data/compiles inside the container. - SANDBOXED_COMPILES_HOST_DIR: '/home/user/sharelatex_data/data/compiles' + SANDBOXED_COMPILES_HOST_DIR_COMPILES: '/home/user/sharelatex_data/data/compiles' + ### Bind-mount source for /var/lib/overleaf/data/output inside the container. + SANDBOXED_COMPILES_HOST_DIR_OUTPUT: '/home/user/sharelatex_data/data/output' ## Works with test LDAP server shown at bottom of docker compose # OVERLEAF_LDAP_URL: 'ldap://ldap:389' diff --git a/services/clsi/README.md b/services/clsi/README.md index 33a9c95c1c..16e40b8990 100644 --- a/services/clsi/README.md +++ b/services/clsi/README.md @@ -20,6 +20,7 @@ The CLSI can be configured through the following environment variables: * `CATCH_ERRORS` - Set to `true` to log uncaught exceptions * `COMPILE_GROUP_DOCKER_CONFIGS` - JSON string of Docker configs for compile groups * `COMPILES_HOST_DIR` - Working directory for LaTeX compiles +* `OUTPUT_HOST_DIR` - Output directory for LaTeX compiles * `COMPILE_SIZE_LIMIT` - Sets the body-parser [limit](https://github.com/expressjs/body-parser#limit) * `DOCKER_RUNNER` - Set to true to use sibling containers * `DOCKER_RUNTIME` - diff --git a/services/clsi/app/js/CLSICacheHandler.js b/services/clsi/app/js/CLSICacheHandler.js index 42d902c8e9..38a04d81ac 100644 --- a/services/clsi/app/js/CLSICacheHandler.js +++ b/services/clsi/app/js/CLSICacheHandler.js @@ -1,3 +1,4 @@ +const crypto = require('node:crypto') const fs = require('node:fs') const Path = require('node:path') const { pipeline } = require('node:stream/promises') @@ -71,6 +72,7 @@ function notifyCLSICacheAboutBuild({ f => f.path === 'output.pdf' || f.path === 'output.log' || + f.path === 'output.synctex.gz' || f.path.endsWith('.blg') ) .map(f => { @@ -110,9 +112,7 @@ async function buildTarball({ projectId, userId, buildId, outputFiles }) { buildId ) - const files = outputFiles.filter( - f => f.path === 'output.synctex.gz' || !isExtraneousFile(f.path) - ) + const files = outputFiles.filter(f => !isExtraneousFile(f.path)) if (files.length > MAX_ENTRIES_IN_OUTPUT_TAR) { Metrics.inc('clsi_cache_build_too_many_entries') throw new Error('too many output files for output.tar.gz') @@ -136,59 +136,80 @@ async function buildTarball({ projectId, userId, buildId, outputFiles }) { } } -/** - * @param {string} projectId - * @param {string} userId - * @param {string} compileDir - * @return {Promise} - */ -async function downloadLatestCompileCache(projectId, userId, compileDir) { - return await _downloadCompileCache( - `${Settings.apis.clsiCache.url}/project/${projectId}/${ - userId ? `user/${userId}/` : '' - }latest/output/output.tar.gz`, - compileDir, - 'tar' - ) -} - /** * @param {string} projectId * @param {string} userId * @param {string} editorId * @param {string} buildId - * @param {string} compileDir + * @param {string} outputDir * @return {Promise} */ -async function downloadOldCompileCache( +async function downloadOutputDotSynctexFromCompileCache( projectId, userId, editorId, buildId, - compileDir + outputDir ) { - return await _downloadCompileCache( - `${Settings.apis.clsiCache.url}/project/${projectId}/${ - userId ? `user/${userId}/` : '' - }build/${editorId}-${buildId}/search/output/output.tar.gz`, - compileDir, - 'synctex' - ) -} - -/** - * @param {string} url - * @param {string} compileDir - * @param {string} method - * @return {Promise} - */ -async function _downloadCompileCache(url, compileDir, method) { if (!Settings.apis.clsiCache.enabled) return false const timer = new Metrics.Timer( 'clsi_cache_download', 1, - { status: 'success', method }, + { method: 'synctex' }, + TIMING_BUCKETS + ) + let stream + try { + stream = await fetchStream( + `${Settings.apis.clsiCache.url}/project/${projectId}/${ + userId ? `user/${userId}/` : '' + }build/${editorId}-${buildId}/search/output/output.synctex.gz`, + { + method: 'GET', + signal: AbortSignal.timeout(10_000), + } + ) + } catch (err) { + if (err instanceof RequestFailedError && err.response.status === 404) { + timer.done({ status: 'not-found' }) + return false + } + timer.done({ status: 'error' }) + throw err + } + await fs.promises.mkdir(outputDir, { recursive: true }) + const dst = Path.join(outputDir, 'output.synctex.gz') + const tmp = dst + crypto.randomUUID() + try { + await pipeline(stream, fs.createWriteStream(tmp)) + await fs.promises.rename(tmp, dst) + } catch (err) { + try { + await fs.promises.unlink(tmp) + } catch {} + throw err + } + timer.done({ status: 'success' }) + return true +} + +/** + * @param {string} projectId + * @param {string} userId + * @param {string} compileDir + * @return {Promise} + */ +async function downloadLatestCompileCache(projectId, userId, compileDir) { + if (!Settings.apis.clsiCache.enabled) return false + + const url = `${Settings.apis.clsiCache.url}/project/${projectId}/${ + userId ? `user/${userId}/` : '' + }latest/output/output.tar.gz` + const timer = new Metrics.Timer( + 'clsi_cache_download', + 1, + { method: 'tar' }, TIMING_BUCKETS ) let stream @@ -199,12 +220,10 @@ async function _downloadCompileCache(url, compileDir, method) { }) } catch (err) { if (err instanceof RequestFailedError && err.response.status === 404) { - timer.labels.status = 'not-found' - timer.done() + timer.done({ status: 'not-found' }) return false } - timer.labels.status = 'error' - timer.done() + timer.done({ status: 'error' }) throw err } let n = 0 @@ -224,7 +243,6 @@ async function _downloadCompileCache(url, compileDir, method) { { url, compileDir, - method, }, 'too many entries in tar-ball from clsi-cache' ) @@ -234,7 +252,6 @@ async function _downloadCompileCache(url, compileDir, method) { { url, compileDir, - method, entryType: header.type, }, 'unexpected entry in tar-ball from clsi-cache' @@ -245,12 +262,12 @@ async function _downloadCompileCache(url, compileDir, method) { }) ) Metrics.count('clsi_cache_download_entries', n) - timer.done() + timer.done({ status: 'success' }) return !abort } module.exports = { notifyCLSICacheAboutBuild, downloadLatestCompileCache, - downloadOldCompileCache, + downloadOutputDotSynctexFromCompileCache, } diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 1e0dce1eef..b256195c9b 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -21,7 +21,7 @@ const { emitPdfStats } = require('./ContentCacheMetrics') const SynctexOutputParser = require('./SynctexOutputParser') const { downloadLatestCompileCache, - downloadOldCompileCache, + downloadOutputDotSynctexFromCompileCache, } = require('./CLSICacheHandler') const COMPILE_TIME_BUCKETS = [ @@ -509,58 +509,72 @@ async function _runSynctex(projectId, userId, command, opts) { throw new Errors.InvalidParameter('invalid buildId') } - const directory = getCompileDir(projectId, userId) + const outputDir = getOutputDir(projectId, userId) + const runInOutputDir = buildId && CommandRunner.canRunSyncTeXInOutputDir() + + const directory = runInOutputDir + ? Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR, buildId) + : getCompileDir(projectId, userId) const timeout = 60 * 1000 // increased to allow for large projects const compileName = getCompileName(projectId, userId) - const compileGroup = 'synctex' + const compileGroup = runInOutputDir ? 'synctex-output' : 'synctex' const defaultImageName = Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.image - try { - await _checkFileExists(directory, 'output.synctex.gz') - } catch (err) { - if ( - err instanceof Errors.NotFoundError && - compileFromClsiCache && - editorId && - buildId - ) { + // eslint-disable-next-line @typescript-eslint/return-await + return await OutputCacheManager.promises.queueDirOperation( + outputDir, + /** + * @return {Promise} + */ + async () => { try { - await downloadOldCompileCache( + await _checkFileExists(directory, 'output.synctex.gz') + } catch (err) { + if ( + err instanceof Errors.NotFoundError && + compileFromClsiCache && + editorId && + buildId + ) { + try { + await downloadOutputDotSynctexFromCompileCache( + projectId, + userId, + editorId, + buildId, + directory + ) + } catch (err) { + logger.warn( + { err, projectId, userId, editorId, buildId }, + 'failed to download output.synctex.gz from clsi-cache' + ) + } + await _checkFileExists(directory, 'output.synctex.gz') + } else { + throw err + } + } + try { + const output = await CommandRunner.promises.run( + compileName, + command, + directory, + imageName || defaultImageName, + timeout, + {}, + compileGroup + ) + return output.stdout + } catch (error) { + throw OError.tag(error, 'error running synctex', { + command, projectId, userId, - editorId, - buildId, - directory - ) - } catch (err) { - logger.warn( - { err, projectId, userId, editorId, buildId }, - 'failed to populate compile dir for synctex using old output' - ) + }) } - await _checkFileExists(directory, 'output.synctex.gz') - } else { - throw err } - } - try { - const output = await CommandRunner.promises.run( - compileName, - command, - directory, - imageName || defaultImageName, - timeout, - {}, - compileGroup - ) - return output.stdout - } catch (error) { - throw OError.tag(error, 'error running synctex', { - command, - projectId, - userId, - }) - } + ) } async function wordcount(projectId, userId, filename, image) { diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js index 7aac613db4..def02eaf5b 100644 --- a/services/clsi/app/js/DockerRunner.js +++ b/services/clsi/app/js/DockerRunner.js @@ -6,21 +6,12 @@ const dockerode = new Docker() const crypto = require('node:crypto') const async = require('async') const LockManager = require('./DockerLockManager') -const fs = require('node:fs') const Path = require('node:path') const _ = require('lodash') const ONE_HOUR_IN_MS = 60 * 60 * 1000 logger.debug('using docker runner') -function usingSiblingContainers() { - return ( - Settings != null && - Settings.path != null && - Settings.path.sandboxedCompilesHostDir != null - ) -} - let containerMonitorTimeout let containerMonitorInterval @@ -35,24 +26,6 @@ const DockerRunner = { compileGroup, callback ) { - if (usingSiblingContainers()) { - const _newPath = Settings.path.sandboxedCompilesHostDir - logger.debug( - { path: _newPath }, - 'altering bind path for sibling containers' - ) - // Server Pro, example: - // '/var/lib/overleaf/data/compiles/' - // ... becomes ... - // '/opt/overleaf_data/data/compiles/' - directory = Path.join( - Settings.path.sandboxedCompilesHostDir, - Path.basename(directory) - ) - } - - const volumes = { [directory]: '/compile' } - command = command.map(arg => arg.toString().replace('$COMPILE_DIR', '/compile') ) @@ -72,7 +45,32 @@ const DockerRunner = { image = `${Settings.texliveImageNameOveride}/${img[2]}` } - if (compileGroup === 'synctex' || compileGroup === 'wordcount') { + if (compileGroup === 'synctex-output') { + // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId' + // directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId' + // sandboxedCompilesHostDirOutput = '/host/output' + // Out: directory = '/host/output/projectId-userId/generated-files/buildId' + directory = Path.join( + Settings.path.sandboxedCompilesHostDirOutput, + ...directory.split('/').slice(-3) + ) + } else { + // In: directory = '/overleaf/services/clsi/compiles/projectId-userId' + // Path.basename(directory) === 'projectId-userId' + // sandboxedCompilesHostDirCompiles = '/host/compiles' + // Out: directory = '/host/compiles/projectId-userId' + directory = Path.join( + Settings.path.sandboxedCompilesHostDirCompiles, + Path.basename(directory) + ) + } + + const volumes = { [directory]: '/compile' } + if ( + compileGroup === 'synctex' || + compileGroup === 'synctex-output' || + compileGroup === 'wordcount' + ) { volumes[directory] += ':ro' } @@ -309,50 +307,17 @@ const DockerRunner = { LockManager.runWithLock( options.name, releaseLock => - // Check that volumes exist before starting the container. - // When a container is started with volume pointing to a - // non-existent directory then docker creates the directory but - // with root ownership. - DockerRunner._checkVolumes(options, volumes, err => { - if (err != null) { - return releaseLock(err) - } - DockerRunner._startContainer( - options, - volumes, - attachStreamHandler, - releaseLock - ) - }), - + DockerRunner._startContainer( + options, + volumes, + attachStreamHandler, + releaseLock + ), callback ) }, // Check that volumes exist and are directories - _checkVolumes(options, volumes, callback) { - if (usingSiblingContainers()) { - // Server Pro, with sibling-containers active, skip checks - return callback(null) - } - - const checkVolume = (path, cb) => - fs.stat(path, (err, stats) => { - if (err != null) { - return cb(err) - } - if (!stats.isDirectory()) { - return cb(new Error('not a directory')) - } - cb() - }) - const jobs = [] - for (const vol in volumes) { - jobs.push(cb => checkVolume(vol, cb)) - } - async.series(jobs, callback) - }, - _startContainer(options, volumes, attachStreamHandler, callback) { callback = _.once(callback) const { name } = options @@ -617,6 +582,10 @@ const DockerRunner = { containerMonitorInterval = undefined } }, + + canRunSyncTeXInOutputDir() { + return Boolean(Settings.path.sandboxedCompilesHostDirOutput) + }, } DockerRunner.startContainerMonitor() diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js index bac7d39400..ce27473358 100644 --- a/services/clsi/app/js/LocalCommandRunner.js +++ b/services/clsi/app/js/LocalCommandRunner.js @@ -99,6 +99,10 @@ module.exports = CommandRunner = { } return callback() }, + + canRunSyncTeXInOutputDir() { + return true + }, } module.exports.promises = { diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js index 1e9a10c921..a1a0a89aa7 100644 --- a/services/clsi/app/js/OutputCacheManager.js +++ b/services/clsi/app/js/OutputCacheManager.js @@ -83,6 +83,13 @@ async function cleanupDirectory(dir, options) { }) } +/** + * @template T + * + * @param {string} dir + * @param {() => Promise} fn + * @return {Promise} + */ async function queueDirOperation(dir, fn) { const pending = PENDING_PROJECT_ACTIONS.get(dir) || Promise.resolve() const p = pending.then(fn, fn).finally(() => { @@ -677,4 +684,5 @@ OutputCacheManager.promises = { saveOutputFilesInBuildDir: promisify( OutputCacheManager.saveOutputFilesInBuildDir ), + queueDirOperation, } diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 756d5c3bb6..66768bb367 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -2,7 +2,7 @@ clsi --data-dirs=cache,compiles,output --dependencies= --docker-repos=gcr.io/overleaf-ops,us-east1-docker.pkg.dev/overleaf-ops/ol-docker ---env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles +--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles,OUTPUT_HOST_DIR=$PWD/output --env-pass-through= --esmock-loader=False --node-version=20.18.2 diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index 51d13f9c48..0c29eaa98a 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -131,6 +131,7 @@ if (process.env.DOCKER_RUNNER) { const defaultCompileGroupConfig = { wordcount: { 'HostConfig.AutoRemove': true }, synctex: { 'HostConfig.AutoRemove': true }, + 'synctex-output': { 'HostConfig.AutoRemove': true }, } module.exports.clsi.docker.compileGroupConfig = Object.assign( defaultCompileGroupConfig, @@ -175,5 +176,8 @@ if (process.env.DOCKER_RUNNER) { module.exports.path.synctexBaseDir = () => '/compile' - module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR + module.exports.path.sandboxedCompilesHostDirCompiles = + process.env.COMPILES_HOST_DIR + module.exports.path.sandboxedCompilesHostDirOutput = + process.env.OUTPUT_HOST_DIR } diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 00f54c6e72..1754a3a916 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -31,6 +31,7 @@ services: TEXLIVE_IMAGE_USER: "tex" DOCKER_RUNNER: "true" COMPILES_HOST_DIR: $PWD/compiles + OUTPUT_HOST_DIR: $PWD/output volumes: - ./compiles:/overleaf/services/clsi/compiles - /var/run/docker.sock:/var/run/docker.sock diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index a525c6029e..3e70c256ea 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -49,5 +49,6 @@ services: TEXLIVE_IMAGE_USER: "tex" DOCKER_RUNNER: "true" COMPILES_HOST_DIR: $PWD/compiles + OUTPUT_HOST_DIR: $PWD/output command: npm run --silent test:acceptance diff --git a/services/clsi/test/unit/js/CompileControllerTests.js b/services/clsi/test/unit/js/CompileControllerTests.js index b06679d994..03cd9932e9 100644 --- a/services/clsi/test/unit/js/CompileControllerTests.js +++ b/services/clsi/test/unit/js/CompileControllerTests.js @@ -33,7 +33,7 @@ describe('CompileController', function () { './CLSICacheHandler': { notifyCLSICacheAboutBuild: sinon.stub(), downloadLatestCompileCache: sinon.stub().resolves(), - downloadOldCompileCache: sinon.stub().resolves(), + downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), }, './Errors': (this.Erros = Errors), }, diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js index f332c3f568..5fe28d27f9 100644 --- a/services/clsi/test/unit/js/CompileManagerTests.js +++ b/services/clsi/test/unit/js/CompileManagerTests.js @@ -62,6 +62,7 @@ describe('CompileManager', function () { } this.OutputCacheManager = { promises: { + queueDirOperation: sinon.stub().callsArg(1), saveOutputFiles: sinon .stub() .resolves({ outputFiles: this.buildFiles, buildId: this.buildId }), @@ -163,7 +164,7 @@ describe('CompileManager', function () { './CLSICacheHandler': { notifyCLSICacheAboutBuild: sinon.stub(), downloadLatestCompileCache: sinon.stub().resolves(), - downloadOldCompileCache: sinon.stub().resolves(), + downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), }, }, }) diff --git a/services/clsi/test/unit/js/DockerRunnerTests.js b/services/clsi/test/unit/js/DockerRunnerTests.js index 6c377d102b..d70aab52c7 100644 --- a/services/clsi/test/unit/js/DockerRunnerTests.js +++ b/services/clsi/test/unit/js/DockerRunnerTests.js @@ -76,8 +76,11 @@ describe('DockerRunner', function () { this.env = {} this.callback = sinon.stub() this.project_id = 'project-id-123' - this.volumes = { '/local/compile/directory': '/compile' } + this.volumes = { '/some/host/dir/compiles/directory': '/compile' } this.Settings.clsi.docker.image = this.defaultImage = 'default-image' + this.Settings.path.sandboxedCompilesHostDirCompiles = + '/some/host/dir/compiles' + this.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output' this.compileGroup = 'compile-group' return (this.Settings.clsi.docker.env = { PATH: 'mock-path' }) }) @@ -151,9 +154,8 @@ describe('DockerRunner', function () { }) }) - describe('when path.sandboxedCompilesHostDir is set', function () { + describe('standard compile', function () { beforeEach(function () { - this.Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles' this.directory = '/var/lib/overleaf/data/compiles/xyz' this.DockerRunner._runAndWaitForContainer = sinon .stub() @@ -183,6 +185,99 @@ describe('DockerRunner', function () { }) }) + describe('synctex-output', function () { + beforeEach(function () { + this.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id' + this.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (this.output = 'mock-output')) + this.DockerRunner.run( + this.project_id, + this.command, + this.directory, + this.image, + this.timeout, + this.env, + 'synctex-output', + this.callback + ) + }) + + it('should re-write the bind directory and set ro flag', function () { + const volumes = + this.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/output/xyz/generated-files/id': '/compile:ro', + }) + }) + + it('should call the callback', function () { + this.callback.calledWith(null, this.output).should.equal(true) + }) + }) + + describe('synctex', function () { + beforeEach(function () { + this.directory = '/var/lib/overleaf/data/compile/xyz' + this.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (this.output = 'mock-output')) + this.DockerRunner.run( + this.project_id, + this.command, + this.directory, + this.image, + this.timeout, + this.env, + 'synctex', + this.callback + ) + }) + + it('should re-write the bind directory', function () { + const volumes = + this.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/compiles/xyz': '/compile:ro', + }) + }) + + it('should call the callback', function () { + this.callback.calledWith(null, this.output).should.equal(true) + }) + }) + + describe('wordcount', function () { + beforeEach(function () { + this.directory = '/var/lib/overleaf/data/compile/xyz' + this.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (this.output = 'mock-output')) + this.DockerRunner.run( + this.project_id, + this.command, + this.directory, + this.image, + this.timeout, + this.env, + 'wordcount', + this.callback + ) + }) + + it('should re-write the bind directory', function () { + const volumes = + this.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/compiles/xyz': '/compile:ro', + }) + }) + + it('should call the callback', function () { + this.callback.calledWith(null, this.output).should.equal(true) + }) + }) + describe('when the run throws an error', function () { beforeEach(function () { let firstTime = true @@ -390,7 +485,7 @@ describe('DockerRunner', function () { const options = this.DockerRunner._runAndWaitForContainer.lastCall.args[0] return expect(options.HostConfig).to.deep.include({ - Binds: ['/local/compile/directory:/compile:rw'], + Binds: ['/some/host/dir/compiles/directory:/compile:rw'], LogConfig: { Type: 'none', Config: {} }, CapDrop: 'ALL', SecurityOpt: ['no-new-privileges'], @@ -562,82 +657,6 @@ describe('DockerRunner', function () { }) }) - describe('when a volume does not exist', function () { - beforeEach(function () { - this.fs.stat = sinon.stub().yields(new Error('no such path')) - return this.DockerRunner.startContainer( - this.options, - this.volumes, - this.attachStreamHandler, - this.callback - ) - }) - - it('should not try to create the container', function () { - return this.createContainer.called.should.equal(false) - }) - - it('should call the callback with an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - }) - }) - - describe('when a volume exists but is not a directory', function () { - beforeEach(function () { - this.fs.stat = sinon.stub().yields(null, { - isDirectory() { - return false - }, - }) - return this.DockerRunner.startContainer( - this.options, - this.volumes, - this.attachStreamHandler, - this.callback - ) - }) - - it('should not try to create the container', function () { - return this.createContainer.called.should.equal(false) - }) - - it('should call the callback with an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - }) - }) - - describe('when a volume does not exist, but sibling-containers are used', function () { - beforeEach(function () { - this.fs.stat = sinon.stub().yields(new Error('no such path')) - this.Settings.path.sandboxedCompilesHostDir = '/some/path' - this.container.start = sinon.stub().yields() - return this.DockerRunner.startContainer( - this.options, - this.volumes, - () => {}, - this.callback - ) - }) - - afterEach(function () { - return delete this.Settings.path.sandboxedCompilesHostDir - }) - - it('should start the container with the given name', function () { - this.getContainer.calledWith(this.options.name).should.equal(true) - return this.container.start.called.should.equal(true) - }) - - it('should not try to create the container', function () { - return this.createContainer.called.should.equal(false) - }) - - return it('should call the callback', function () { - this.callback.called.should.equal(true) - return this.callback.calledWith(new Error()).should.equal(false) - }) - }) - return describe('when the container tries to be created, but already has been (race condition)', function () {}) }) diff --git a/services/web/app/src/Features/Compile/ClsiCacheHandler.js b/services/web/app/src/Features/Compile/ClsiCacheHandler.js index 80296f7f8d..14c742d0ae 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheHandler.js +++ b/services/web/app/src/Features/Compile/ClsiCacheHandler.js @@ -15,6 +15,7 @@ function validateFilename(filename) { 'output.blg', 'output.log', 'output.pdf', + 'output.synctex.gz', 'output.overleaf.json', 'output.tar.gz', ].includes(filename) ||