From 6c6e8d9a9712724e447fc42b929ea2330adc207a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 24 Feb 2026 08:04:48 +0100 Subject: [PATCH] [monorepo] switch all output file reads to clsi-nginx (#31691) * [monorepo] switch all output file reads to clsi-nginx * [clsi-lb] allow gallery download requests * [terraform] clsi: use nginx.conf from clsi service * [clsi] fix flakey tests * [clsi] replace alias with rewrite and root in nginx config * [k8s] clsi-lb: expose download port on internal service * [web] add explicit endpoint for downloading all output files Serve the output.zip endpoint from clsi. * [clsi] fix regex for latexqc submission ids Previously, we only handled template submission ids. GitOrigin-RevId: 6c3b21b01ec41ae767530b14aac31fbe3d640dd5 --- develop/dev.env | 1 + develop/docker-compose.yml | 11 +++++ server-ce/nginx/clsi-nginx.conf | 30 ++++++------ services/clsi/app/js/CompileController.js | 2 +- services/clsi/buildscript.txt | 2 +- services/clsi/config/settings.defaults.cjs | 2 +- services/clsi/docker-compose.ci.yml | 17 +++++++ services/clsi/docker-compose.yml | 16 ++++++ services/clsi/nginx.conf | 37 ++++++-------- .../acceptance/js/ExampleDocumentTests.js | 2 +- .../clsi/test/acceptance/js/helpers/Client.js | 3 +- .../test/unit/js/CompileController.test.js | 8 +-- .../Features/Compile/ClsiCookieManager.mjs | 15 +++--- .../app/src/Features/Compile/ClsiManager.mjs | 8 +-- .../Features/Compile/CompileController.mjs | 49 ++++++++++++++++++- services/web/app/src/router.mjs | 14 ++++++ services/web/config/settings.defaults.js | 4 +- .../config/settings.test.defaults.js | 1 + services/web/test/acceptance/src/Init.mjs | 2 + .../test/acceptance/src/mocks/MockClsiApi.mjs | 26 +--------- .../acceptance/src/mocks/MockClsiNginxApi.mjs | 37 ++++++++++++++ .../src/Compile/CompileController.test.mjs | 21 ++++---- 22 files changed, 213 insertions(+), 95 deletions(-) create mode 100644 services/web/test/acceptance/src/mocks/MockClsiNginxApi.mjs diff --git a/develop/dev.env b/develop/dev.env index cf8d1991c1..f2e6004eec 100644 --- a/develop/dev.env +++ b/develop/dev.env @@ -1,5 +1,6 @@ CHAT_HOST=chat CLSI_HOST=clsi +DOWNLOAD_HOST=clsi-nginx CONTACTS_HOST=contacts DOCSTORE_HOST=docstore DOCUMENT_UPDATER_HOST=document-updater diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index 3beac219b7..dbdc89beb6 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -36,6 +36,17 @@ services: - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock - clsi-cache:/overleaf/services/clsi/cache + clsi-nginx: + image: nginx:1.28 + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + - /run + volumes: + - ${PWD}/output:/output:ro + - ../services/clsi/nginx.conf:/etc/nginx/conf.d/nginx.conf:ro + contacts: build: context: .. diff --git a/server-ce/nginx/clsi-nginx.conf b/server-ce/nginx/clsi-nginx.conf index aac976ecd8..5ce22a3f56 100644 --- a/server-ce/nginx/clsi-nginx.conf +++ b/server-ce/nginx/clsi-nginx.conf @@ -1,12 +1,11 @@ -# keep in sync with clsi-startup.sh files -# keep in sync with clsi/nginx.conf +# keep in sync with services/clsi/nginx.conf # Changes to the above: -# - added debug header +# - added Security-Headers # - remove CORS rules, Server-CE/Server-Pro runs behind a single origin # - change /output path to /var/lib/overleaf/data/output +# - remove tiny.pdf endpoints server { - # Extra header for debugging. add_header 'X-Served-By' 'clsi-nginx' always; # Security-Headers @@ -30,20 +29,14 @@ server { application/pdf pdf; } # handle output files for specific users - location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { - alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/output.$4; - } - # handle .blg files for specific users - location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ { - alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/$4.blg; + location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ { + rewrite ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$4 break; + root /var/lib/overleaf/data/output/$1-$2/generated-files/$3/; } # handle output files for anonymous users - location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { - alias /var/lib/overleaf/data/output/$1/generated-files/$2/output.$3; - } - # handle .blg files for anonymous users - location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ { - alias /var/lib/overleaf/data/output/$1/generated-files/$2/$3.blg; + location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ { + rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break; + root /var/lib/overleaf/data/output/$1/generated-files/$2/; } # PDF range for specific users @@ -58,4 +51,9 @@ server { expires 1d; alias /var/lib/overleaf/data/output/$1/content/$2; } + + # Do not look up any non matching files in the default root. + location / { + return 404; + } } diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js index 17b031d661..c0f9d61248 100644 --- a/services/clsi/app/js/CompileController.js +++ b/services/clsi/app/js/CompileController.js @@ -153,7 +153,7 @@ function compile(req, res, next) { outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix, outputFiles: outputFiles.map(file => ({ url: - `${Settings.apis.clsi.url}/project/${request.project_id}` + + `${Settings.apis.clsi.downloadHost}/project/${request.project_id}` + (request.user_id != null ? `/user/${request.user_id}` : '') + diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index aed8e981cb..5f6bf7d7ab 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -1,7 +1,7 @@ clsi --data-dirs=cache,compiles,output --dependencies= ---env-add=ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2025.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output +--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2025.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output --env-pass-through= --esmock-loader=False --node-version=24.13.0 diff --git a/services/clsi/config/settings.defaults.cjs b/services/clsi/config/settings.defaults.cjs index 0ee6cf1c4a..37fbe6cf48 100644 --- a/services/clsi/config/settings.defaults.cjs +++ b/services/clsi/config/settings.defaults.cjs @@ -49,7 +49,7 @@ module.exports = { outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`, clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID, - downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:3013', + downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:8080', }, clsiPerf: { host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${ diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 524ee04b61..02ce8698b6 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -27,6 +27,7 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + DOWNLOAD_HOST: http://clsi-nginx:8080 ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file" ENABLE_PDF_CACHING: "true" PDF_CACHING_ENABLE_WORKER_POOL: "true" @@ -40,7 +41,11 @@ services: volumes: - ./reports:/overleaf/services/clsi/reports - ./compiles:/overleaf/services/clsi/compiles + - ./output:/overleaf/services/clsi/output - /var/run/docker.sock:/var/run/docker.sock + depends_on: + clsi-nginx: + condition: service_started command: npm run test:acceptance tar: @@ -50,3 +55,15 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root + + clsi-nginx: + image: nginx:1.28 + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + - /run + volumes: + - ./output:/output:ro + - ./nginx.conf:/etc/nginx/conf.d/nginx.conf:ro + - ./tiny.pdf:/var/clsi/tiny.pdf:ro diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 4207048e1f..2812700d38 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -41,6 +41,7 @@ services: LOG_LEVEL: ${LOG_LEVEL:-} NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + DOWNLOAD_HOST: http://clsi-nginx:8080 ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file" ENABLE_PDF_CACHING: "true" PDF_CACHING_ENABLE_WORKER_POOL: "true" @@ -51,4 +52,19 @@ services: SANDBOXED_COMPILES: "true" SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output + depends_on: + clsi-nginx: + condition: service_started command: npm run --silent test:acceptance + + clsi-nginx: + image: nginx:1.28 + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + - /run + volumes: + - ./output:/output:ro + - ./nginx.conf:/etc/nginx/conf.d/nginx.conf:ro + - ./tiny.pdf:/var/clsi/tiny.pdf:ro diff --git a/services/clsi/nginx.conf b/services/clsi/nginx.conf index 604eb93fbf..2d869eeb93 100644 --- a/services/clsi/nginx.conf +++ b/services/clsi/nginx.conf @@ -1,14 +1,10 @@ -# keep in sync with clsi-startup.sh files # keep in sync with server-ce/nginx/clsi-nginx.conf -# Changes to the above: -# - added debug header server { - # Extra header for dev-env. add_header 'X-Served-By' 'clsi-nginx' always; listen 8080; - server_name clsi-proxy; + server_name clsi-nginx; server_tokens off; access_log off; # Ignore symlinks possibly created by users @@ -46,40 +42,34 @@ server { } # handle output files for specific users - location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { + location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ { if ($request_method = 'OPTIONS') { # handle OPTIONS method for CORS requests add_header 'Allow' 'GET,HEAD'; return 204; } - alias /output/$1-$2/generated-files/$3/output.$4; - } - # handle .blg files for specific users - location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ { - if ($request_method = 'OPTIONS') { - # handle OPTIONS method for CORS requests - add_header 'Allow' 'GET,HEAD'; - return 204; - } - alias /output/$1-$2/generated-files/$3/$4.blg; + rewrite ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$4 break; + root /output/$1-$2/generated-files/$3/; } # handle output files for anonymous users - location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { + location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ { if ($request_method = 'OPTIONS') { # handle OPTIONS method for CORS requests add_header 'Allow' 'GET,HEAD'; return 204; } - alias /output/$1/generated-files/$2/output.$3; + rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break; + root /output/$1/generated-files/$2/; } - # handle .blg files for anonymous users - location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ { + # handle output files for submissions + location ~ ^/project/([a-z0-9_-]+)/build/([0-9a-f-]+)/output/(.+)$ { if ($request_method = 'OPTIONS') { # handle OPTIONS method for CORS requests add_header 'Allow' 'GET,HEAD'; return 204; } - alias /output/$1/generated-files/$2/$3.blg; + rewrite ^/project/([a-z0-9_-]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break; + root /output/$1/generated-files/$2/; } # PDF range for specific users @@ -114,4 +104,9 @@ server { location = /instance-state { alias /var/clsi/instance-state; } + + # Do not look up any non matching files in the default root. + location / { + return 404; + } } diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js index 57a86f67c8..bfb3853c8a 100644 --- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js +++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js @@ -156,7 +156,7 @@ describe('Example Documents', function () { (exampleDir => describe(exampleDir, function () { before(function () { - this.project_id = Client.randomId() + '_' + exampleDir + this.project_id = Client.randomId() this.outputFiles = [] // Allow each test to provide a configuration file const checksJsonPath = fixturePath( diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js index 6448d0b4fb..32b87ce38b 100644 --- a/services/clsi/test/acceptance/js/helpers/Client.js +++ b/services/clsi/test/acceptance/js/helpers/Client.js @@ -7,7 +7,8 @@ import Settings from '@overleaf/settings' const host = Settings.apis.clsi.url function randomId() { - return Math.random().toString(16).slice(2) + // Avoid ids starting with 0, which get a dummy PDF served. + return 'a' + Math.random().toString(16).slice(2) } function compile(projectId, data) { diff --git a/services/clsi/test/unit/js/CompileController.test.js b/services/clsi/test/unit/js/CompileController.test.js index ebdf4cb2c4..d609476eb5 100644 --- a/services/clsi/test/unit/js/CompileController.test.js +++ b/services/clsi/test/unit/js/CompileController.test.js @@ -144,7 +144,7 @@ describe('CompileController', () => { buildId: ctx.buildId, outputUrlPrefix: '/zone/b', outputFiles: ctx.output_files.map(file => ({ - url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, ...file, })), clsiCacheShard: undefined, @@ -172,7 +172,7 @@ describe('CompileController', () => { buildId: ctx.buildId, outputUrlPrefix: '', outputFiles: ctx.output_files.map(file => ({ - url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, ...file, })), clsiCacheShard: undefined, @@ -220,7 +220,7 @@ describe('CompileController', () => { outputUrlPrefix: '/zone/b', buildId: ctx.buildId, outputFiles: ctx.output_files.map(file => ({ - url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, ...file, })), clsiCacheShard: undefined, @@ -268,7 +268,7 @@ describe('CompileController', () => { timings: ctx.timings, outputUrlPrefix: '/zone/b', outputFiles: ctx.output_files.map(file => ({ - url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, ...file, })), clsiCacheShard: undefined, diff --git a/services/web/app/src/Features/Compile/ClsiCookieManager.mjs b/services/web/app/src/Features/Compile/ClsiCookieManager.mjs index 0cd3221d28..7ccbfcdfd7 100644 --- a/services/web/app/src/Features/Compile/ClsiCookieManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiCookieManager.mjs @@ -118,18 +118,17 @@ const ClsiCookieManagerFactory = function (backendGroup) { ) { let status try { - const params = new URLSearchParams({ + const url = new URL(Settings.apis.clsi.url) + url.pathname = '/instance-state' + url.search = new URLSearchParams({ clsiserverid, compileGroup, compileBackendClass, }).toString() - const { response, body } = await fetchStringWithResponse( - `${Settings.apis.clsi.url}/instance-state?${params}`, - { - method: 'GET', - signal: AbortSignal.timeout(30_000), - } - ) + const { response, body } = await fetchStringWithResponse(url.href, { + method: 'GET', + signal: AbortSignal.timeout(30_000), + }) status = response.status === 200 && body === `${clsiserverid},UP\n` ? 'load-shedding' diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index c52fa281fa..55a8f32cd1 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -557,7 +557,8 @@ function _getCompilerUrl( userId, action ) { - const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url) + const u = new URL(Settings.apis.clsi.url) + u.pathname = `/project/${projectId}` if (userId != null) { u.pathname += `/user/${userId}` } @@ -729,9 +730,8 @@ async function getOutputFileStream( outputFilePath ) { const { compileBackendClass, compileGroup } = options - const url = new URL( - `${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}` - ) + const url = new URL(Settings.apis.clsi.downloadHost) + url.pathname = `/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}` url.searchParams.set('compileBackendClass', compileBackendClass) url.searchParams.set('compileGroup', compileGroup) url.searchParams.set('clsiserverid', clsiServerId) diff --git a/services/web/app/src/Features/Compile/CompileController.mjs b/services/web/app/src/Features/Compile/CompileController.mjs index 857c1ae718..4f9ba4fd01 100644 --- a/services/web/app/src/Features/Compile/CompileController.mjs +++ b/services/web/app/src/Features/Compile/CompileController.mjs @@ -23,6 +23,7 @@ import { } from '@overleaf/fetch-utils' import Features from '../../infrastructure/Features.mjs' import ClsiCacheController from './ClsiCacheController.mjs' +import { prepareZipAttachment } from '../../infrastructure/Response.mjs' const { z, zz, parseReq } = Validation const ClsiCookieManager = ClsiCookieManagerFactory( @@ -31,6 +32,8 @@ const ClsiCookieManager = ClsiCookieManagerFactory( const COMPILE_TIMEOUT_MS = 10 * 60 * 1000 +const buildIdSchema = z.string().regex(/[a-z0-9-]/) + const pdfDownloadRateLimiter = new RateLimiter('full-pdf-download', { points: 1000, duration: 60 * 60, @@ -409,6 +412,33 @@ const _CompileController = { ) }, + async getOutputZipFromClsi(req, res) { + const projectId = req.params.Project_id + const userId = CompileController._getUserIdForCompile(req) + + const project = await ProjectGetter.promises.getProject(projectId, { + name: 1, + }) + const filename = `${_CompileController._getSafeProjectName(project)}-output.zip` + prepareZipAttachment(res, filename) + + const qs = {} + const url = _CompileController._getFileUrl( + projectId, + userId, + req.params.build_id, + 'output.zip' + ) + await CompileController._proxyToClsi( + projectId, + 'output-zip-file', + url, + qs, + req, + res + ) + }, + async getFileFromClsi(req, res) { const projectId = req.params.Project_id const userId = CompileController._getUserIdForCompile(req) @@ -465,6 +495,7 @@ const _CompileController = { } else if (userId != null) { url = `/project/${projectId}/user/${userId}/output/${file}` } else if (buildId != null) { + buildId = buildIdSchema.parse(buildId) url = `/project/${projectId}/build/${buildId}/output/${file}` } else { url = `/project/${projectId}/output/${file}` @@ -523,7 +554,15 @@ const _CompileController = { ) }, - async _proxyToClsiWithLimits(projectId, action, url, qs, limits, req, res) { + async _proxyToClsiWithLimits( + projectId, + action, + requestPath, + qs, + limits, + req, + res + ) { const persistenceOptions = await _getPersistenceOptions( req, projectId, @@ -534,7 +573,12 @@ const _CompileController = { throw err }) - url = new URL(`${Settings.apis.clsi.url}${url}`) + const url = new URL( + action === 'output-zip-file' + ? Settings.apis.clsi.url + : Settings.apis.clsi.downloadHost + ) + url.pathname = requestPath const searchParams = { ...persistenceOptions.qs, @@ -718,6 +762,7 @@ const CompileController = { downloadPdf: expressify(_CompileController.downloadPdf), // compileAndDownloadPdf: expressify(_CompileController.compileAndDownloadPdf), deleteAuxFiles: expressify(_CompileController.deleteAuxFiles), + getOutputZipFromClsi: expressify(_CompileController.getOutputZipFromClsi), getFileFromClsi: expressify(_CompileController.getFileFromClsi), getFileFromClsiWithoutUser: expressify( _CompileController.getFileFromClsiWithoutUser diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index 938f8ac877..c91ece4de3 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -609,6 +609,20 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { { params: ['Project_id'] } ) + // Download all the output files of a specific build + webRouter.get( + '/project/:Project_id/build/:build_id/output/output.zip', + rateLimiterMiddlewareOutputFiles, + AuthorizationMiddleware.ensureUserCanReadProject, + CompileController.getOutputZipFromClsi + ) + webRouter.get( + '/project/:Project_id/user/:user_id/build/:build_id/output/output.zip', + rateLimiterMiddlewareOutputFiles, + AuthorizationMiddleware.ensureUserCanReadProject, + CompileController.getOutputZipFromClsi + ) + // direct url access to output files for a specific build webRouter.get( /^\/project\/([^/]*)\/build\/([0-9a-f-]+)\/output\/(.*)$/, diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index f853232fdf..78e1f5b917 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -237,7 +237,9 @@ module.exports = { }, clsi: { url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`, - // url: "http://#{process.env['CLSI_LB_HOST']}:3014" + downloadHost: process.env.CLSI_LB_IP + ? `http://${process.env.CLSI_LB_IP}:80` + : `http://${process.env.DOWNLOAD_HOST || '127.0.0.1'}:8080`, backendGroupName: undefined, submissionBackendClass: process.env.CLSI_SUBMISSION_BACKEND_CLASS || 'c3d', diff --git a/services/web/test/acceptance/config/settings.test.defaults.js b/services/web/test/acceptance/config/settings.test.defaults.js index 860b5e2066..17fede0a59 100644 --- a/services/web/test/acceptance/config/settings.test.defaults.js +++ b/services/web/test/acceptance/config/settings.test.defaults.js @@ -67,6 +67,7 @@ module.exports = { }, clsi: { url: 'http://127.0.0.1:23013', + downloadHost: 'http://127.0.0.1:23080', }, realTime: { url: 'http://127.0.0.1:23026', diff --git a/services/web/test/acceptance/src/Init.mjs b/services/web/test/acceptance/src/Init.mjs index 2179e27aeb..fd16363e31 100644 --- a/services/web/test/acceptance/src/Init.mjs +++ b/services/web/test/acceptance/src/Init.mjs @@ -4,6 +4,7 @@ import Features from '../../../app/src/infrastructure/Features.mjs' import MockAnalyticsApi from './mocks/MockAnalyticsApi.mjs' import MockChatApi from './mocks/MockChatApi.mjs' import MockClsiApi from './mocks/MockClsiApi.mjs' +import MockClsiNginxApi from './mocks/MockClsiNginxApi.mjs' import MockDocstoreApi from './mocks/MockDocstoreApi.mjs' import MockDocUpdaterApi from './mocks/MockDocUpdaterApi.mjs' import MockGitBridgeApi from './mocks/MockGitBridgeApi.mjs' @@ -22,6 +23,7 @@ const mockOpts = { MockChatApi.initialize(23010, mockOpts) MockClsiApi.initialize(23013, mockOpts) +MockClsiNginxApi.initialize(23080, mockOpts) MockDocstoreApi.initialize(23016, mockOpts) MockDocUpdaterApi.initialize(23003, mockOpts) MockNotificationsApi.initialize(23042, mockOpts) diff --git a/services/web/test/acceptance/src/mocks/MockClsiApi.mjs b/services/web/test/acceptance/src/mocks/MockClsiApi.mjs index 32db8f1847..ffd065a87e 100644 --- a/services/web/test/acceptance/src/mocks/MockClsiApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockClsiApi.mjs @@ -1,5 +1,4 @@ import AbstractMockApi from './AbstractMockApi.mjs' -import { plainTextResponse } from '../../../../app/src/infrastructure/Response.mjs' class MockClsiApi extends AbstractMockApi { static compile(req, res) { @@ -9,13 +8,13 @@ class MockClsiApi extends AbstractMockApi { error: null, outputFiles: [ { - url: `http://clsi:3013/project/${req.params.project_id}/build/1234/output/output.pdf`, + url: `http://clsi:8080/project/${req.params.project_id}/build/1234/output/output.pdf`, path: 'output.pdf', type: 'pdf', build: 1234, }, { - url: `http://clsi:3013/project/${req.params.project_id}/build/1234/output/output.log`, + url: `http://clsi:8080/project/${req.params.project_id}/build/1234/output/output.log`, path: 'output.log', type: 'log', build: 1234, @@ -32,27 +31,6 @@ class MockClsiApi extends AbstractMockApi { MockClsiApi.compile ) - this.app.get( - '/project/:project_id/build/:build_id/output/*', - (req, res) => { - const filename = req.params[0] - if (filename === 'output.pdf') { - plainTextResponse(res, 'mock-pdf') - } else if (filename === 'output.log') { - plainTextResponse(res, 'mock-log') - } else { - res.sendStatus(404) - } - } - ) - - this.app.get( - '/project/:project_id/user/:user_id/build/:build_id/output/:output_path', - (req, res) => { - plainTextResponse(res, 'hello') - } - ) - this.app.get('/project/:project_id/status', (req, res) => { res.sendStatus(200) }) diff --git a/services/web/test/acceptance/src/mocks/MockClsiNginxApi.mjs b/services/web/test/acceptance/src/mocks/MockClsiNginxApi.mjs new file mode 100644 index 0000000000..8d0b75c8ab --- /dev/null +++ b/services/web/test/acceptance/src/mocks/MockClsiNginxApi.mjs @@ -0,0 +1,37 @@ +import AbstractMockApi from './AbstractMockApi.mjs' +import { plainTextResponse } from '../../../../app/src/infrastructure/Response.mjs' + +class MockClsiNginxApi extends AbstractMockApi { + applyRoutes() { + this.app.get( + '/project/:project_id/build/:build_id/output/*', + (req, res) => { + const filename = req.params[0] + if (filename === 'output.pdf') { + plainTextResponse(res, 'mock-pdf') + } else if (filename === 'output.log') { + plainTextResponse(res, 'mock-log') + } else { + res.sendStatus(404) + } + } + ) + + this.app.get( + '/project/:project_id/user/:user_id/build/:build_id/output/:output_path', + (req, res) => { + plainTextResponse(res, 'hello') + } + ) + } +} + +export default MockClsiNginxApi + +// type hint for the inherited `instance` method +/** + * @function instance + * @memberOf MockClsiNginxApi + * @static + * @returns {MockClsiNginxApi} + */ diff --git a/services/web/test/unit/src/Compile/CompileController.test.mjs b/services/web/test/unit/src/Compile/CompileController.test.mjs index 9e7109ba69..be98f0c0c1 100644 --- a/services/web/test/unit/src/Compile/CompileController.test.mjs +++ b/services/web/test/unit/src/Compile/CompileController.test.mjs @@ -42,7 +42,8 @@ describe('CompileController', function () { ctx.settings = { apis: { clsi: { - url: 'http://clsi.example.com', + url: 'http://clsi.example.com:3013', + downloadHost: 'http://clsi.example.com:8080', submissionBackendClass: 'c3d', }, clsi_priority: { @@ -776,7 +777,7 @@ describe('CompileController', function () { it('should open a request to the CLSI', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d&query=foo` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d&query=foo` ) }) @@ -806,7 +807,7 @@ describe('CompileController', function () { it('should open a request to the CLSI', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` ) }) }) @@ -826,7 +827,7 @@ describe('CompileController', function () { }) ctx.fetchUtils.fetchStreamWithResponse.rejects( new RequestFailedError( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, { method: 'GET' }, { status: 404 } ) @@ -844,7 +845,7 @@ describe('CompileController', function () { it('should open a request to the CLSI', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` ) }) @@ -873,7 +874,7 @@ describe('CompileController', function () { }) ctx.fetchUtils.fetchStreamWithResponse.rejects( new RequestFailedError( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, { method: 'GET' }, { status: 404 } ) @@ -891,7 +892,7 @@ describe('CompileController', function () { it('should open a request to the CLSI', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` ) }) @@ -924,7 +925,7 @@ describe('CompileController', function () { it('should open a request to the CLSI', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` ) }) @@ -955,7 +956,7 @@ describe('CompileController', function () { it('should proxy to the standard url', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` ) }) }) @@ -982,7 +983,7 @@ describe('CompileController', function () { it('should proxy to the standard url without the build parameter', function (ctx) { ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( - `${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` + `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` ) }) })