[monorepo] switch all output file reads to clsi-nginx (#31691)

* [monorepo] switch all output file reads to clsi-nginx

* [clsi-lb] allow gallery download requests

* [terraform] clsi: use nginx.conf from clsi service

* [clsi] fix flakey tests

* [clsi] replace alias with rewrite and root in nginx config

* [k8s] clsi-lb: expose download port on internal service

* [web] add explicit endpoint for downloading all output files

Serve the output.zip endpoint from clsi.

* [clsi] fix regex for latexqc submission ids

Previously, we only handled template submission ids.

GitOrigin-RevId: 6c3b21b01ec41ae767530b14aac31fbe3d640dd5
This commit is contained in:
Jakob Ackermann
2026-02-24 08:04:48 +01:00
committed by Copybot
parent 892047fcf6
commit 6c6e8d9a97
22 changed files with 213 additions and 95 deletions

View File

@@ -1,5 +1,6 @@
CHAT_HOST=chat CHAT_HOST=chat
CLSI_HOST=clsi CLSI_HOST=clsi
DOWNLOAD_HOST=clsi-nginx
CONTACTS_HOST=contacts CONTACTS_HOST=contacts
DOCSTORE_HOST=docstore DOCSTORE_HOST=docstore
DOCUMENT_UPDATER_HOST=document-updater DOCUMENT_UPDATER_HOST=document-updater

View File

@@ -36,6 +36,17 @@ services:
- ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
- clsi-cache:/overleaf/services/clsi/cache - clsi-cache:/overleaf/services/clsi/cache
clsi-nginx:
image: nginx:1.28
read_only: true
tmpfs:
- /tmp
- /var/cache/nginx
- /run
volumes:
- ${PWD}/output:/output:ro
- ../services/clsi/nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
contacts: contacts:
build: build:
context: .. context: ..

View File

@@ -1,12 +1,11 @@
# keep in sync with clsi-startup.sh files # keep in sync with services/clsi/nginx.conf
# keep in sync with clsi/nginx.conf
# Changes to the above: # Changes to the above:
# - added debug header # - added Security-Headers
# - remove CORS rules, Server-CE/Server-Pro runs behind a single origin # - remove CORS rules, Server-CE/Server-Pro runs behind a single origin
# - change /output path to /var/lib/overleaf/data/output # - change /output path to /var/lib/overleaf/data/output
# - remove tiny.pdf endpoints
server { server {
# Extra header for debugging.
add_header 'X-Served-By' 'clsi-nginx' always; add_header 'X-Served-By' 'clsi-nginx' always;
# Security-Headers # Security-Headers
@@ -30,20 +29,14 @@ server {
application/pdf pdf; application/pdf pdf;
} }
# handle output files for specific users # handle output files for specific users
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/output.$4; rewrite ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$4 break;
} root /var/lib/overleaf/data/output/$1-$2/generated-files/$3/;
# handle .blg files for specific users
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ {
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/$4.blg;
} }
# handle output files for anonymous users # handle output files for anonymous users
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
alias /var/lib/overleaf/data/output/$1/generated-files/$2/output.$3; rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
} root /var/lib/overleaf/data/output/$1/generated-files/$2/;
# handle .blg files for anonymous users
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ {
alias /var/lib/overleaf/data/output/$1/generated-files/$2/$3.blg;
} }
# PDF range for specific users # PDF range for specific users
@@ -58,4 +51,9 @@ server {
expires 1d; expires 1d;
alias /var/lib/overleaf/data/output/$1/content/$2; alias /var/lib/overleaf/data/output/$1/content/$2;
} }
# Do not look up any non matching files in the default root.
location / {
return 404;
}
} }

View File

@@ -153,7 +153,7 @@ function compile(req, res, next) {
outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix, outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
outputFiles: outputFiles.map(file => ({ outputFiles: outputFiles.map(file => ({
url: url:
`${Settings.apis.clsi.url}/project/${request.project_id}` + `${Settings.apis.clsi.downloadHost}/project/${request.project_id}` +
(request.user_id != null (request.user_id != null
? `/user/${request.user_id}` ? `/user/${request.user_id}`
: '') + : '') +

View File

@@ -1,7 +1,7 @@
clsi clsi
--data-dirs=cache,compiles,output --data-dirs=cache,compiles,output
--dependencies= --dependencies=
--env-add=ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2025.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output --env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS="clsi-perf simple-latex-file",ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2025.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output
--env-pass-through= --env-pass-through=
--esmock-loader=False --esmock-loader=False
--node-version=24.13.0 --node-version=24.13.0

View File

@@ -49,7 +49,7 @@ module.exports = {
outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`, outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`,
clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID, clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID,
downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:3013', downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:8080',
}, },
clsiPerf: { clsiPerf: {
host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${ host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${

View File

@@ -27,6 +27,7 @@ services:
MOCHA_GREP: ${MOCHA_GREP} MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict" NODE_OPTIONS: "--unhandled-rejections=strict"
DOWNLOAD_HOST: http://clsi-nginx:8080
ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file" ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file"
ENABLE_PDF_CACHING: "true" ENABLE_PDF_CACHING: "true"
PDF_CACHING_ENABLE_WORKER_POOL: "true" PDF_CACHING_ENABLE_WORKER_POOL: "true"
@@ -40,7 +41,11 @@ services:
volumes: volumes:
- ./reports:/overleaf/services/clsi/reports - ./reports:/overleaf/services/clsi/reports
- ./compiles:/overleaf/services/clsi/compiles - ./compiles:/overleaf/services/clsi/compiles
- ./output:/overleaf/services/clsi/output
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
depends_on:
clsi-nginx:
condition: service_started
command: npm run test:acceptance command: npm run test:acceptance
tar: tar:
@@ -50,3 +55,15 @@ services:
- ./:/tmp/build/ - ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root user: root
clsi-nginx:
image: nginx:1.28
read_only: true
tmpfs:
- /tmp
- /var/cache/nginx
- /run
volumes:
- ./output:/output:ro
- ./nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
- ./tiny.pdf:/var/clsi/tiny.pdf:ro

View File

@@ -41,6 +41,7 @@ services:
LOG_LEVEL: ${LOG_LEVEL:-} LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict" NODE_OPTIONS: "--unhandled-rejections=strict"
DOWNLOAD_HOST: http://clsi-nginx:8080
ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file" ALLOWED_COMPILE_GROUPS: "clsi-perf simple-latex-file"
ENABLE_PDF_CACHING: "true" ENABLE_PDF_CACHING: "true"
PDF_CACHING_ENABLE_WORKER_POOL: "true" PDF_CACHING_ENABLE_WORKER_POOL: "true"
@@ -51,4 +52,19 @@ services:
SANDBOXED_COMPILES: "true" SANDBOXED_COMPILES: "true"
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
depends_on:
clsi-nginx:
condition: service_started
command: npm run --silent test:acceptance command: npm run --silent test:acceptance
clsi-nginx:
image: nginx:1.28
read_only: true
tmpfs:
- /tmp
- /var/cache/nginx
- /run
volumes:
- ./output:/output:ro
- ./nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
- ./tiny.pdf:/var/clsi/tiny.pdf:ro

View File

@@ -1,14 +1,10 @@
# keep in sync with clsi-startup.sh files
# keep in sync with server-ce/nginx/clsi-nginx.conf # keep in sync with server-ce/nginx/clsi-nginx.conf
# Changes to the above:
# - added debug header
server { server {
# Extra header for dev-env.
add_header 'X-Served-By' 'clsi-nginx' always; add_header 'X-Served-By' 'clsi-nginx' always;
listen 8080; listen 8080;
server_name clsi-proxy; server_name clsi-nginx;
server_tokens off; server_tokens off;
access_log off; access_log off;
# Ignore symlinks possibly created by users # Ignore symlinks possibly created by users
@@ -46,40 +42,34 @@ server {
} }
# handle output files for specific users # handle output files for specific users
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
if ($request_method = 'OPTIONS') { if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests # handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD'; add_header 'Allow' 'GET,HEAD';
return 204; return 204;
} }
alias /output/$1-$2/generated-files/$3/output.$4; rewrite ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$4 break;
} root /output/$1-$2/generated-files/$3/;
# handle .blg files for specific users
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ {
if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD';
return 204;
}
alias /output/$1-$2/generated-files/$3/$4.blg;
} }
# handle output files for anonymous users # handle output files for anonymous users
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ { location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
if ($request_method = 'OPTIONS') { if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests # handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD'; add_header 'Allow' 'GET,HEAD';
return 204; return 204;
} }
alias /output/$1/generated-files/$2/output.$3; rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
root /output/$1/generated-files/$2/;
} }
# handle .blg files for anonymous users # handle output files for submissions
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)\.blg$ { location ~ ^/project/([a-z0-9_-]+)/build/([0-9a-f-]+)/output/(.+)$ {
if ($request_method = 'OPTIONS') { if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests # handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD'; add_header 'Allow' 'GET,HEAD';
return 204; return 204;
} }
alias /output/$1/generated-files/$2/$3.blg; rewrite ^/project/([a-z0-9_-]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
root /output/$1/generated-files/$2/;
} }
# PDF range for specific users # PDF range for specific users
@@ -114,4 +104,9 @@ server {
location = /instance-state { location = /instance-state {
alias /var/clsi/instance-state; alias /var/clsi/instance-state;
} }
# Do not look up any non matching files in the default root.
location / {
return 404;
}
} }

View File

@@ -156,7 +156,7 @@ describe('Example Documents', function () {
(exampleDir => (exampleDir =>
describe(exampleDir, function () { describe(exampleDir, function () {
before(function () { before(function () {
this.project_id = Client.randomId() + '_' + exampleDir this.project_id = Client.randomId()
this.outputFiles = [] this.outputFiles = []
// Allow each test to provide a configuration file // Allow each test to provide a configuration file
const checksJsonPath = fixturePath( const checksJsonPath = fixturePath(

View File

@@ -7,7 +7,8 @@ import Settings from '@overleaf/settings'
const host = Settings.apis.clsi.url const host = Settings.apis.clsi.url
function randomId() { function randomId() {
return Math.random().toString(16).slice(2) // Avoid ids starting with 0, which get a dummy PDF served.
return 'a' + Math.random().toString(16).slice(2)
} }
function compile(projectId, data) { function compile(projectId, data) {

View File

@@ -144,7 +144,7 @@ describe('CompileController', () => {
buildId: ctx.buildId, buildId: ctx.buildId,
outputUrlPrefix: '/zone/b', outputUrlPrefix: '/zone/b',
outputFiles: ctx.output_files.map(file => ({ outputFiles: ctx.output_files.map(file => ({
url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`,
...file, ...file,
})), })),
clsiCacheShard: undefined, clsiCacheShard: undefined,
@@ -172,7 +172,7 @@ describe('CompileController', () => {
buildId: ctx.buildId, buildId: ctx.buildId,
outputUrlPrefix: '', outputUrlPrefix: '',
outputFiles: ctx.output_files.map(file => ({ outputFiles: ctx.output_files.map(file => ({
url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`,
...file, ...file,
})), })),
clsiCacheShard: undefined, clsiCacheShard: undefined,
@@ -220,7 +220,7 @@ describe('CompileController', () => {
outputUrlPrefix: '/zone/b', outputUrlPrefix: '/zone/b',
buildId: ctx.buildId, buildId: ctx.buildId,
outputFiles: ctx.output_files.map(file => ({ outputFiles: ctx.output_files.map(file => ({
url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`,
...file, ...file,
})), })),
clsiCacheShard: undefined, clsiCacheShard: undefined,
@@ -268,7 +268,7 @@ describe('CompileController', () => {
timings: ctx.timings, timings: ctx.timings,
outputUrlPrefix: '/zone/b', outputUrlPrefix: '/zone/b',
outputFiles: ctx.output_files.map(file => ({ outputFiles: ctx.output_files.map(file => ({
url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, url: `${ctx.Settings.apis.clsi.downloadHost}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`,
...file, ...file,
})), })),
clsiCacheShard: undefined, clsiCacheShard: undefined,

View File

@@ -118,18 +118,17 @@ const ClsiCookieManagerFactory = function (backendGroup) {
) { ) {
let status let status
try { try {
const params = new URLSearchParams({ const url = new URL(Settings.apis.clsi.url)
url.pathname = '/instance-state'
url.search = new URLSearchParams({
clsiserverid, clsiserverid,
compileGroup, compileGroup,
compileBackendClass, compileBackendClass,
}).toString() }).toString()
const { response, body } = await fetchStringWithResponse( const { response, body } = await fetchStringWithResponse(url.href, {
`${Settings.apis.clsi.url}/instance-state?${params}`, method: 'GET',
{ signal: AbortSignal.timeout(30_000),
method: 'GET', })
signal: AbortSignal.timeout(30_000),
}
)
status = status =
response.status === 200 && body === `${clsiserverid},UP\n` response.status === 200 && body === `${clsiserverid},UP\n`
? 'load-shedding' ? 'load-shedding'

View File

@@ -557,7 +557,8 @@ function _getCompilerUrl(
userId, userId,
action action
) { ) {
const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url) const u = new URL(Settings.apis.clsi.url)
u.pathname = `/project/${projectId}`
if (userId != null) { if (userId != null) {
u.pathname += `/user/${userId}` u.pathname += `/user/${userId}`
} }
@@ -729,9 +730,8 @@ async function getOutputFileStream(
outputFilePath outputFilePath
) { ) {
const { compileBackendClass, compileGroup } = options const { compileBackendClass, compileGroup } = options
const url = new URL( const url = new URL(Settings.apis.clsi.downloadHost)
`${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}` url.pathname = `/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}`
)
url.searchParams.set('compileBackendClass', compileBackendClass) url.searchParams.set('compileBackendClass', compileBackendClass)
url.searchParams.set('compileGroup', compileGroup) url.searchParams.set('compileGroup', compileGroup)
url.searchParams.set('clsiserverid', clsiServerId) url.searchParams.set('clsiserverid', clsiServerId)

View File

@@ -23,6 +23,7 @@ import {
} from '@overleaf/fetch-utils' } from '@overleaf/fetch-utils'
import Features from '../../infrastructure/Features.mjs' import Features from '../../infrastructure/Features.mjs'
import ClsiCacheController from './ClsiCacheController.mjs' import ClsiCacheController from './ClsiCacheController.mjs'
import { prepareZipAttachment } from '../../infrastructure/Response.mjs'
const { z, zz, parseReq } = Validation const { z, zz, parseReq } = Validation
const ClsiCookieManager = ClsiCookieManagerFactory( const ClsiCookieManager = ClsiCookieManagerFactory(
@@ -31,6 +32,8 @@ const ClsiCookieManager = ClsiCookieManagerFactory(
const COMPILE_TIMEOUT_MS = 10 * 60 * 1000 const COMPILE_TIMEOUT_MS = 10 * 60 * 1000
const buildIdSchema = z.string().regex(/[a-z0-9-]/)
const pdfDownloadRateLimiter = new RateLimiter('full-pdf-download', { const pdfDownloadRateLimiter = new RateLimiter('full-pdf-download', {
points: 1000, points: 1000,
duration: 60 * 60, duration: 60 * 60,
@@ -409,6 +412,33 @@ const _CompileController = {
) )
}, },
async getOutputZipFromClsi(req, res) {
const projectId = req.params.Project_id
const userId = CompileController._getUserIdForCompile(req)
const project = await ProjectGetter.promises.getProject(projectId, {
name: 1,
})
const filename = `${_CompileController._getSafeProjectName(project)}-output.zip`
prepareZipAttachment(res, filename)
const qs = {}
const url = _CompileController._getFileUrl(
projectId,
userId,
req.params.build_id,
'output.zip'
)
await CompileController._proxyToClsi(
projectId,
'output-zip-file',
url,
qs,
req,
res
)
},
async getFileFromClsi(req, res) { async getFileFromClsi(req, res) {
const projectId = req.params.Project_id const projectId = req.params.Project_id
const userId = CompileController._getUserIdForCompile(req) const userId = CompileController._getUserIdForCompile(req)
@@ -465,6 +495,7 @@ const _CompileController = {
} else if (userId != null) { } else if (userId != null) {
url = `/project/${projectId}/user/${userId}/output/${file}` url = `/project/${projectId}/user/${userId}/output/${file}`
} else if (buildId != null) { } else if (buildId != null) {
buildId = buildIdSchema.parse(buildId)
url = `/project/${projectId}/build/${buildId}/output/${file}` url = `/project/${projectId}/build/${buildId}/output/${file}`
} else { } else {
url = `/project/${projectId}/output/${file}` url = `/project/${projectId}/output/${file}`
@@ -523,7 +554,15 @@ const _CompileController = {
) )
}, },
async _proxyToClsiWithLimits(projectId, action, url, qs, limits, req, res) { async _proxyToClsiWithLimits(
projectId,
action,
requestPath,
qs,
limits,
req,
res
) {
const persistenceOptions = await _getPersistenceOptions( const persistenceOptions = await _getPersistenceOptions(
req, req,
projectId, projectId,
@@ -534,7 +573,12 @@ const _CompileController = {
throw err throw err
}) })
url = new URL(`${Settings.apis.clsi.url}${url}`) const url = new URL(
action === 'output-zip-file'
? Settings.apis.clsi.url
: Settings.apis.clsi.downloadHost
)
url.pathname = requestPath
const searchParams = { const searchParams = {
...persistenceOptions.qs, ...persistenceOptions.qs,
@@ -718,6 +762,7 @@ const CompileController = {
downloadPdf: expressify(_CompileController.downloadPdf), // downloadPdf: expressify(_CompileController.downloadPdf), //
compileAndDownloadPdf: expressify(_CompileController.compileAndDownloadPdf), compileAndDownloadPdf: expressify(_CompileController.compileAndDownloadPdf),
deleteAuxFiles: expressify(_CompileController.deleteAuxFiles), deleteAuxFiles: expressify(_CompileController.deleteAuxFiles),
getOutputZipFromClsi: expressify(_CompileController.getOutputZipFromClsi),
getFileFromClsi: expressify(_CompileController.getFileFromClsi), getFileFromClsi: expressify(_CompileController.getFileFromClsi),
getFileFromClsiWithoutUser: expressify( getFileFromClsiWithoutUser: expressify(
_CompileController.getFileFromClsiWithoutUser _CompileController.getFileFromClsiWithoutUser

View File

@@ -609,6 +609,20 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) {
{ params: ['Project_id'] } { params: ['Project_id'] }
) )
// Download all the output files of a specific build
webRouter.get(
'/project/:Project_id/build/:build_id/output/output.zip',
rateLimiterMiddlewareOutputFiles,
AuthorizationMiddleware.ensureUserCanReadProject,
CompileController.getOutputZipFromClsi
)
webRouter.get(
'/project/:Project_id/user/:user_id/build/:build_id/output/output.zip',
rateLimiterMiddlewareOutputFiles,
AuthorizationMiddleware.ensureUserCanReadProject,
CompileController.getOutputZipFromClsi
)
// direct url access to output files for a specific build // direct url access to output files for a specific build
webRouter.get( webRouter.get(
/^\/project\/([^/]*)\/build\/([0-9a-f-]+)\/output\/(.*)$/, /^\/project\/([^/]*)\/build\/([0-9a-f-]+)\/output\/(.*)$/,

View File

@@ -237,7 +237,9 @@ module.exports = {
}, },
clsi: { clsi: {
url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`, url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`,
// url: "http://#{process.env['CLSI_LB_HOST']}:3014" downloadHost: process.env.CLSI_LB_IP
? `http://${process.env.CLSI_LB_IP}:80`
: `http://${process.env.DOWNLOAD_HOST || '127.0.0.1'}:8080`,
backendGroupName: undefined, backendGroupName: undefined,
submissionBackendClass: submissionBackendClass:
process.env.CLSI_SUBMISSION_BACKEND_CLASS || 'c3d', process.env.CLSI_SUBMISSION_BACKEND_CLASS || 'c3d',

View File

@@ -67,6 +67,7 @@ module.exports = {
}, },
clsi: { clsi: {
url: 'http://127.0.0.1:23013', url: 'http://127.0.0.1:23013',
downloadHost: 'http://127.0.0.1:23080',
}, },
realTime: { realTime: {
url: 'http://127.0.0.1:23026', url: 'http://127.0.0.1:23026',

View File

@@ -4,6 +4,7 @@ import Features from '../../../app/src/infrastructure/Features.mjs'
import MockAnalyticsApi from './mocks/MockAnalyticsApi.mjs' import MockAnalyticsApi from './mocks/MockAnalyticsApi.mjs'
import MockChatApi from './mocks/MockChatApi.mjs' import MockChatApi from './mocks/MockChatApi.mjs'
import MockClsiApi from './mocks/MockClsiApi.mjs' import MockClsiApi from './mocks/MockClsiApi.mjs'
import MockClsiNginxApi from './mocks/MockClsiNginxApi.mjs'
import MockDocstoreApi from './mocks/MockDocstoreApi.mjs' import MockDocstoreApi from './mocks/MockDocstoreApi.mjs'
import MockDocUpdaterApi from './mocks/MockDocUpdaterApi.mjs' import MockDocUpdaterApi from './mocks/MockDocUpdaterApi.mjs'
import MockGitBridgeApi from './mocks/MockGitBridgeApi.mjs' import MockGitBridgeApi from './mocks/MockGitBridgeApi.mjs'
@@ -22,6 +23,7 @@ const mockOpts = {
MockChatApi.initialize(23010, mockOpts) MockChatApi.initialize(23010, mockOpts)
MockClsiApi.initialize(23013, mockOpts) MockClsiApi.initialize(23013, mockOpts)
MockClsiNginxApi.initialize(23080, mockOpts)
MockDocstoreApi.initialize(23016, mockOpts) MockDocstoreApi.initialize(23016, mockOpts)
MockDocUpdaterApi.initialize(23003, mockOpts) MockDocUpdaterApi.initialize(23003, mockOpts)
MockNotificationsApi.initialize(23042, mockOpts) MockNotificationsApi.initialize(23042, mockOpts)

View File

@@ -1,5 +1,4 @@
import AbstractMockApi from './AbstractMockApi.mjs' import AbstractMockApi from './AbstractMockApi.mjs'
import { plainTextResponse } from '../../../../app/src/infrastructure/Response.mjs'
class MockClsiApi extends AbstractMockApi { class MockClsiApi extends AbstractMockApi {
static compile(req, res) { static compile(req, res) {
@@ -9,13 +8,13 @@ class MockClsiApi extends AbstractMockApi {
error: null, error: null,
outputFiles: [ outputFiles: [
{ {
url: `http://clsi:3013/project/${req.params.project_id}/build/1234/output/output.pdf`, url: `http://clsi:8080/project/${req.params.project_id}/build/1234/output/output.pdf`,
path: 'output.pdf', path: 'output.pdf',
type: 'pdf', type: 'pdf',
build: 1234, build: 1234,
}, },
{ {
url: `http://clsi:3013/project/${req.params.project_id}/build/1234/output/output.log`, url: `http://clsi:8080/project/${req.params.project_id}/build/1234/output/output.log`,
path: 'output.log', path: 'output.log',
type: 'log', type: 'log',
build: 1234, build: 1234,
@@ -32,27 +31,6 @@ class MockClsiApi extends AbstractMockApi {
MockClsiApi.compile MockClsiApi.compile
) )
this.app.get(
'/project/:project_id/build/:build_id/output/*',
(req, res) => {
const filename = req.params[0]
if (filename === 'output.pdf') {
plainTextResponse(res, 'mock-pdf')
} else if (filename === 'output.log') {
plainTextResponse(res, 'mock-log')
} else {
res.sendStatus(404)
}
}
)
this.app.get(
'/project/:project_id/user/:user_id/build/:build_id/output/:output_path',
(req, res) => {
plainTextResponse(res, 'hello')
}
)
this.app.get('/project/:project_id/status', (req, res) => { this.app.get('/project/:project_id/status', (req, res) => {
res.sendStatus(200) res.sendStatus(200)
}) })

View File

@@ -0,0 +1,37 @@
import AbstractMockApi from './AbstractMockApi.mjs'
import { plainTextResponse } from '../../../../app/src/infrastructure/Response.mjs'
class MockClsiNginxApi extends AbstractMockApi {
applyRoutes() {
this.app.get(
'/project/:project_id/build/:build_id/output/*',
(req, res) => {
const filename = req.params[0]
if (filename === 'output.pdf') {
plainTextResponse(res, 'mock-pdf')
} else if (filename === 'output.log') {
plainTextResponse(res, 'mock-log')
} else {
res.sendStatus(404)
}
}
)
this.app.get(
'/project/:project_id/user/:user_id/build/:build_id/output/:output_path',
(req, res) => {
plainTextResponse(res, 'hello')
}
)
}
}
export default MockClsiNginxApi
// type hint for the inherited `instance` method
/**
* @function instance
* @memberOf MockClsiNginxApi
* @static
* @returns {MockClsiNginxApi}
*/

View File

@@ -42,7 +42,8 @@ describe('CompileController', function () {
ctx.settings = { ctx.settings = {
apis: { apis: {
clsi: { clsi: {
url: 'http://clsi.example.com', url: 'http://clsi.example.com:3013',
downloadHost: 'http://clsi.example.com:8080',
submissionBackendClass: 'c3d', submissionBackendClass: 'c3d',
}, },
clsi_priority: { clsi_priority: {
@@ -776,7 +777,7 @@ describe('CompileController', function () {
it('should open a request to the CLSI', function (ctx) { it('should open a request to the CLSI', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d&query=foo` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d&query=foo`
) )
}) })
@@ -806,7 +807,7 @@ describe('CompileController', function () {
it('should open a request to the CLSI', function (ctx) { it('should open a request to the CLSI', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`
) )
}) })
}) })
@@ -826,7 +827,7 @@ describe('CompileController', function () {
}) })
ctx.fetchUtils.fetchStreamWithResponse.rejects( ctx.fetchUtils.fetchStreamWithResponse.rejects(
new RequestFailedError( new RequestFailedError(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`,
{ method: 'GET' }, { method: 'GET' },
{ status: 404 } { status: 404 }
) )
@@ -844,7 +845,7 @@ describe('CompileController', function () {
it('should open a request to the CLSI', function (ctx) { it('should open a request to the CLSI', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`
) )
}) })
@@ -873,7 +874,7 @@ describe('CompileController', function () {
}) })
ctx.fetchUtils.fetchStreamWithResponse.rejects( ctx.fetchUtils.fetchStreamWithResponse.rejects(
new RequestFailedError( new RequestFailedError(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`, `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`,
{ method: 'GET' }, { method: 'GET' },
{ status: 404 } { status: 404 }
) )
@@ -891,7 +892,7 @@ describe('CompileController', function () {
it('should open a request to the CLSI', function (ctx) { it('should open a request to the CLSI', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=priority&compileBackendClass=c4d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=priority&compileBackendClass=c4d`
) )
}) })
@@ -924,7 +925,7 @@ describe('CompileController', function () {
it('should open a request to the CLSI', function (ctx) { it('should open a request to the CLSI', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d`
) )
}) })
@@ -955,7 +956,7 @@ describe('CompileController', function () {
it('should proxy to the standard url', function (ctx) { it('should proxy to the standard url', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d`
) )
}) })
}) })
@@ -982,7 +983,7 @@ describe('CompileController', function () {
it('should proxy to the standard url without the build parameter', function (ctx) { it('should proxy to the standard url without the build parameter', function (ctx) {
ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith( ctx.fetchUtils.fetchStreamWithResponse.should.have.been.calledWith(
`${ctx.settings.apis.clsi.url}${ctx.url}?compileGroup=standard&compileBackendClass=c3d` `${ctx.settings.apis.clsi.downloadHost}${ctx.url}?compileGroup=standard&compileBackendClass=c3d`
) )
}) })
}) })