mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-13 21:53:43 +01:00
* Add png to pdf converter for optimising image loading in pdftex * refactor * move fixtures * refactor and add unit tests * Remove gamma correction from PNG processing and update related tests * Add crc32fast dependency and enhance PNG processing tests * Add fixture test cases for PNG processing * fix alpha channel to match pdftex pdftex always uses an 8 bit alpha channel in the output even for 16 bit images * Refactor PNG processing to use Option for alpha stream and update related logic * Enhance PDF writing by adding DecodeParms for alpha stream and optimize PNG encoding process * Add pixel dimensions support in PNG processing and PDF writing * [copybara] move clsi excludes * [clsi] move gitignore rules * [png2pdf] move into dockerfiles folder * [png2pdf] use atomic writes * [clsi] integrate png2pdf * [saas-e2e] add coverage for png2pdf * [png2pdf] make compatible with seccomp profile in clsi * [copybara] remove exclude following move of png2pdf * [copybara] remove spurious excludes * fix broken detection of malformed chunk headers Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * [project-history] add build script flag for depending on doc-updater types * [png2pdf] run tests as part of clsi pipeline * [png2pdf] fix calculation of PDF page size * [web] put png2pdf behind feature flag and make it opt-in * [web] fix ClsiManager.promises.sendRequest invocation after merge * extend is_copy_safe function to follow pdftex see the conditions in pdftex's writepng.c https://svn.tug.org:8369/texlive/trunk/Build/source/texk/web2c/pdftexdir/writepng.c?revision=78081&view=markup#l575 the criteria are not an exact match because the png crate does not support all the png chunk types https://github.com/image-rs/image-png/issues/116 if the criteria result in a png being converted or skipped the final result should be the same. in one case the conversion will be done by pdftex (at runtime) in the other case the conversion will be done by png2pdf - both conversions should give the same result. * include tests in png2pdf Dockerfile * [clsi] explicitly disable png2pdf conversion for project conversion * [clsi] png2pdf: count converted files * [png2pdf] bump date in docker image tag * [web] make draft-setting option wider when png2pdf is available --------- Co-authored-by: Brian Gough <brian.gough@overleaf.com> GitOrigin-RevId: d0d952ee009bf93c72bba2ffe763ae0160d396e4
264 lines
9.3 KiB
JavaScript
264 lines
9.3 KiB
JavaScript
const Path = require('node:path')
|
|
const os = require('node:os')
|
|
const fs = require('node:fs')
|
|
|
|
const isSpotInstance = process.env.PREEMPTIBLE === 'TRUE'
|
|
const CLSI_SERVER_ID = os.hostname().replace('-ctr', '')
|
|
|
|
module.exports = {
|
|
compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
|
|
|
|
processLifespanLimitMs:
|
|
parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2,
|
|
|
|
catchErrors: process.env.CATCH_ERRORS === 'true',
|
|
|
|
path: {
|
|
compilesDir:
|
|
process.env.CLSI_COMPILES_PATH || Path.resolve(__dirname, '../compiles'),
|
|
outputDir:
|
|
process.env.CLSI_OUTPUT_PATH || Path.resolve(__dirname, '../output'),
|
|
clsiCacheDir:
|
|
process.env.CLSI_CACHE_PATH || Path.resolve(__dirname, '../cache'),
|
|
uploadFolder:
|
|
process.env.CLSI_UPLOAD_PATH || Path.resolve(__dirname, '../uploads'),
|
|
synctexBaseDir(projectId) {
|
|
return Path.join(this.compilesDir, projectId)
|
|
},
|
|
},
|
|
|
|
conversionTimeoutSeconds:
|
|
parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
|
|
pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
|
|
enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
|
|
pdftocairoImage:
|
|
process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
|
|
enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
|
|
png2pdfImage:
|
|
process.env.PNG2PDF_IMAGE || 'quay.io/sharelatex/png2pdf:2026-06-24',
|
|
enablePng2pdfConversions: process.env.ENABLE_PNG2PDF_CONVERSIONS === 'true',
|
|
maxUploadSize: 50 * 1024 * 1024,
|
|
preciousFilePattern: process.env.PRECIOUS_FILE_PATTERN || '',
|
|
|
|
internal: {
|
|
clsi: {
|
|
port: 3013,
|
|
host: process.env.LISTEN_ADDRESS || '127.0.0.1',
|
|
},
|
|
|
|
load_balancer_agent: {
|
|
report_load: process.env.LOAD_BALANCER_AGENT_REPORT_LOAD !== 'false',
|
|
load_port: 3048,
|
|
local_port: 3049,
|
|
allow_maintenance:
|
|
(
|
|
process.env.LOAD_BALANCER_AGENT_ALLOW_MAINTENANCE ?? ''
|
|
).toLowerCase() !== 'false',
|
|
},
|
|
},
|
|
apis: {
|
|
clsi: {
|
|
// Internal requests (used by tests only at the time of writing).
|
|
url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`,
|
|
// External url prefix for output files, e.g. for requests via load-balancers.
|
|
outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`,
|
|
clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID,
|
|
instanceType: process.env.INSTANCE_TYPE,
|
|
zone: process.env.ZONE,
|
|
isSpotInstance,
|
|
|
|
downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:8080',
|
|
},
|
|
clsiPerf: {
|
|
host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${
|
|
process.env.CLSI_PERF_PORT || '3043'
|
|
}`,
|
|
},
|
|
clsiCache: {
|
|
enabled: !!process.env.CLSI_CACHE_INSTANCES,
|
|
shards: JSON.parse(process.env.CLSI_CACHE_INSTANCES || '[]').filter(
|
|
({ zone, readOnly }) => zone === process.env.ZONE && !readOnly
|
|
),
|
|
currentShards: parseInt(process.env.CLSI_CACHE_CURRENT_SHARDS, 10),
|
|
desiredShards: parseInt(process.env.CLSI_CACHE_DESIRED_SHARDS, 10),
|
|
reshardFrom: new Date(process.env.CLSI_CACHE_RESHARD_FROM),
|
|
reshardUntil: new Date(process.env.CLSI_CACHE_RESHARD_UNTIL),
|
|
},
|
|
filestore: {
|
|
url:
|
|
process.env.FILESTORE_DOMAIN_OVERRIDE ||
|
|
`http://${process.env.FILESTORE_HOST || '127.0.0.1'}:3009`,
|
|
},
|
|
},
|
|
|
|
smokeTest: process.env.SMOKE_TEST || false,
|
|
project_cache_length_ms: 1000 * 60 * 60 * 24,
|
|
parallelFileDownloads:
|
|
parseInt(process.env.FILESTORE_PARALLEL_FILE_DOWNLOADS, 10) || 1,
|
|
filestoreDomainOveride: process.env.FILESTORE_DOMAIN_OVERRIDE,
|
|
texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
|
|
texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
|
|
texliveMaxPrintLine: process.env.TEXLIVE_MAX_PRINT_LINE,
|
|
enablePdfCaching: process.env.ENABLE_PDF_CACHING === 'true',
|
|
enablePdfCachingDark: process.env.ENABLE_PDF_CACHING_DARK === 'true',
|
|
pdfCachingMinChunkSize:
|
|
parseInt(process.env.PDF_CACHING_MIN_CHUNK_SIZE, 10) || 1024,
|
|
pdfCachingMaxProcessingTime:
|
|
parseInt(process.env.PDF_CACHING_MAX_PROCESSING_TIME, 10) || 10 * 1000,
|
|
pdfCachingEnableWorkerPool:
|
|
process.env.PDF_CACHING_ENABLE_WORKER_POOL === 'true',
|
|
pdfCachingWorkerPoolSize:
|
|
parseInt(process.env.PDF_CACHING_WORKER_POOL_SIZE, 10) || 4,
|
|
pdfCachingWorkerPoolBackLogLimit:
|
|
parseInt(process.env.PDF_CACHING_WORKER_POOL_BACK_LOG_LIMIT, 10) || 40,
|
|
compileConcurrencyLimit: isSpotInstance ? 32 : 64,
|
|
performanceLogSamplingPercentage:
|
|
parseFloat(process.env.CLSI_PERFORMANCE_LOG_SAMPLING, 10) || 0,
|
|
}
|
|
|
|
if (process.env.ALLOWED_COMPILE_GROUPS) {
|
|
try {
|
|
module.exports.allowedCompileGroups =
|
|
process.env.ALLOWED_COMPILE_GROUPS.split(' ')
|
|
} catch (error) {
|
|
console.error(error, 'could not apply allowed compile group setting')
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
|
|
if (
|
|
!fs.existsSync(Path.join(__dirname, '..', 'app', 'js', 'DockerRunner.js'))
|
|
) {
|
|
console.error(
|
|
'Sandboxed compiles are only available with Overleaf Server Pro. Compare Server Pro with Community Edition here: https://docs.overleaf.com/on-premises/welcome/server-pro-vs.-community-edition'
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
module.exports.clsi = {
|
|
dockerRunner: true,
|
|
docker: {
|
|
runtime: process.env.DOCKER_RUNTIME,
|
|
image:
|
|
process.env.TEXLIVE_IMAGE ||
|
|
process.env.TEX_LIVE_DOCKER_IMAGE ||
|
|
'quay.io/sharelatex/texlive-full:2017.1',
|
|
env: {
|
|
HOME: '/tmp',
|
|
CLSI: 1,
|
|
},
|
|
socketPath: '/var/run/docker.sock',
|
|
user: process.env.TEXLIVE_IMAGE_USER || 'tex',
|
|
},
|
|
optimiseInDocker: true,
|
|
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
|
|
checkProjectsIntervalMs: 10 * 60 * 1000,
|
|
}
|
|
|
|
try {
|
|
// Override individual docker settings using path-based keys, e.g.:
|
|
// compileGroupDockerConfigs = {
|
|
// priority: { 'HostConfig.CpuShares': 100 }
|
|
// beta: { 'dotted.path.here', 'value'}
|
|
// }
|
|
const compileGroupConfig = JSON.parse(
|
|
process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
|
|
)
|
|
// Automatically clean up wordcount and synctex containers
|
|
const defaultCompileGroupConfig = {
|
|
wordcount: { 'HostConfig.AutoRemove': true },
|
|
synctex: { 'HostConfig.AutoRemove': true },
|
|
'synctex-output': { 'HostConfig.AutoRemove': true },
|
|
conversions: { 'HostConfig.AutoRemove': true },
|
|
// The png2pdf image is built FROM scratch and has no 'tex' user, so run
|
|
// it as the same uid/gid as this clsi process, which owns the cache files
|
|
// it converts in place.
|
|
png2pdf: {
|
|
'HostConfig.AutoRemove': true,
|
|
User: `${process.getuid()}:${process.getgid()}`,
|
|
},
|
|
}
|
|
module.exports.clsi.docker.compileGroupConfig = Object.assign(
|
|
defaultCompileGroupConfig,
|
|
compileGroupConfig
|
|
)
|
|
} catch (error) {
|
|
console.error(error, 'could not apply compile group docker configs')
|
|
process.exit(1)
|
|
}
|
|
|
|
let seccompProfilePath
|
|
try {
|
|
seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
|
|
module.exports.clsi.docker.seccomp_profile =
|
|
process.env.SECCOMP_PROFILE ||
|
|
JSON.stringify(
|
|
JSON.parse(require('node:fs').readFileSync(seccompProfilePath))
|
|
)
|
|
} catch (error) {
|
|
console.error(
|
|
error,
|
|
`could not load seccomp profile from ${seccompProfilePath}`
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
if (process.env.APPARMOR_PROFILE) {
|
|
try {
|
|
module.exports.clsi.docker.apparmor_profile = process.env.APPARMOR_PROFILE
|
|
} catch (error) {
|
|
console.error(error, 'could not apply apparmor profile setting')
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
if (process.env.NEW_APPARMOR_PROFILE) {
|
|
try {
|
|
module.exports.clsi.docker.new_apparmor_profile =
|
|
process.env.NEW_APPARMOR_PROFILE
|
|
} catch (error) {
|
|
console.error(error, 'could not apply new apparmor profile setting')
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
if (process.env.ALLOWED_IMAGES) {
|
|
try {
|
|
module.exports.clsi.docker.allowedImages =
|
|
process.env.ALLOWED_IMAGES.split(' ')
|
|
} catch (error) {
|
|
console.error(error, 'could not apply allowed images setting')
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
module.exports.path.synctexBaseDir = () => '/compile'
|
|
|
|
module.exports.path.sandboxedCompilesHostDirCompiles =
|
|
process.env.SANDBOXED_COMPILES_HOST_DIR_COMPILES ||
|
|
process.env.SANDBOXED_COMPILES_HOST_DIR ||
|
|
process.env.COMPILES_HOST_DIR
|
|
if (!module.exports.path.sandboxedCompilesHostDirCompiles) {
|
|
throw new Error(
|
|
'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_COMPILES not set'
|
|
)
|
|
}
|
|
|
|
// Host path of the clsi cache dir, so that the png2pdf conversion container
|
|
// (a sibling container started via the docker socket) can bind-mount a
|
|
// project's cache dir and convert downloaded PNGs in place.
|
|
module.exports.path.sandboxedCompilesHostDirCache =
|
|
process.env.SANDBOXED_COMPILES_HOST_DIR_CACHE
|
|
|
|
module.exports.path.sandboxedCompilesHostDirOutput =
|
|
process.env.SANDBOXED_COMPILES_HOST_DIR_OUTPUT ||
|
|
process.env.OUTPUT_HOST_DIR
|
|
if (!module.exports.path.sandboxedCompilesHostDirOutput) {
|
|
// TODO(das7pad): Enforce in a future major version of Server Pro.
|
|
// throw new Error(
|
|
// 'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_OUTPUT not set'
|
|
// )
|
|
}
|
|
}
|