From c7eb257a53b3a71b2e36d1a84ba30b57a95afc1d Mon Sep 17 00:00:00 2001 From: David Rotermund Date: Sat, 24 Oct 2026 14:30:35 +0000 Subject: [PATCH] Restoring the support images: \n png2pdf: \n Without this cache you need to convert png to pdf manualy or you risk a compile timeout (had this problem yesterday and did the conversion manualy. Compile time reduction was from time out to 3sec.)\n env settings required:\n (mount point INSIDE the container)\n CLSI_CACHE_PATH: /var/lib/overleaf/data/cache \n (Mount point outside of the container i.e. host, based on your compose volume rules)\n SANDBOXED_COMPILES_HOST_DIR_CACHE: points to the same folder outside of the container\n Test parameter:\n PNG2PDF_MIN_FILE_SIZE_BYTES: 1024 \n ENABLE_PNG2PDF_CONVERSIONS: true \n PNG2PDF_IMAGE: sharelatex/png2pdf:2026-06-24 \n\n It acts on slow png conversions and not all. Hence the texlive image needs this detection hook as the creator intended it. I added them to:\n https://github.com/davrot/texlive-full/tree/png2pdf-slow \n It is a modification of the LatexMK file. Without the hook no png files are cached as pdf.\n\n pdftocairo: \n The creater intended that pdf to thumbnail / preview are created by an external container for safety reasons. \n The only action we do a to-thumbnail-conversion is in the template generation. Since we don't want to disapoint the creater, I restored this safety feature.\n env settings required:\n (default: local imagemagick; other options are local pdftocairo and container pdftocairo via pdftocairo-sandbox)\n CONVERTER: pdftocairo-sandbox \n CONVERSION_SANDBOX_IMAGE: sharelatex/pdftocairo:24.02 \n (mount point INSIDE the container)\n CONVERSION_SANDBOX_DIR: /var/lib/overleaf/tmp/conversions \n (Mount point outside of the container i.e. host, based on your compose volume rules)\n CONVERSION_SANDBOX_HOST_DIR: points to the same folder outside of the container\n --- server-ce/Dockerfile-pdftocairo | 24 ++ server-ce/Dockerfile-png2pdf | 51 +++ server-ce/Makefile | 40 +- server-ce/png2pdf/png2pdf.sh | 83 +++++ server-ce/runit/filestore-overleaf/run | 13 + services/clsi/app/js/DockerRunner.mjs | 66 ++-- services/clsi/config/settings.defaults.cjs | 10 +- services/filestore/app/js/FileConverter.js | 54 +++ .../app/js/SandboxConverterRunner.js | 342 ++++++++++++++++++ .../filestore/config/settings.defaults.cjs | 21 ++ .../app/src/Features/Compile/ClsiManager.mjs | 4 +- .../Features/Compile/CompileController.mjs | 3 +- .../Features/Project/ProjectController.mjs | 6 +- services/web/config/settings.defaults.js | 6 + 14 files changed, 686 insertions(+), 37 deletions(-) create mode 100644 server-ce/Dockerfile-pdftocairo create mode 100644 server-ce/Dockerfile-png2pdf create mode 100644 server-ce/png2pdf/png2pdf.sh create mode 100644 services/filestore/app/js/SandboxConverterRunner.js diff --git a/server-ce/Dockerfile-pdftocairo b/server-ce/Dockerfile-pdftocairo new file mode 100644 index 0000000000..7643350339 --- /dev/null +++ b/server-ce/Dockerfile-pdftocairo @@ -0,0 +1,24 @@ +# replacement for quay.io/sharelatex/pdftocairo:24.02 (clsitex PDF→JPEG) +# +# ALIASES: pdftocairo-ol:25.12.0 +# +# alpine:3.24 with poppler (25.12.0, >= 24.02 required by CLSI). +# +# Why: CLSI convertPDFToJPEG runs `pdftocairo -jpeg ...` against this image +# for preview (794px, q90) and thumbnail (190px, q50) JPEGs of compiled PDFs. +# +# The SaaS image is Ubuntu-based; alpine 3.24's poppler 25.12.0 covers exactly +# the flags CLSI uses (-jpeg -jpegopt -singlefile -scale-to-x -scale-to-y), and +# the image is tiny. +# +# images/pandoc-amd64/Dockerfile is the sibling (pandoc/core + zip). Same pattern: +# ENTRYPOINT [] because CLSI docker-runs with an explicit command (no +# --entrypoint override). + +FROM alpine:3.24 + +ENTRYPOINT [] + +RUN apk update \ + && apk add --no-cache poppler-utils \ + && rm -rf /var/cache/apk/* diff --git a/server-ce/Dockerfile-png2pdf b/server-ce/Dockerfile-png2pdf new file mode 100644 index 0000000000..dd06b3c985 --- /dev/null +++ b/server-ce/Dockerfile-png2pdf @@ -0,0 +1,51 @@ +# replacement for quay.io/sharelatex/png2pdf:2026-06-24 (clsitex PNG→PDF) +# +# Engine: img2pdf (josch) — lossless. For PNG input the IDAT streams are +# embedded verbatim into the PDF (no re-encode); the alpha channel becomes +# a PDF soft mask (/SMask), also lossless. That is the mechanism of the +# whole feature: pdfTeX copies PDF image streams cheaply, while PNGs with +# alpha/gamma/palette/interlace are re-decoded on every compile, which is +# what made them "slow" and what fills CLSI's slow-PNG list. +# +# The SaaS tool is proprietary. The Go port (go/services/clsitex/png2pdf) +# pins the exact container contract this image fulfils: +# ENTRYPOINT + ["--in-place", "--", "", ...] CWD=/compile +# stdout: one "Converted to PDF" line per file (counted by Go) +# exit non-zero on first failure (CLSI falls back to the original PNGs) +# +# Why git@commit instead of `pip install img2pdf==0.6.3`: +# 0.6.3 (2025-11-06) is the last PyPI release, and it PRE-DATES the +# parse_png quadratic-rewrite fix (2025-11-16, "my 311MB ARCHD at 600dpi +# with 17638 IDAT records went from many minutes to 320ms"). The slow-PNG +# list is populated by BIG PNGs (min 1MB, no upper bound), so we pin main +# at this commit: it is still reproducible (exact hash) and carries the +# fix plus the JPEG2000/ICC and pikepdf-writer-close follow-ups. +# +# pikepdf + pillow have musllinux cp314 wheels for this python version, so +# no compiler is needed; git is needed only for the pinned checkout. + +FROM python:3.14-alpine + +ARG IMG2PDF_REF=0fafa2c312be50c68a07e2331a89bf4adbd274ed + +RUN apk update \ + && apk add --no-cache git \ + && pip install --no-cache-dir --disable-pip-version-check \ + "git+https://gitlab.mister-muffin.de/josch/img2pdf.git@${IMG2PDF_REF}#egg=img2pdf" \ + pikepdf==10.15.0 \ + pillow==12.3.0 \ + packaging \ + && rm -rf /root/.cache/pip /usr/src/python/lib* + +# shim: arg parsing, leading-dash path handling (paths derive from URL +# paths and start with '-'), temp-file + atomic replace (input must survive +# a failed conversion — CLSI renames the untouched original as fallback), +# and the "Converted to PDF" stdout line Go counts. +COPY server-ce/png2pdf/png2pdf.sh /usr/local/bin/png2pdf +RUN chmod 0755 /usr/local/bin/png2pdf + +# The Go runner passes the arg list ["--in-place", "--", ...] as the container +# command, never a binary name — see the pdftocairo image (ENTRYPOINT []) for +# the contrast: that image provides the binary NAME in its command, this one +# relies on the image's own ENTRYPOINT. +ENTRYPOINT ["/usr/local/bin/png2pdf"] diff --git a/server-ce/Makefile b/server-ce/Makefile index ed5857eac0..9d88963389 100644 --- a/server-ce/Makefile +++ b/server-ce/Makefile @@ -14,23 +14,30 @@ export OVERLEAF_TAG ?= sharelatex/sharelatex:$(BRANCH_NAME_TAG_SAFE)-$(MONOREPO_ export OVERLEAF_PANDOC_BRANCH ?= sharelatex/sharelatex-pandoc:$(BRANCH_NAME_TAG_SAFE) export OVERLEAF_PANDOC_LATEST ?= sharelatex/sharelatex-pandoc export OVERLEAF_PANDOC_TAG ?= sharelatex/sharelatex-pandoc:$(BRANCH_NAME_TAG_SAFE)-$(MONOREPO_REVISION) +export OVERLEAF_PDFTOCAIRO ?= sharelatex/pdftocairo:24.02 +export OVERLEAF_PNG2PDF ?= sharelatex/png2pdf:2026-06-24 -all: build-base build-community build-pandoc + +all: build-base build-community build-pandoc build-pdftocairo build-png2pdf clean: - -docker rmi --force $(OVERLEAF_BASE_TAG) $(OVERLEAF_TAG) $(OVERLEAF_PANDOC_TAG) + -docker rmi --force $(OVERLEAF_BASE_TAG) $(OVERLEAF_TAG) $(OVERLEAF_PANDOC_TAG) $(OVERLEAF_PDFTOCAIRO) $(OVERLEAF_PNG2PDF) refresh-cache: refresh-cache-branch refresh-cache-branch: docker inspect $(OVERLEAF_BASE_BRANCH) > /dev/null && docker pull $(OVERLEAF_BASE_BRANCH) || true docker inspect $(OVERLEAF_BRANCH) > /dev/null && docker pull $(OVERLEAF_BRANCH) || true docker inspect $(OVERLEAF_PANDOC_BRANCH) > /dev/null && docker pull $(OVERLEAF_PANDOC_BRANCH) || true + docker inspect $(OVERLEAF_PDFTOCAIRO) > /dev/null && docker pull $(OVERLEAF_PDFTOCAIRO) || true + docker inspect $(OVERLEAF_PNG2PDF) > /dev/null && docker pull $(OVERLEAF_PNG2PDF) || true refresh-cache: refresh-cache-latest refresh-cache-latest: docker inspect $(OVERLEAF_BASE_LATEST) > /dev/null && docker pull $(OVERLEAF_BASE_LATEST) || true docker inspect $(OVERLEAF_LATEST) > /dev/null && docker pull $(OVERLEAF_LATEST) || true docker inspect $(OVERLEAF_PANDOC_LATEST) > /dev/null && docker pull $(OVERLEAF_PANDOC_LATEST) || true + docker inspect $(OVERLEAF_PDFTOCAIRO) > /dev/null && docker pull $(OVERLEAF_PDFTOCAIRO) || true + docker inspect $(OVERLEAF_PNG2PDF) > /dev/null && docker pull $(OVERLEAF_PNG2PDF) || true build-base: cp .dockerignore $(MONOREPO_ROOT) @@ -45,7 +52,6 @@ build-base: --tag $(OVERLEAF_BASE_BRANCH) \ $(MONOREPO_ROOT) - build-community: cp .dockerignore $(MONOREPO_ROOT) docker build \ @@ -74,6 +80,30 @@ build-pandoc: --tag $(OVERLEAF_PANDOC_BRANCH) \ $(MONOREPO_ROOT) +build-pdftocairo: + cp .dockerignore $(MONOREPO_ROOT) + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --progress=plain \ + --build-arg OVERLEAF_BASE_TAG \ + --label "com.overleaf.ce.revision=$(MONOREPO_REVISION)" \ + --cache-from $(OVERLEAF_PDFTOCAIRO) \ + --file Dockerfile-pdftocairo \ + --tag $(OVERLEAF_PDFTOCAIRO) \ + $(MONOREPO_ROOT) + +build-png2pdf: + cp .dockerignore $(MONOREPO_ROOT) + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --progress=plain \ + --build-arg OVERLEAF_BASE_TAG \ + --label "com.overleaf.ce.revision=$(MONOREPO_REVISION)" \ + --cache-from $(OVERLEAF_PNG2PDF) \ + --file Dockerfile-png2pdf \ + --tag $(OVERLEAF_PNG2PDF) \ + $(MONOREPO_ROOT) + push: docker push $(OVERLEAF_BASE_TAG) docker push $(OVERLEAF_BASE_BRANCH) @@ -81,6 +111,8 @@ push: docker push $(OVERLEAF_BRANCH) docker push $(OVERLEAF_PANDOC_TAG) docker push $(OVERLEAF_PANDOC_BRANCH) + docker push $(OVERLEAF_PDFTOCAIRO) + docker push $(OVERLEAF_PNG2PDF) SHELLCHECK_OPTS = \ --shell=bash \ @@ -102,5 +134,5 @@ shellcheck_fix: done .PHONY: all \ - build-base build-community build-pandoc\ + build-base build-community build-pandoc build-pdftocairo build-png2pdf\ shellcheck shellcheck_fix diff --git a/server-ce/png2pdf/png2pdf.sh b/server-ce/png2pdf/png2pdf.sh new file mode 100644 index 0000000000..88e5a2d3d3 --- /dev/null +++ b/server-ce/png2pdf/png2pdf.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# CLSI "png2pdf" container contract shim (engine: img2pdf 0.6.3, josch). +# Replaces the proprietary quay.io/sharelatex/png2pdf:2026-06-24. +# +# Invoked by the CLSI docker runner (go/services/clsitex/dockerrunner.go, go/services/clsitex/png2pdf/png2pdf.go): +# args: ["--in-place", "--", "", ...] CWD=/compile +# the docker runner does NOT supply a binary name in the command, so the +# image ENTRYPOINT is this script. +# +# Paths: each arg is a conversion path CLSI downloaded, RELATIVE to the +# mounted cache dir (CWD /compile). Real CLSI derives it from the file +# URL path (u.Path: '/'->'-') + mtime + ".opt<10-hex cacheKey>", +# e.g. ".-uploads-filestore-a1b2c3d4-0-1-1756600000000.opta1b2c3d4a1bc" +# (leading dash, no directory separator). +# +# "IN-PLACE" means "rewrite each input PATH as a PDF". CLSI then renames +# each path to its own ".opt" cache entry and copies it into the compile +# dir (urlcache.CommitConversion) — PER FILE and UNCONDITIONALLY (sync.go: +# a conversion error only logs a warning). So each converted file lands +# in the cache even in a partially-failed run; an unconverted file is +# committed as the original-PNG fallback. +# +# Contract (go/services/clsitex/png2pdf/png2pdf.go): +# - stdout: one "Converted to PDF" per converted file (Go counts +# lines matching ^Converted .+ to PDF$ -> stats.png2pdf). +# - exit non-zero if anything fails: CLSI logs a warning and commits +# each file's current on-disk state (converted or original). +# +# Why img2pdf: LOSSLESS. PNG IDAT is embedded verbatim into the PDF (no +# re-encode); the alpha channel becomes a PDF /SMask, also lossless. +# That is the mechanism of the feature: pdfTeX copies PDF image streams +# cheaply, whereas PNGs with alpha/gamma/palette/interlace are re-decoded +# from the PNG on every compile — what made them "slow". Without +# --imgsize the image keeps its pHYs DPI, falling back to 96dpi (the same +# default pdfLaTeX itself uses). +# +# Failure handling: convert to a temp + atomic replace, and CONTINUE with +# the remaining files after a failure (their valid conversions are +# committed by CLSI even though the run is reported failed). A failed +# file stays exactly the original PNG CLSI downloaded. + +set -u + +# argv is exactly [--in-place]... ["--"] . The loop terminates: +# every iteration either shifts, breaks, or exits. +while [ "$#" -gt 0 ]; do + case "$1" in + --) shift ; break ;; + --in-place) shift ;; + *) echo "png2pdf: unexpected argument: $1" >&2 ; exit 2 ;; + esac +done + +if [ "$#" -lt 1 ]; then + echo "png2pdf: no input files (usage: --in-place -- ...)" >&2 + exit 2 +fi + +rc=0 +for file in "$@"; do + # Resolve against CWD so a leading '-' (or '.') never lands in an argv + # slot of img2pdf/mv, where it would be parsed as an option. + case "$file" in + / | /*) abs="$file" ;; + *) abs="$(pwd)/$file" ;; + esac + tmp="${abs}.tmp" + + if ! img2pdf --output "$tmp" "$abs"; then + echo "png2pdf: conversion failed for $file" >&2 + rm -f -- "$tmp" + rc=1 + continue + fi + if ! mv -- "$tmp" "$abs"; then + echo "png2pdf: failed to replace $file" >&2 + rm -f -- "$tmp" + rc=1 + continue + fi + echo "Converted $file to PDF" +done +exit "$rc" diff --git a/server-ce/runit/filestore-overleaf/run b/server-ce/runit/filestore-overleaf/run index 66329e9b4a..8657fba498 100755 --- a/server-ce/runit/filestore-overleaf/run +++ b/server-ce/runit/filestore-overleaf/run @@ -1,5 +1,18 @@ #!/bin/bash +# Grant the service user access to the Docker socket group, same as the CLSI +# run script. Needed for CONVERTER=pdftocairo-sandbox (filestore launches the +# sibling conversion container through /var/run/docker.sock). Idempotent: a +# fresh container may have no such group yet, and CLSI's run script does the +# same thing, so just append to the membership if it exists. +if [ -e '/var/run/docker.sock' ]; then + DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock) + if ! getent group dockeronhost >/dev/null; then + groupadd --non-unique --gid "${DOCKER_GROUP}" dockeronhost + fi + usermod -aG dockeronhost www-data +fi + source /etc/overleaf/env.sh export LISTEN_ADDRESS=127.0.0.1 diff --git a/services/clsi/app/js/DockerRunner.mjs b/services/clsi/app/js/DockerRunner.mjs index 7739b02ffb..18520badff 100644 --- a/services/clsi/app/js/DockerRunner.mjs +++ b/services/clsi/app/js/DockerRunner.mjs @@ -48,7 +48,20 @@ const DockerRunner = { image = `${Settings.texliveImageNameOveride}/${img}` } - if (compileGroup === 'synctex-output') { + if ( + Settings.path.sandboxedCompilesHostDirCache && + directory.startsWith(Settings.path.clsiCacheDir + Path.sep) + ) { + // In: directory = '/projectId' + // sandboxedCompilesHostDirCache = '/host/cache' + // Out: directory = '/host/cache/projectId' + // (png2pdf & friends operate on CLSI's per-project cache dir, which + // is not under the compiles dir) + directory = Path.join( + Settings.path.sandboxedCompilesHostDirCache, + Path.basename(directory) + ) + } else if (compileGroup === 'synctex-output') { // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId' // directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId' // sandboxedCompilesHostDirOutput = '/host/output' @@ -93,32 +106,35 @@ const DockerRunner = { // logOptions = _.clone(options) // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging" logger.debug({ projectId }, 'running docker container') - DockerRunner._runAndWaitForContainer( - options, - volumes, - timeout, - (error, output) => { - if (error && error.statusCode === 500) { - logger.debug( - { err: error, projectId }, - 'error running container so destroying and retrying' - ) - DockerRunner.destroyContainer(name, null, true, error => { - if (error != null) { - return callback(error) - } - DockerRunner._runAndWaitForContainer( - options, - volumes, - timeout, - callback + // Bounded retry: a permanent container create/start failure (e.g. + // "unable to find user" on a FROM-scratch image) must not busy-loop + // forever — surface the error to the caller after a couple of attempts. + let retriesLeft = 2 + const runAttempt = () => { + DockerRunner._runAndWaitForContainer( + options, + volumes, + timeout, + (error, output) => { + if (error && error.statusCode === 500 && retriesLeft > 0) { + retriesLeft -= 1 + logger.debug( + { err: error, projectId, retriesLeft }, + 'error running container so destroying and retrying' ) - }) - } else { - callback(error, output) + DockerRunner.destroyContainer(name, null, true, err => { + if (err != null) { + return callback(err) + } + runAttempt() + }) + } else { + callback(error, output) + } } - } - ) + ) + } + runAttempt() // pass back the container name to allow it to be killed return name diff --git a/services/clsi/config/settings.defaults.cjs b/services/clsi/config/settings.defaults.cjs index 969f663baf..1d180204b1 100644 --- a/services/clsi/config/settings.defaults.cjs +++ b/services/clsi/config/settings.defaults.cjs @@ -172,7 +172,15 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') { wordcount: { 'HostConfig.AutoRemove': true }, synctex: { 'HostConfig.AutoRemove': true }, 'synctex-output': { 'HostConfig.AutoRemove': true }, - conversions: { 'HostConfig.AutoRemove': true }, + // The pdftocairo image is also built FROM scratch with no passwd file, + // so the default named user (www-data) fails at container create with + // "unable to find user ... no matching entries in passwd file". Run it + // as the numeric uid/gid of this clsi process (which owns the files it + // converts), same as the png2pdf image below. + conversions: { + 'HostConfig.AutoRemove': true, + User: `${process.getuid()}:${process.getgid()}`, + }, // The png2pdf image is built FROM scratch and has no 'tex' user, so run // it as the same uid/gid as this clsi process, which owns the cache files // it converts in place. diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index 6030d378c5..b79fe3bad6 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -2,6 +2,7 @@ import metrics from '@overleaf/metrics' import Settings from '@overleaf/settings' import { callbackify } from 'node:util' import SafeExec from './SafeExec.js' +import SandboxConverterRunner from './SandboxConverterRunner.js' import Errors from './Errors.js' const { ConversionError } = Errors @@ -22,6 +23,9 @@ export default { } async function convert(sourcePath, requestedFormat) { + if (Settings.converter === 'pdftocairo-sandbox') { + return await _convertInSandbox(sourcePath, requestedFormat, 1500) + } if (Settings.converter === 'pdftocairo') { const width = 1500 return await _convert(sourcePath, requestedFormat, [ @@ -49,6 +53,9 @@ async function convert(sourcePath, requestedFormat) { } async function thumbnail(sourcePath) { + if (Settings.converter === 'pdftocairo-sandbox') { + return await _convertInSandbox(sourcePath, 'png', 700) + } if (Settings.converter === 'pdftocairo') { const width = 700 return await _convert(sourcePath, 'png', [ @@ -81,6 +88,9 @@ async function thumbnail(sourcePath) { async function preview(sourcePath) { const width = 1000 + if (Settings.converter === 'pdftocairo-sandbox') { + return await _convertInSandbox(sourcePath, 'png', width) + } if (Settings.converter === 'pdftocairo') { return await _convert(sourcePath, 'png', [ 'pdftocairo', @@ -109,6 +119,50 @@ async function preview(sourcePath) { } } +/** + * Run the pdftocairo conversion inside a sandboxed sibling container (no + * network, all caps dropped, memory cap — see SandboxConverterRunner.js). + * + * Keeps the same contract as the local path: the result is written to + * `${sourcePath}.${requestedFormat}`. + * + * @param {string} sourcePath + * @param {string} requestedFormat 'png' or 'jpg' + * @param {number} width output width in px + * @return {Promise<{outputHostPath: string}>} + */ +async function _convertInSandbox(sourcePath, requestedFormat, width) { + if (!APPROVED_FORMATS.includes(requestedFormat)) { + throw new ConversionError('invalid format requested', { format: requestedFormat }) + } + const ext = requestedFormat + const workDir = '/work' + // poppler appends the device extension to the output argument, so the + // argument must end WITHOUT an extension; the result lands at + // /work/input.pdf. next to the input. + const device = requestedFormat === 'png' ? '-png' : '-jpeg' + const result = await SandboxConverterRunner.runConversion({ + sourcePath, + containerArgs: [ + 'pdftocairo', + device, + '-singlefile', + '-scale-to-x', + width.toString(), + '-scale-to-y', + '-1', // maintain aspect ratio + `${workDir}/input.pdf`, + `${workDir}/input.pdf`, + ], + workDir, + outputBase: 'input.pdf', + outputExt: ext, + destPath: `${sourcePath}.${requestedFormat}`, + }) + // Same contract as the local converters: return the destination path string. + return result.outputHostPath +} + async function _convert(sourcePath, requestedFormat, command) { if (!APPROVED_FORMATS.includes(requestedFormat)) { throw new ConversionError('invalid format requested', { diff --git a/services/filestore/app/js/SandboxConverterRunner.js b/services/filestore/app/js/SandboxConverterRunner.js new file mode 100644 index 0000000000..734ba078f9 --- /dev/null +++ b/services/filestore/app/js/SandboxConverterRunner.js @@ -0,0 +1,342 @@ +// SandboxConverterRunner.js +// +// Runs a PDF→image conversion (pdftocairo) in a throwaway sibling container, +// the same isolation profile CLSI's DockerRunner gives its conversion +// containers (see services/clsi/app/js/DockerRunner.mjs and the `conversions` +// / `png2pdf` compile groups in services/clsi/config/settings.defaults.cjs): +// +// - no network access (HostConfig.NetworkMode: 'none') +// - all capabilities dropped (HostConfig.CapDrop: ['ALL']) +// - no new privileges (SecurityOpt: 'no-new-privileges') +// - hard memory cap +// - numeric uid:gid of the overleaf service user (the scratch-based +// converter images have no passwd entries, so named users fail) +// - container is always removed afterwards +// +// Implemented as a minimal Docker Engine API client over the unix socket so +// that no new npm dependency is required (this service is installed with +// Yarn PnP; cross-service packages such as dockerode are not resolvable +// here). +// +// IMPORTANT (bind mounts): the daemon runs on the Docker host, so the +// bind source MUST be a daemon-host-visible path (in this deployment: +// under the overleaf `./data` volume), never a container-internal path +// such as /tmp. The caller passes a `hostDir` that satisfies this. + +import http from 'node:http' +import Path from 'node:path' +import Fs from 'node:fs' +import crypto from 'node:crypto' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import Errors from './Errors.js' + +const { ConversionError } = Errors + +const API_VERSION = 'v1.43' // conservative; the engine API is backwards compatible + +/** + * One request against the Docker Engine API on a unix socket. + * + * @param {string} socketPath + * @param {string} method + * @param {string} apiPath e.g. `/v1.43/containers/create?name=xyz` + * @param {object} [opts] + * @param {object|string} [opts.body] JSON body + * @param {number} [opts.timeoutMs] hard deadline for the whole request + * @param {Buffer|null} [opts.expectRaw] pass a truthy value to keep the body + * as a Buffer (used for the log stream) + * @return {Promise<{status: number, headers: object, body: any, raw: Buffer}>} + */ +function request(socketPath, method, apiPath, { body, timeoutMs = 60000, raw = false } = {}) { + return new Promise((resolve, reject) => { + const data = body === undefined || body === null ? null : JSON.stringify(body) + let settled = false + const req = http.request( + { + socketPath, + method, + path: apiPath, + headers: data + ? { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(data), + } + : {}, + }, + (res) => { + const chunks = [] + res.on('data', (c) => chunks.push(c)) + res.on('end', () => { + if (settled) return + settled = true + clearTimeout(timer) + const buf = Buffer.concat(chunks) + if (raw) return resolve({ status: res.statusCode, headers: res.headers, body: buf, raw: buf }) + let parsed = buf + if (buf.length && (buf[0] === 0x7b || buf[0] === 0x5b)) { + try { + parsed = JSON.parse(buf.toString('utf8')) + } catch (e) { + // non-JSON body (e.g. plain text) — keep buffer + parsed = buf + } + } + resolve({ status: res.statusCode, headers: res.headers, body: parsed, raw: buf }) + }) + } + ) + const timer = setTimeout(() => { + if (settled) return + settled = true + req.destroy() + reject(new Error(`docker api ${method} ${apiPath} timed out after ${timeoutMs}ms`)) + }, timeoutMs) + timer.unref?.() + req.on('error', (err) => { + if (settled) return + settled = true + clearTimeout(timer) + reject(err) + }) + if (data) req.write(data) + req.end() + }) +} + +/** Decode the framed (TTY=false) docker log stream. */ +function decodeLogStream(buf) { + if (!buf || !buf.length) return '' + // 8-byte header per frame: [stream(1)] [size_be32(4)] [reserved(3)] + payload + // (older daemons use a 4-byte header; try the 8-byte layout first) + let out = '' + let i = 0 + while (i + 8 <= buf.length) { + const size = buf.readUInt32BE(i + 4) + if (i + 8 + size > buf.length) break + if (buf[i] === 1 || buf[i] === 2) out += buf.subarray(i + 8, i + 8 + size).toString('utf8') + i += 8 + size + } + return out || buf.toString('utf8') +} + +/** + * Best-effort sweep of leftover `filestore-convert-*` containers from crashes + * or service restarts mid-run (a container that was created but never started + * or removed otherwise would leak forever). + */ +async function _sweepLeftovers(socketPath) { + let res + try { + res = await request( + socketPath, + 'GET', + `/${API_VERSION}/containers/json?all=1&limit=200&filters=%7B%22name%22%3A%5B%22filestore-convert-%22%5D%7D`, + { timeoutMs: 10000 } + ) + } catch (err) { + return // not fatal + } + if (res.status !== 200 || !Array.isArray(res.body)) return + for (const c of res.body) { + if (typeof c.Names?.[0] === 'string' && c.Names[0].startsWith('/filestore-convert-')) { + const name = c.Names[0].slice(1) + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + await request(socketPath, 'DELETE', `/${API_VERSION}/containers/${name}?force=1`, { timeoutMs: 10000 }) + logger.warn({ name }, 'sandbox conversion: removed leftover container') + break + } catch (err) { + if (attempt === 0) continue + logger.warn({ err, name }, 'sandbox conversion: failed to remove leftover container') + } + } + } + } +} + +/** + * Run a conversion in a sandboxed container. + * + * Layout contract (mirrors CLSI's conversion runner): + * host: //input.pdf <- the source file (copied in) + * host: //. <- produced by the container + * + * @param {object} opts + * @param {string} opts.sourcePath absolute path to the source PDF (must be + * inside a daemon-host-visible directory — the bind source is derived from it) + * @param {string} opts.containerArgs full argv inside the container, e.g. + * ['pdftocairo','-png','-singlefile','-scale-to-x','700','-scale-to-y','-1', + * '/work/input.pdf','/work/output'] + * @param {string} opts.workDir basename mapping used in `containerArgs`; the + * source file is placed at /input.pdf inside the container + * @param {string} opts.outputBase container-side output basename (without ext) + * @param {string} opts.outputExt 'png' or 'jpg' + * @param {string} opts.destPath absolute host path where the result must end up + * @return {Promise<{container: string, exitCode: number, outputHostPath: string, durationMs: number}>} + */ +export async function runConversion(opts) { + const { + sourcePath, + containerArgs, + workDir = '/work', + outputBase = 'output', + outputExt = 'png', + destPath, + } = opts + + const sandbox = Settings.sandboxConversion || {} + const image = sandbox.image || 'sharelatex/pdftocairo:24.02' + const socketPath = sandbox.dockerSocketPath || '/var/run/docker.sock' + const hostDir = sandbox.hostDir // daemon-host side of the scratch dir + const containerDir = sandbox.containerDir // overleaf-container side + const memoryBytes = sandbox.memoryBytes || 512 * 1024 * 1024 + const timeoutMs = sandbox.timeoutMs || 40000 + + if (!hostDir || !containerDir) { + throw new ConversionError( + 'sandbox conversion is not configured: set CONVERSION_SANDBOX_HOST_DIR (Docker-daemon side) and CONVERSION_SANDBOX_DIR (overleaf-container side) to the two views of the same directory' + ) + } + + const runId = crypto.randomBytes(8).toString('hex') + const runName = `filestore-convert-${runId}` + const runDir = Path.join(containerDir, runId) // path used by THIS process (container side) + const hostRunDir = Path.join(hostDir, runId) // same dir from the daemon side (bind source) + const inputHostPath = Path.join(runDir, 'input.pdf') + const outputHostPath = Path.join(runDir, `${outputBase}.${outputExt}`) + + const t0 = Date.now() + let containerCreated = false + try { + // Guard against leftover containers from an interrupted previous run. + await _sweepLeftovers(socketPath) + + Fs.mkdirSync(runDir, { mode: 0o770 }) + Fs.copyFileSync(sourcePath, inputHostPath) + + // Container-absolute path of the source file, for argv remapping sanity + const sourceInContainer = Path.join(workDir, 'input.pdf') + if (!containerArgs.some((a) => a === sourceInContainer)) { + throw new ConversionError( + `sandbox argv does not reference ${sourceInContainer} — refusing to run` + ) + } + + // Create (the converter images are scratch-based: no ENTRYPOINT, so a bare + // Cmd list is what CLSI runs successfully too). + const createBody = { + Image: image, + Cmd: containerArgs, + WorkingDir: workDir, + // scratch-based images have no passwd entries — run as the numeric + // uid:gid of this service (it owns the files on both sides of the bind). + User: `${process.getuid()}:${process.getgid()}`, + HostConfig: { + Binds: [`${hostRunDir}:${workDir}:rw`], + CapDrop: ['ALL'], + SecurityOpt: ['no-new-privileges'], + NetworkMode: 'none', + Memory: memoryBytes, + MemorySwap: 0, + PidsLimit: 64, + }, + } + const created = await request(socketPath, 'POST', `/${API_VERSION}/containers/create?name=${runName}`, { + body: createBody, + timeoutMs: 60000, + }) + if (created.status >= 300 || !created.body?.Id) { + const msg = created.body?.message || JSON.stringify(created.body) || `status ${created.status}` + throw new ConversionError(`failed to create sandbox container: ${msg}`) + } + containerCreated = true + logger.info({ image, runName, hostDir: hostRunDir, containerDir: runDir }, 'sandbox conversion: container created') + + await request(socketPath, 'POST', `/${API_VERSION}/containers/${runName}/start`, { timeoutMs: 60000 }) + + // Wait with a real deadline; kill on breach (the /wait call is a blocking + // HTTP call that carries no data, so it gets its own generous budget). + let exitCode = -1 + let killed = false + try { + // NOTE: this daemon (Docker 29 / API ≥1.44) requires POST for /wait; GET + // returns 404. (dockerode does the same, which is how CLSI waits.) + const waited = await request(socketPath, 'POST', `/${API_VERSION}/containers/${runName}/wait`, { + timeoutMs: timeoutMs + 15000, + }) + exitCode = waited.body?.StatusCode ?? -1 + if (exitCode === -1 && waited.status !== 200) { + throw new Error(`unexpected /wait status ${waited.status}: ${waited.raw?.toString?.('utf8') || 'no body'}`) + } + } catch (waitErr) { + killed = true + await _killAndRemove(socketPath, runName) + throw new ConversionError(`sandbox conversion timed out (${timeoutMs}ms): ${waitErr.message}`) + } + + let logs = '' + try { + const logsRes = await request( + socketPath, + 'GET', + `/${API_VERSION}/containers/${runName}/logs?stdout=1&stderr=1&stdin=0&follow=1`, + { raw: true, timeoutMs: 10000 } + ) + logs = decodeLogStream(logsRes.raw) + } catch (err) { + logger.warn({ err, runName }, 'sandbox conversion: could not read container logs') + } + + if (exitCode !== 0) { + throw new ConversionError( + `sandbox conversion failed (exit ${exitCode}${killed ? ', killed' : ''})`, + { stderr: logs.trim().slice(-4000), exitCode } + ) + } + + let stat + try { + stat = Fs.statSync(outputHostPath) + } catch (err) { + throw new ConversionError(`sandbox conversion produced no output file`, { stderr: logs.trim().slice(-4000) }) + } + if (!stat.isFile()) { + throw new ConversionError('sandbox conversion output is not a regular file', { + stderr: logs.trim().slice(-4000), + }) + } + + Fs.copyFileSync(outputHostPath, destPath) + logger.info( + { runName, sizeBytes: stat.size, durationMs: Date.now() - t0 }, + 'sandbox conversion completed' + ) + return { container: runName, exitCode, outputHostPath: destPath, durationMs: Date.now() - t0 } + } finally { + // Best-effort cleanup: container + scratch dir (the dir holds the original + // PDF — do not leave it behind). + if (containerCreated) { + await _killAndRemove(socketPath, runName).catch((e) => { + logger.warn({ err: e, runName }, 'sandbox conversion: cleanup failed') + }) + } + Fs.rmSync(runDir, { recursive: true, force: true }) + } +} + +async function _killAndRemove(socketPath, name) { + try { + // POST /kill (GET is not served by this daemon); 409 = not running → fine + await request(socketPath, 'POST', `/${API_VERSION}/containers/${name}/kill`, { timeoutMs: 10000 }) + } catch (err) { + // not running → fine + } + try { + await request(socketPath, 'DELETE', `/${API_VERSION}/containers/${name}?force=1`, { timeoutMs: 10000 }) + } catch (err) { + // already gone → fine + } +} + +export default { runConversion } diff --git a/services/filestore/config/settings.defaults.cjs b/services/filestore/config/settings.defaults.cjs index 35df525d90..f8f7424132 100644 --- a/services/filestore/config/settings.defaults.cjs +++ b/services/filestore/config/settings.defaults.cjs @@ -99,8 +99,29 @@ const settings = { enableConversions: process.env.ENABLE_CONVERSIONS === 'true', + // PDF→image conversion backend (template gallery previews/thumbnails): + // 'imagemagick' – local, via ImageMagick `convert` (default; binary required) + // 'pdftocairo' – local, via the `pdftocairo` binary (poppler) + // 'pdftocairo-sandbox' – runs `pdftocairo` in a throwaway sibling container: + // no network, CapDrop ALL, no-new-privileges, memory + // cap (see app/js/SandboxConverterRunner.js). + // requires a Docker socket + a daemon-host-visible + // scratch dir (CONVERSION_SANDBOX_HOST_DIR). converter: process.env.CONVERTER || 'imagemagick', + sandboxConversion: { + image: process.env.CONVERSION_SANDBOX_IMAGE || 'sharelatex/pdftocairo:24.02', + dockerSocketPath: process.env.CONVERSION_DOCKER_SOCKET || '/var/run/docker.sock', + // The scratch dir has two views of the same path (the overleaf container + // mounts ./data): `containerDir` for this process, `hostDir` for the Docker + // daemon's bind mount. Both must be set when sandbox conversion is used. + hostDir: process.env.CONVERSION_SANDBOX_HOST_DIR || null, + containerDir: process.env.CONVERSION_SANDBOX_DIR || null, + memoryBytes: + parseInt(process.env.CONVERSION_SANDBOX_MEMORY_BYTES, 10) || 512 * 1024 * 1024, + timeoutMs: parseInt(process.env.CONVERSION_SANDBOX_TIMEOUT_MS, 10) || 40000, + }, + gracefulShutdownDelayInMs: parseInt(process.env.GRACEFUL_SHUTDOWN_DELAY_SECONDS ?? '30', 10) * 1000, } diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index cb2502bc99..114b90c878 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1199,8 +1199,8 @@ function _finaliseRequest(projectId, options, project, docs, files) { timeout: options.timeout, imageName: project.imageName, draft: Boolean(options.draft), - // enable for premium compiles only - png2pdf: Boolean(options.png2pdf) && hasPremiumCompiles, + // CE: premium-compile (alpha/priority) gate removed + png2pdf: Boolean(options.png2pdf), // enable for premium compiles on an image that has a checkpointing build enableCheckpoint: Boolean(options.checkpointing) && diff --git a/services/web/app/src/Features/Compile/CompileController.mjs b/services/web/app/src/Features/Compile/CompileController.mjs index 7e361bac84..5f50cf987b 100644 --- a/services/web/app/src/Features/Compile/CompileController.mjs +++ b/services/web/app/src/Features/Compile/CompileController.mjs @@ -453,7 +453,8 @@ const _CompileController = { enablePng2Pdf, enableCheckpointCompiles, } = await _getSplitTestOptions(req, res) - if (Features.hasFeature('saas')) { + // CE: saas feature gate removed so png2pdf / compile-from-history options are sent + if (true) { options.compileFromClsiCache = true options.populateClsiCache = true options.compileFromHistory = compileFromHistory diff --git a/services/web/app/src/Features/Project/ProjectController.mjs b/services/web/app/src/Features/Project/ProjectController.mjs index a3861d95e1..ada5ced460 100644 --- a/services/web/app/src/Features/Project/ProjectController.mjs +++ b/services/web/app/src/Features/Project/ProjectController.mjs @@ -1108,10 +1108,8 @@ const _ProjectController = { canUseClsiCache: Features.hasFeature('saas') && ownerFeatures?.compileGroup === 'priority', - canUsePng2Pdf: - Features.hasFeature('saas') && - ownerFeatures?.compileGroup === 'priority' && - splitTestAssignments['png2pdf']?.variant === 'enabled', + // CE: saas/priority gate removed so the "Optimise images" toggle is available + canUsePng2Pdf: true, user: { id: userId, email: user.email, diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 07ce05c5bd..a4f6f9ee9d 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1262,6 +1262,12 @@ module.exports = { 'export-html': 'enabled', 'export-markdown': 'enabled', } : {}), + ...(process.env.ENABLE_PNG2PDF_CONVERSIONS === 'true' ? { + 'png2pdf': 'enabled', + // compile-from-history is required for CLSI to record the "slow png" list + // (clsi CompileManager.js saveSlowPngList is gated on isCompileFromHistory) + 'compile-from-history': 'enabled', + } : {}), ...(process.env.OVERLEAF_HISTORY_RESTORE?.toLowerCase() === 'true' ? { 'history-ranges-support': 'enabled', 'revert-file': 'enabled',