From 69ccb3c31608cf5832bdb8da553698eb0af7e33c Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 6 Jul 2026 10:18:17 +0200 Subject: [PATCH] [clsi] integrate png2pdf (#34796) * Add png to pdf converter for optimising image loading in pdftex * refactor * move fixtures * refactor and add unit tests * Remove gamma correction from PNG processing and update related tests * Add crc32fast dependency and enhance PNG processing tests * Add fixture test cases for PNG processing * fix alpha channel to match pdftex pdftex always uses an 8 bit alpha channel in the output even for 16 bit images * Refactor PNG processing to use Option for alpha stream and update related logic * Enhance PDF writing by adding DecodeParms for alpha stream and optimize PNG encoding process * Add pixel dimensions support in PNG processing and PDF writing * [copybara] move clsi excludes * [clsi] move gitignore rules * [png2pdf] move into dockerfiles folder * [png2pdf] use atomic writes * [clsi] integrate png2pdf * [saas-e2e] add coverage for png2pdf * [png2pdf] make compatible with seccomp profile in clsi * [copybara] remove exclude following move of png2pdf * [copybara] remove spurious excludes * fix broken detection of malformed chunk headers Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * [project-history] add build script flag for depending on doc-updater types * [png2pdf] run tests as part of clsi pipeline * [png2pdf] fix calculation of PDF page size * [web] put png2pdf behind feature flag and make it opt-in * [web] fix ClsiManager.promises.sendRequest invocation after merge * extend is_copy_safe function to follow pdftex see the conditions in pdftex's writepng.c https://svn.tug.org:8369/texlive/trunk/Build/source/texk/web2c/pdftexdir/writepng.c?revision=78081&view=markup#l575 the criteria are not an exact match because the png crate does not support all the png chunk types https://github.com/image-rs/image-png/issues/116 if the criteria result in a png being converted or skipped the final result should be the same. in one case the conversion will be done by pdftex (at runtime) in the other case the conversion will be done by png2pdf - both conversions should give the same result. * include tests in png2pdf Dockerfile * [clsi] explicitly disable png2pdf conversion for project conversion * [clsi] png2pdf: count converted files * [png2pdf] bump date in docker image tag * [web] make draft-setting option wider when png2pdf is available --------- Co-authored-by: Brian Gough GitOrigin-RevId: d0d952ee009bf93c72bba2ffe763ae0160d396e4 --- services/clsi/app/js/CompileManager.js | 3 +- services/clsi/app/js/ConversionController.js | 6 +- services/clsi/app/js/HistoryResourceWriter.js | 149 ++++++++++++++--- services/clsi/app/js/Png2Pdf.js | 96 +++++++++++ services/clsi/app/js/RequestParser.js | 4 + services/clsi/app/js/UrlCache.js | 116 ++++++++++--- services/clsi/config/settings.defaults.cjs | 16 ++ .../test/unit/js/ConversionController.test.js | 33 +++- services/clsi/test/unit/js/Png2Pdf.test.js | 156 ++++++++++++++++++ services/clsi/test/unit/js/UrlCache.test.js | 83 ++++++++++ .../app/src/Features/Compile/ClsiManager.mjs | 1 + .../Features/Compile/CompileController.mjs | 4 + .../Features/Project/ProjectController.mjs | 1 + .../web/frontend/extracted-translations.json | 1 + .../compiler-settings/draft-setting.tsx | 49 ++++-- .../components/pdf-compile-button.tsx | 37 ++++- .../js/features/pdf-preview/util/compiler.ts | 2 + .../shared/context/detach-compile-context.tsx | 18 ++ .../shared/context/local-compile-context.tsx | 22 +++ services/web/locales/en.json | 1 + .../settings/draft-setting.test.tsx | 56 +++++-- .../frontend/helpers/editor-providers.tsx | 2 + .../unit/src/Compile/ClsiManager.test.mjs | 23 +++ services/web/types/compile.ts | 1 + 24 files changed, 793 insertions(+), 87 deletions(-) create mode 100644 services/clsi/app/js/Png2Pdf.js create mode 100644 services/clsi/test/unit/js/Png2Pdf.test.js diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 11d730f016..0393ac9e56 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -114,7 +114,8 @@ async function doCompile(request, stats, timings) { userId, request, compileDir, - timings + timings, + stats )) } else { // NOTE: resourceList is insecure, it should only be used to exclude files from the output list diff --git a/services/clsi/app/js/ConversionController.js b/services/clsi/app/js/ConversionController.js index 1e62cd9873..30adf0c3da 100644 --- a/services/clsi/app/js/ConversionController.js +++ b/services/clsi/app/js/ConversionController.js @@ -140,6 +140,9 @@ async function convertProjectToDocument(req, res) { request.project_id = projectId request.user_id = userId request.metricsOpts = {} + // Document conversions reuse the history writer but must never run png2pdf: + // the converted PDF bytes would be fed to pandoc instead of the original png. + request.png2pdf = false const responseFormat = req.query.responseFormat === 'json' ? 'json' : 'stream' @@ -176,7 +179,8 @@ async function convertProjectToDocument(req, res) { userId, request, conversionDir, - {} + {}, // timings + {} // stats ) } catch (err) { if (err instanceof Errors.MissingUpdatesError) { diff --git a/services/clsi/app/js/HistoryResourceWriter.js b/services/clsi/app/js/HistoryResourceWriter.js index 872f41eaeb..df5845d02e 100644 --- a/services/clsi/app/js/HistoryResourceWriter.js +++ b/services/clsi/app/js/HistoryResourceWriter.js @@ -25,10 +25,19 @@ import { promiseMapSettledWithLimit } from '@overleaf/promise-utils' import Metrics from '@overleaf/metrics' import TikzManager from './TikzManager.js' import DraftModeManager from './DraftModeManager.js' +import Png2Pdf from './Png2Pdf.js' const gzip = promisify(zlib.gzip) const gunzip = promisify(zlib.gunzip) +/** + * @param {string} path + * @return {boolean} + */ +function isPng(path) { + return Path.extname(path).toLowerCase() === '.png' +} + export const clearCacheCb = callbackify(clearCache) /** @@ -76,7 +85,7 @@ function isENOENT(err) { * @param {string} cacheKey * @param {number} remoteBaseVersion * @param {boolean} populateClsiCache - * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>} + * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[], png2pdf: boolean}>} */ async function loadSnapshot( projectId, @@ -137,7 +146,7 @@ async function loadSnapshot( * @param {string} userId * @param {string} cacheKey * @param {number} remoteBaseVersion - * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>} + * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[], png2pdf: boolean}>} */ async function loadSnapshotFromClsiCache( projectId, @@ -168,7 +177,7 @@ async function loadSnapshotFromClsiCache( * @param {string} path * @param {number} remoteBaseVersion * @param {boolean} fullSync - * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean, dirty: string[]}>} + * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean, dirty: string[], png2pdf: boolean}>} */ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) { let blob = await fs.promises.readFile(path) @@ -178,13 +187,21 @@ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) { globalBlobs, localBaseVersion, dirty = [], // added later, provide a default value. + png2pdf = false, // the png2pdf mode used for the last sync, added later. } = JSON.parse(blob.toString('utf-8')) if (localBaseVersion < remoteBaseVersion) { throw new Errors.MissingUpdatesError('missing updates', { baseHistoryVersion: localBaseVersion, }) } - return { rawSnapshot, globalBlobs, localBaseVersion, fullSync, dirty } + return { + rawSnapshot, + globalBlobs, + localBaseVersion, + fullSync, + dirty, + png2pdf, + } } /** @@ -193,6 +210,7 @@ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) { * @param {number} localBaseVersion * @param {string[]} globalBlobs * @param {string[]} dirty + * @param {boolean} png2pdf the png2pdf mode used for this sync * @return {Promise} */ async function saveSnapshot( @@ -200,7 +218,8 @@ async function saveSnapshot( snapshot, localBaseVersion, globalBlobs, - dirty + dirty, + png2pdf ) { const { dir, path } = snapshotPath(cacheKey) await fs.promises.mkdir(dir, { recursive: true }) @@ -213,6 +232,7 @@ async function saveSnapshot( localBaseVersion, rawSnapshot: snapshot.toRaw(), dirty, + png2pdf, }), // use cheapest gzip compression level { level: 1 } @@ -363,6 +383,7 @@ function changesFromRawChangeOperations(raw) { * @param {Object} request * @param {string} compileDir * @param {Record} timings + * @param {Record} stats * @return {Promise<{baseHistoryVersion: number, resourceList: {path: string}[]}>} */ export async function syncResourcesToDisk( @@ -370,23 +391,36 @@ export async function syncResourcesToDisk( userId, request, compileDir, - timings + timings, + stats ) { // - logged in user: - // - anonymous user: // - conversion job: const cacheKey = Path.basename(compileDir) const remoteBaseVersion = request.baseHistoryVersion - let rawSnapshot, globalBlobs, localBaseVersion, source, dirty, fullSync + let rawSnapshot, + globalBlobs, + localBaseVersion, + source, + dirty, + fullSync, + lastPng2pdf try { - ;({ rawSnapshot, globalBlobs, fullSync, localBaseVersion, dirty } = - await loadSnapshot( - projectId, - userId, - cacheKey, - remoteBaseVersion, - request.populateClsiCache - )) + ;({ + rawSnapshot, + globalBlobs, + fullSync, + localBaseVersion, + dirty, + png2pdf: lastPng2pdf, + } = await loadSnapshot( + projectId, + userId, + cacheKey, + remoteBaseVersion, + request.populateClsiCache + )) source = fullSync ? 'clsi-cache' : 'local' logger.debug( { projectId, userId, cacheKey, localBaseVersion, remoteBaseVersion }, @@ -431,6 +465,8 @@ export async function syncResourcesToDisk( const entriesDepthFirst = await discoverExistingEntries(compileDir) await removeExtraneousEntries(compileDir, snapshot, entriesDepthFirst) + const pngModeChanged = lastPng2pdf !== request.png2pdf + const changedPaths = [] if (fullSync) { changedPaths.push(...snapshot.getFilePathnames()) @@ -443,6 +479,12 @@ export async function syncResourcesToDisk( if (request.draft) { dedupe.add(request.rootResourcePath) } + if (pngModeChanged) { + // When the png2pdf mode changed since the last sync, the on-disk images are in the wrong variant (optimized vs original). Re-sync them so they are converted (served from the .opt cache) or restored to the original png. + for (const path of snapshot.getFilePathnames()) { + if (isPng(path)) dedupe.add(path) + } + } for (const change of changes) { for (const operation of change.getOperations()) { if (operation instanceof AddFileOperation) { @@ -490,6 +532,8 @@ export async function syncResourcesToDisk( const wasDirty = dirty.length > 0 dirty = [] let createCacheFolder + const pngFilesToConvert = [] + // Use Promise.allSettled to ensure that all writes have stopped when we exit. const allDone = await promiseMapSettledWithLimit( Settings.parallelFileDownloads, @@ -526,16 +570,33 @@ export async function syncResourcesToDisk( } await createCacheFolder const url = blobStore.getBlobURL(hash).href + const destPath = Path.join(compileDir, path) try { const fallbackURL = null // no fallback const lastModified = new Date(0) // content is static - await UrlCache.promises.downloadUrlToFile( - projectId, - url, - fallbackURL, - Path.join(compileDir, path), - lastModified - ) + if (request.png2pdf && Png2Pdf.isEnabled() && isPng(path)) { + // PNG files go through a batch conversion process first. + const toConvert = await UrlCache.promises.downloadUrlToFile( + projectId, + url, + fallbackURL, + destPath, + lastModified, + // Avoid sharing the conversion file between two users. + cacheKey + ) + if (toConvert) { + pngFilesToConvert.push(toConvert) + } + } else { + await UrlCache.promises.downloadUrlToFile( + projectId, + url, + fallbackURL, + destPath, + lastModified + ) + } } catch (err) { logger.err( { err, projectId, path, resourceUrl: url }, @@ -551,14 +612,54 @@ export async function syncResourcesToDisk( const path = changedPaths[idx] throw OError.tag(result.reason, 'write failed', { path }) } + + if (pngFilesToConvert.length) { + const cacheDir = UrlCache.getProjectCacheDir(projectId) + try { + await Png2Pdf.convertPngFilesInCacheDir( + projectId, + cacheDir, + pngFilesToConvert.map(f => Path.relative(cacheDir, f.conversionPath)), + stats, + timings + ) + } catch (err) { + logger.warn( + { err, projectId, userId, count: pngFilesToConvert.length }, + 'png2pdf conversion failed, using original png(s)' + ) + } + for (const f of pngFilesToConvert) { + try { + await UrlCache.promises.commitConversion( + f.conversionPath, + f.cachePath, + f.destPath + ) + } catch (err) { + logger.err( + { err, projectId, userId, path: f.destPath }, + 'error copying file for resources' + ) + Metrics.inc('download-failed') + } + } + } const baseHistoryVersion = localBaseVersion + changes.length - if (fullSync || changes.length || wasDirty || dirty.length) { + if ( + fullSync || + changes.length || + wasDirty || + dirty.length || + pngModeChanged + ) { await saveSnapshot( cacheKey, snapshot, baseHistoryVersion, globalBlobs, - dirty + dirty, + request.png2pdf ) } if (fullSync) { diff --git a/services/clsi/app/js/Png2Pdf.js b/services/clsi/app/js/Png2Pdf.js new file mode 100644 index 0000000000..a665da33e0 --- /dev/null +++ b/services/clsi/app/js/Png2Pdf.js @@ -0,0 +1,96 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import Metrics from '@overleaf/metrics' +import OError from '@overleaf/o-error' +import CommandRunner from './CommandRunner.js' + +// Convert "slow" PNGs (e.g. with transparency) to PDF ahead of the compile, so +// that pdflatex can include them via the fast native path instead of paying the +// conversion cost on every run. The png2pdf tool overwrites each file in place +// with PDF bytes (pdflatex looks at file content, not the extension); PNGs that +// are already fast-includable are left unchanged. +// +// The conversion runs in an isolated, network-disabled container built FROM +// scratch (see dockerfiles/png2pdf), with the project's cache dir bind-mounted. + +/** + * Whether png-to-pdf conversion is available in this deployment. It requires + * sandboxed (docker) compiles, the host path of the cache dir (so the sibling + * container can bind-mount it) and a configured image. + * + * @return {boolean} + */ +export function isEnabled() { + return Boolean( + Settings.enablePng2pdfConversions && + Settings.clsi?.dockerRunner === true && + Settings.path.sandboxedCompilesHostDirCache && + Settings.png2pdfImage + ) +} + +// The tool prints one "Converted to PDF" line per file it actually +// converts; already-fast PNGs are skipped silently. +const CONVERTED_LINE = /^Converted .+ to PDF$/gm + +/** + * Run the png2pdf binary over a list of files living directly inside a project's + * cache dir, converting them in place. Files are processed sequentially by the + * binary in a single container invocation. + * + * @param {string} projectId + * @param {string} cacheProjectDir absolute path to the project's cache dir + * @param {string[]} relativePaths file names within cacheProjectDir to convert + * @param {Record} stats + * @param {Record} timings + * @return {Promise} + */ +export async function convertPngFilesInCacheDir( + projectId, + cacheProjectDir, + relativePaths, + stats, + timings +) { + if (!isEnabled()) return + + const timer = new Metrics.Timer('png2pdf') + try { + const { stdout, stderr, exitCode } = await CommandRunner.promises.run( + projectId, + // `--` terminates option parsing: the cache file names start with a `-` + // (the leading `/` of the blob URL path), which clap would otherwise treat + // as a flag. + ['--in-place', '--', ...relativePaths], + cacheProjectDir, + Settings.png2pdfImage, + Settings.conversionTimeoutSeconds * 1000, + {}, + 'png2pdf', + null + ) + if (exitCode !== 0) { + throw new OError('non-zero exit code from png2pdf', { + exitCode, + stdout, + stderr, + }) + } + const converted = (stdout.match(CONVERTED_LINE) || []).length + logger.debug( + { projectId, attempted: relativePaths.length, converted, stdout, stderr }, + 'png2pdf conversion completed' + ) + Metrics.count('png2pdf-converted', converted) + stats.png2pdf = converted + timings.png2pdf = timer.done({ status: 'success' }) + } catch (err) { + timings.png2pdf = timer.done({ status: 'error' }) + throw OError.tag(err, 'png2pdf conversion failed', { + projectId, + count: relativePaths.length, + }) + } +} + +export default { isEnabled, convertPngFilesInCacheDir } diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js index c911dca76a..f702eccbe7 100644 --- a/services/clsi/app/js/RequestParser.js +++ b/services/clsi/app/js/RequestParser.js @@ -89,6 +89,10 @@ function parse(body, callback) { default: false, type: 'boolean', }) + response.png2pdf = _parseAttribute('png2pdf', compile.options.png2pdf, { + default: false, + type: 'boolean', + }) response.stopOnFirstError = _parseAttribute( 'stopOnFirstError', compile.options.stopOnFirstError, diff --git a/services/clsi/app/js/UrlCache.js b/services/clsi/app/js/UrlCache.js index 83abc95104..b41e40df72 100644 --- a/services/clsi/app/js/UrlCache.js +++ b/services/clsi/app/js/UrlCache.js @@ -48,50 +48,90 @@ async function createProjectDir(projectId) { await fs.promises.mkdir(getProjectDir(projectId), { recursive: true }) } +async function tryCopyFile(src, dest) { + try { + await fs.promises.copyFile(src, dest) + return true + } catch (err) { + if (err.code === 'ENOENT') return false + throw err + } +} + +/** + * Copy a url's file from the cache into destPath, downloading it first if it is + * not cached yet. + * + * When `conversionSuffix` is set, the optimized result is cached alongside the + * original at `.opt`. The file is downloaded to a per-user conversion + * path and NOT copied to destPath: the caller is expected to process it (e.g. + * convert a slow png to pdf) and then commit it with commitConversion (which + * renames it onto `.opt`). In that case a + * `{ conversionPath, cachePath, destPath }` handle is returned for the files that + * were freshly downloaded (a cache hit returns undefined, as the already-final + * file has been copied to destPath). + * + * @param {string} projectId + * @param {string} url + * @param {string|null} fallbackURL + * @param {string} destPath + * @param {Date} lastModified + * @param {string} [conversionSuffix] when set the download is deferred to a + * conversion path for processing + commitConversion instead of being copied + * to destPath + * @return {Promise<{ conversionPath: string, cachePath: string, destPath: string }|undefined>} + */ async function downloadUrlToFile( projectId, url, fallbackURL, destPath, - lastModified + lastModified, + conversionSuffix = '' ) { - const cachePath = getCachePath(projectId, url, lastModified) - try { + // When converting, the optimized result is cached alongside the original at + // `.opt`, so switching modes can reuse either variant. + const cachePath = + getCachePath(projectId, url, lastModified) + + (conversionSuffix ? '.opt' : '') + + { const timer = new Metrics.Timer('url_cache', { status: 'cache-hit', path: 'copy', }) - try { - await fs.promises.copyFile(cachePath, destPath) - } catch (err) { - if (err.code === 'ENOENT' && fallbackURL) { - const fallbackPath = getCachePath(projectId, fallbackURL, lastModified) - await fs.promises.copyFile(fallbackPath, destPath) - } else { - throw err - } + let copied = await tryCopyFile(cachePath, destPath) + if (!copied && fallbackURL) { + copied = await tryCopyFile( + getCachePath(projectId, fallbackURL, lastModified) + + (conversionSuffix ? '.opt' : ''), + destPath + ) } // the metric is only updated if the file is present in the cache - timer.done() - return - } catch (e) { - if (e.code !== 'ENOENT') { - throw e + if (copied) { + timer.done() + return } } - // time the download + + const conversionPath = cachePath + conversionSuffix { const timer = new Metrics.Timer('url_cache', { status: 'cache-miss', path: 'download', }) try { - await download(url, fallbackURL, cachePath) + await download(url, fallbackURL, conversionPath) } finally { timer.done() } } - // time the file copy + + if (conversionSuffix) { + return { conversionPath, cachePath, destPath } + } + { const timer = new Metrics.Timer('url_cache', { status: 'cache-miss', @@ -102,8 +142,13 @@ async function downloadUrlToFile( } } -async function download(url, fallbackURL, cachePath) { - let pending = PENDING_DOWNLOADS.get(cachePath) +/** + * Download a url to `targetPath`. UrlFetcher writes atomically (to its own + * `targetPath + '~'` then renames onto targetPath). Concurrent downloads of the + * same target are deduplicated. + */ +async function download(url, fallbackURL, targetPath) { + let pending = PENDING_DOWNLOADS.get(targetPath) if (pending) { return pending } @@ -111,23 +156,44 @@ async function download(url, fallbackURL, cachePath) { pending = UrlFetcher.promises.pipeUrlToFileWithRetry( url, fallbackURL, - cachePath + targetPath ) - PENDING_DOWNLOADS.set(cachePath, pending) + PENDING_DOWNLOADS.set(targetPath, pending) try { await pending } finally { - PENDING_DOWNLOADS.delete(cachePath) + PENDING_DOWNLOADS.delete(targetPath) } } +/** + * Commit a converted cache file: atomically rename it onto the cache path, then + * copy it to the destination. + * + * @param {string} conversionPath + * @param {string} cachePath + * @param {string} destPath + * @return {Promise} + */ +async function commitConversion(conversionPath, cachePath, destPath) { + await fs.promises.rename(conversionPath, cachePath) + const timer = new Metrics.Timer('url_cache', { + status: 'cache-miss', + path: 'copy', + }) + await fs.promises.copyFile(cachePath, destPath) + timer.done() +} + export default { clearProject: callbackify(clearProject), createProjectDir: callbackify(createProjectDir), downloadUrlToFile: callbackify(downloadUrlToFile), + getProjectCacheDir: getProjectDir, promises: { clearProject, createProjectDir, downloadUrlToFile, + commitConversion, }, } diff --git a/services/clsi/config/settings.defaults.cjs b/services/clsi/config/settings.defaults.cjs index eb576f8701..8477cd26d8 100644 --- a/services/clsi/config/settings.defaults.cjs +++ b/services/clsi/config/settings.defaults.cjs @@ -34,6 +34,9 @@ module.exports = { pdftocairoImage: process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02', enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true', + png2pdfImage: + process.env.PNG2PDF_IMAGE || 'quay.io/sharelatex/png2pdf:2026-06-24', + enablePng2pdfConversions: process.env.ENABLE_PNG2PDF_CONVERSIONS === 'true', maxUploadSize: 50 * 1024 * 1024, preciousFilePattern: process.env.PRECIOUS_FILE_PATTERN || '', @@ -168,6 +171,13 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') { synctex: { 'HostConfig.AutoRemove': true }, 'synctex-output': { 'HostConfig.AutoRemove': true }, conversions: { 'HostConfig.AutoRemove': true }, + // The png2pdf image is built FROM scratch and has no 'tex' user, so run + // it as the same uid/gid as this clsi process, which owns the cache files + // it converts in place. + png2pdf: { + 'HostConfig.AutoRemove': true, + User: `${process.getuid()}:${process.getgid()}`, + }, } module.exports.clsi.docker.compileGroupConfig = Object.assign( defaultCompileGroupConfig, @@ -235,6 +245,12 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') { ) } + // Host path of the clsi cache dir, so that the png2pdf conversion container + // (a sibling container started via the docker socket) can bind-mount a + // project's cache dir and convert downloaded PNGs in place. + module.exports.path.sandboxedCompilesHostDirCache = + process.env.SANDBOXED_COMPILES_HOST_DIR_CACHE + module.exports.path.sandboxedCompilesHostDirOutput = process.env.SANDBOXED_COMPILES_HOST_DIR_OUTPUT || process.env.OUTPUT_HOST_DIR diff --git a/services/clsi/test/unit/js/ConversionController.test.js b/services/clsi/test/unit/js/ConversionController.test.js index 4ae462a64d..72016a617e 100644 --- a/services/clsi/test/unit/js/ConversionController.test.js +++ b/services/clsi/test/unit/js/ConversionController.test.js @@ -49,10 +49,10 @@ describe('ConversionController', function () { }, } + // HistoryResourceWriter is consumed via a namespace import, so the named + // export lives at the top level (not under `.promises`). ctx.HistoryResourceWriter = { - promises: { - syncResourcesToDisk: sinon.stub().resolves(), - }, + syncResourcesToDisk: sinon.stub().resolves(), } ctx.RequestParser = { @@ -429,6 +429,33 @@ describe('ConversionController', function () { const uuidDirPattern = /^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + describe('when compiling from history', function () { + beforeEach(async function (ctx) { + ctx.parsedRequest.isCompileFromHistory = true + // Even if the request asked for png2pdf, conversions must disable it. + ctx.parsedRequest.png2pdf = true + + await ctx.ConversionController.convertProjectToDocument( + ctx.req, + ctx.res, + sinon.stub() + ) + }) + + it('should sync resources via the history writer with png2pdf disabled', function (ctx) { + sinon.assert.calledWith( + ctx.HistoryResourceWriter.syncResourcesToDisk, + 'test-project-id', + 'test-user-id', + sinon.match({ png2pdf: false }), + sinon.match(uuidDirPattern), + sinon.match.object, // timings + sinon.match.object // stats + ) + sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk) + }) + }) + describe('successfully (default streaming response)', function () { beforeEach(async function (ctx) { await ctx.ConversionController.convertProjectToDocument( diff --git a/services/clsi/test/unit/js/Png2Pdf.test.js b/services/clsi/test/unit/js/Png2Pdf.test.js new file mode 100644 index 0000000000..d9d77c763e --- /dev/null +++ b/services/clsi/test/unit/js/Png2Pdf.test.js @@ -0,0 +1,156 @@ +import Path from 'node:path' +import sinon from 'sinon' +import { vi, describe, beforeEach, it, expect } from 'vitest' + +const MODULE_PATH = Path.join(import.meta.dirname, '../../../app/js/Png2Pdf') + +describe('Png2Pdf', function () { + beforeEach(async function (ctx) { + ctx.CommandRunner = { + promises: { + run: sinon.stub().resolves({ stdout: '', stderr: '', exitCode: 0 }), + }, + } + + ctx.Settings = { + png2pdfImage: 'mock-png2pdf-image', + enablePng2pdfConversions: true, + conversionTimeoutSeconds: 60, + clsi: { dockerRunner: true }, + path: { sandboxedCompilesHostDirCache: '/host/cache' }, + } + + vi.doMock('../../../app/js/CommandRunner', () => ({ + default: ctx.CommandRunner, + })) + + vi.doMock('@overleaf/settings', () => ({ default: ctx.Settings })) + + vi.doMock('@overleaf/logger', () => ({ + default: { debug: sinon.stub(), warn: sinon.stub(), err: sinon.stub() }, + })) + + ctx.Metrics = { + Timer: sinon.stub().returns({ done: sinon.stub().returns(42) }), + count: sinon.stub(), + } + vi.doMock('@overleaf/metrics', () => ({ default: ctx.Metrics })) + + ctx.Png2Pdf = (await import(MODULE_PATH)).default + }) + + describe('isEnabled', function () { + it('is true when enabled, sandboxed, a cache host dir and an image are set', function (ctx) { + expect(ctx.Png2Pdf.isEnabled()).to.equal(true) + }) + + it('is false when conversions are disabled', function (ctx) { + ctx.Settings.enablePng2pdfConversions = false + expect(ctx.Png2Pdf.isEnabled()).to.equal(false) + }) + + it('is false without the docker runner', function (ctx) { + ctx.Settings.clsi.dockerRunner = false + expect(ctx.Png2Pdf.isEnabled()).to.equal(false) + }) + + it('is false without a cache host dir', function (ctx) { + ctx.Settings.path.sandboxedCompilesHostDirCache = undefined + expect(ctx.Png2Pdf.isEnabled()).to.equal(false) + }) + + it('is false without an image', function (ctx) { + ctx.Settings.png2pdfImage = undefined + expect(ctx.Png2Pdf.isEnabled()).to.equal(false) + }) + }) + + describe('convertPngFilesInCacheDir', function () { + it('runs png2pdf over the files in the cache dir', async function (ctx) { + ctx.CommandRunner.promises.run.resolves({ + stdout: 'Converted a.tmp to PDF\nConverted b.tmp to PDF\n', + stderr: '', + exitCode: 0, + }) + const stats = {} + const timings = {} + await ctx.Png2Pdf.convertPngFilesInCacheDir( + 'project-1', + '/cache/dir', + ['a.tmp', 'b.tmp'], + stats, + timings + ) + sinon.assert.calledWith( + ctx.CommandRunner.promises.run, + 'project-1', + ['--in-place', '--', 'a.tmp', 'b.tmp'], + '/cache/dir', + 'mock-png2pdf-image', + 60 * 1000, + {}, + 'png2pdf', + null + ) + expect(stats.png2pdf).to.equal(2) + expect(timings.png2pdf).to.equal(42) + }) + + it('records only the files actually converted (skipped pngs print nothing)', async function (ctx) { + // The tool converted one of the two attempted files; the other was + // already fast-includable and skipped silently. + ctx.CommandRunner.promises.run.resolves({ + stdout: 'Converted a.tmp to PDF\n', + stderr: '', + exitCode: 0, + }) + const stats = {} + await ctx.Png2Pdf.convertPngFilesInCacheDir( + 'project-1', + '/cache/dir', + ['a.tmp', 'b.tmp'], + stats, + {} + ) + expect(stats.png2pdf).to.equal(1) + sinon.assert.calledWith(ctx.Metrics.count, 'png2pdf-converted', 1) + }) + + it('does nothing when disabled', async function (ctx) { + ctx.Settings.clsi.dockerRunner = false + const stats = {} + const timings = {} + await ctx.Png2Pdf.convertPngFilesInCacheDir( + 'project-1', + '/cache/dir', + ['a.tmp'], + stats, + timings + ) + sinon.assert.notCalled(ctx.CommandRunner.promises.run) + expect(stats.png2pdf).to.be.undefined + expect(timings.png2pdf).to.be.undefined + }) + + it('records the timing but not stats on a non-zero exit code', async function (ctx) { + ctx.CommandRunner.promises.run.resolves({ + stdout: '', + stderr: 'boom', + exitCode: 1, + }) + const stats = {} + const timings = {} + await expect( + ctx.Png2Pdf.convertPngFilesInCacheDir( + 'project-1', + '/cache/dir', + ['a.tmp'], + stats, + timings + ) + ).to.be.rejectedWith('non-zero exit code from png2pdf') + expect(stats.png2pdf).to.be.undefined + expect(timings.png2pdf).to.equal(42) + }) + }) +}) diff --git a/services/clsi/test/unit/js/UrlCache.test.js b/services/clsi/test/unit/js/UrlCache.test.js index 8cff7337ab..a9ee7ee0cb 100644 --- a/services/clsi/test/unit/js/UrlCache.test.js +++ b/services/clsi/test/unit/js/UrlCache.test.js @@ -31,10 +31,19 @@ describe('UrlCache', () => { }, })) + vi.doMock('../../../app/js/Png2Pdf', () => ({ + default: (ctx.Png2Pdf = { + isEnabled: sinon.stub().returns(false), + convertPngFilesInCacheDir: sinon.stub().resolves(), + }), + })) + ctx.fs = { promises: { rm: sinon.stub().resolves(), copyFile: sinon.stub().resolves(), + rename: sinon.stub().resolves(), + access: sinon.stub().resolves(), }, } @@ -136,6 +145,80 @@ describe('UrlCache', () => { }) }) + describe('downloadUrlToFile with a conversion suffix', () => { + beforeEach(ctx => { + ctx.destPath = 'path/to/destination' + ctx.conversionSuffix = '-user-cache-key' + }) + + it('should serve the optimized .opt variant from the cache', async ctx => { + const result = await ctx.UrlCache.promises.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + ctx.conversionSuffix + ) + // a cache hit copies the already-final file to destPath and returns nothing + expect(result).to.be.undefined + expect(ctx.fs.promises.copyFile.firstCall.args[0]).to.match(/\.opt$/) + expect(ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.called).to.equal( + false + ) + }) + + it('should return a conversion handle pointing at the .opt path on a cache miss', async ctx => { + const codedError = new Error() + codedError.code = 'ENOENT' + // miss on both the primary and the fallback .opt cache paths + ctx.fs.promises.copyFile.onCall(0).rejects(codedError) + ctx.fs.promises.copyFile.onCall(1).rejects(codedError) + + const result = await ctx.UrlCache.promises.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + ctx.conversionSuffix + ) + + expect(result.destPath).to.equal(ctx.destPath) + expect(result.cachePath).to.match(/\.opt$/) + expect(result.conversionPath).to.equal( + result.cachePath + ctx.conversionSuffix + ) + // the original is downloaded to the per-user conversion path, not destPath + expect( + ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.calledWith( + ctx.url, + ctx.fallbackURL, + result.conversionPath + ) + ).to.equal(true) + }) + }) + + describe('commitConversion', () => { + it('should rename the conversion file onto the cache path and copy it to dest', async ctx => { + const conversionPath = '/cache/dir/project/key.opt-user-cache-key' + const cachePath = '/cache/dir/project/key.opt' + const destPath = 'path/to/destination' + await ctx.UrlCache.promises.commitConversion( + conversionPath, + cachePath, + destPath + ) + expect( + ctx.fs.promises.rename.calledWith(conversionPath, cachePath) + ).to.equal(true) + expect(ctx.fs.promises.copyFile.calledWith(cachePath, destPath)).to.equal( + true + ) + }) + }) + describe('clearProject', () => { beforeEach(async ctx => { await ctx.UrlCache.promises.clearProject(ctx.project_id) diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index a659796fce..23eaf6230d 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1142,6 +1142,7 @@ function _finaliseRequest(projectId, options, project, docs, files) { timeout: options.timeout, imageName: project.imageName, draft: Boolean(options.draft), + png2pdf: Boolean(options.png2pdf), stopOnFirstError: Boolean(options.stopOnFirstError), check: options.check, syncType: options.syncType, diff --git a/services/web/app/src/Features/Compile/CompileController.mjs b/services/web/app/src/Features/Compile/CompileController.mjs index eb27104dbb..5fd42bca0d 100644 --- a/services/web/app/src/Features/Compile/CompileController.mjs +++ b/services/web/app/src/Features/Compile/CompileController.mjs @@ -202,6 +202,9 @@ const _CompileController = { if (req.body.draft) { options.draft = req.body.draft } + if (req.body.png2pdf) { + options.png2pdf = req.body.png2pdf + } if (['validate', 'error', 'silent'].includes(req.body.check)) { options.check = req.body.check } @@ -288,6 +291,7 @@ const _CompileController = { restoredClsiCache: stats?.restoredClsiCache === 1, stopOnFirstError, isDraftMode: !!options.draft, + isPng2pdf: !!options.png2pdf, } ) } diff --git a/services/web/app/src/Features/Project/ProjectController.mjs b/services/web/app/src/Features/Project/ProjectController.mjs index 4e6fd66b45..0152910bc2 100644 --- a/services/web/app/src/Features/Project/ProjectController.mjs +++ b/services/web/app/src/Features/Project/ProjectController.mjs @@ -449,6 +449,7 @@ const _ProjectController = { } const splitTests = [ + 'png2pdf', 'plugin-dimensions', 'bibtex-visual-editor', 'compile-log-events', diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index a7a4889338..f8fd27e392 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -664,6 +664,7 @@ "fair_usage_policy_applies": "", "fast": "", "fast_draft": "", + "fast_optimize_images": "", "feature_enabled_or_disabled": "", "features_like_track_changes": "", "feedback": "", diff --git a/services/web/frontend/js/features/ide-settings/components/compiler-settings/draft-setting.tsx b/services/web/frontend/js/features/ide-settings/components/compiler-settings/draft-setting.tsx index 30d398166e..658dbd7bdd 100644 --- a/services/web/frontend/js/features/ide-settings/components/compiler-settings/draft-setting.tsx +++ b/services/web/frontend/js/features/ide-settings/components/compiler-settings/draft-setting.tsx @@ -1,26 +1,46 @@ import { useTranslation } from 'react-i18next' import { useDetachCompileContext as useCompileContext } from '@/shared/context/detach-compile-context' -import { useMemo } from 'react' +import { useCallback, useMemo } from 'react' import DropdownSetting from '../dropdown-setting' -import { useSetCompilationSettingWithEvent } from '@/features/ide-settings/hooks/use-set-compilation-setting' +import { useEditorAnalytics } from '@/shared/hooks/use-editor-analytics' +import { useFeatureFlag } from '@/shared/context/split-test-context' + +type CompileMode = 'normal' | 'png2pdf' | 'fast_draft' export default function DraftSetting() { - const { draft, setDraft } = useCompileContext() + const { draft, setDraft, png2pdf, setPng2pdf } = useCompileContext() const { t } = useTranslation() - const changeDraft = useSetCompilationSettingWithEvent( - 'compile-mode', - setDraft + const { sendEvent } = useEditorAnalytics() + const png2pdfEnabled = useFeatureFlag('png2pdf') + + // The three modes are mutually exclusive, derived from the two underlying flags. + const mode: CompileMode = png2pdf + ? 'png2pdf' + : draft + ? 'fast_draft' + : 'normal' + + const changeMode = useCallback( + (value: CompileMode) => { + sendEvent('recompile-setting-changed', { + setting: 'compile-mode', + settingVal: value, + }) + setDraft(value === 'fast_draft') + setPng2pdf(value === 'png2pdf') + }, + [sendEvent, setDraft, setPng2pdf] ) const options = useMemo( () => [ - { label: t('normal'), value: false }, - { - label: t('fast_draft'), - value: true, - }, + { label: t('normal'), value: 'normal' as const }, + ...(png2pdfEnabled + ? [{ label: t('fast_optimize_images'), value: 'png2pdf' as const }] + : []), + { label: t('fast_draft'), value: 'fast_draft' as const }, ], - [t] + [t, png2pdfEnabled] ) return ( @@ -29,8 +49,9 @@ export default function DraftSetting() { label={t('compile_mode')} options={options} description={t('switch_compile_mode_for_faster_draft_compilation')} - value={draft} - onChange={changeDraft} + value={mode} + onChange={changeMode} + width={png2pdfEnabled ? 'wide' : 'default'} /> ) } diff --git a/services/web/frontend/js/features/pdf-preview/components/pdf-compile-button.tsx b/services/web/frontend/js/features/pdf-preview/components/pdf-compile-button.tsx index 564715cd46..db2f9ca68f 100644 --- a/services/web/frontend/js/features/pdf-preview/components/pdf-compile-button.tsx +++ b/services/web/frontend/js/features/pdf-preview/components/pdf-compile-button.tsx @@ -18,6 +18,7 @@ import OLButton from '@/shared/components/ol/ol-button' import OLButtonGroup from '@/shared/components/ol/ol-button-group' import { useLayoutContext } from '@/shared/context/layout-context' import { useCommandProvider } from '@/features/ide-react/hooks/use-command-provider' +import { useFeatureFlag } from '@/shared/context/split-test-context' const modifierKey = /Mac/i.test(navigator.platform) ? 'Cmd' : 'Ctrl' @@ -39,9 +40,11 @@ function PdfCompileButton() { autoCompile, compiling, draft, + png2pdf, hasChanges, setAutoCompile, setDraft, + setPng2pdf, setStopOnValidationError, stopOnFirstError, stopOnValidationError, @@ -54,6 +57,22 @@ function PdfCompileButton() { const { t } = useTranslation() + const png2pdfEnabled = useFeatureFlag('png2pdf') + + // The three compile modes (Normal / Fast [optimize images] / Fast [draft]) are + // mutually exclusive, so each selection sets both underlying flags. + const setCompileMode = useCallback( + (mode: 'normal' | 'png2pdf' | 'draft') => { + eventTracking.sendMB('recompile-setting-changed', { + setting: 'compile-mode', + settingVal: mode, + }) + setDraft(mode === 'draft') + setPng2pdf(mode === 'png2pdf') + }, + [setDraft, setPng2pdf] + ) + const { detachRole } = useLayoutContext() const fromScratchWithEvent = useCallback(() => { @@ -174,16 +193,28 @@ function PdfCompileButton() {
  • sendEventAndSet(false, setDraft, 'compile-mode')} - trailingIcon={!draft ? 'check' : null} + onClick={() => setCompileMode('normal')} + trailingIcon={!draft && !png2pdf ? 'check' : null} > {t('normal')}
    • + {png2pdfEnabled && ( +
  • + setCompileMode('png2pdf')} + trailingIcon={png2pdf ? 'check' : null} + > + {t('fast')}  + [optimize images] + +
    • + )}
  • sendEventAndSet(true, setDraft, 'compile-mode')} + onClick={() => setCompileMode('draft')} trailingIcon={draft ? 'check' : null} > {t('fast')} [draft] diff --git a/services/web/frontend/js/features/pdf-preview/util/compiler.ts b/services/web/frontend/js/features/pdf-preview/util/compiler.ts index 769486e634..f992526484 100644 --- a/services/web/frontend/js/features/pdf-preview/util/compiler.ts +++ b/services/web/frontend/js/features/pdf-preview/util/compiler.ts @@ -87,6 +87,7 @@ export default class DocumentCompiler { this.timer = 0 this.defaultOptions = { draft: false, + png2pdf: false, stopOnFirstError: false, } @@ -139,6 +140,7 @@ export default class DocumentCompiler { rootDoc_id: rootDocId, rootResourcePath, draft: options.draft, + png2pdf: options.png2pdf, check: 'silent', // NOTE: 'error' and 'validate' are possible, but unused // use incremental compile for all users but revert to a full compile // if there was previously a server error diff --git a/services/web/frontend/js/shared/context/detach-compile-context.tsx b/services/web/frontend/js/shared/context/detach-compile-context.tsx index 7d815e1ff6..a697a36683 100644 --- a/services/web/frontend/js/shared/context/detach-compile-context.tsx +++ b/services/web/frontend/js/shared/context/detach-compile-context.tsx @@ -28,6 +28,7 @@ export const DetachCompileProvider: FC = ({ compiling: _compiling, deliveryLatencies: _deliveryLatencies, draft: _draft, + png2pdf: _png2pdf, editedSinceCompileStarted: _editedSinceCompileStarted, error: _error, fileList: _fileList, @@ -45,6 +46,7 @@ export const DetachCompileProvider: FC = ({ setAnimateCompileDropdownArrow: _setAnimateCompileDropdownArrow, setAutoCompile: _setAutoCompile, setDraft: _setDraft, + setPng2pdf: _setPng2pdf, setError: _setError, setHasLintingError: _setHasLintingError, setHighlights: _setHighlights, @@ -119,6 +121,12 @@ export const DetachCompileProvider: FC = ({ 'detached' ) const [draft] = useDetachStateWatcher('draft', _draft, 'detacher', 'detached') + const [png2pdf] = useDetachStateWatcher( + 'png2pdf', + _png2pdf, + 'detacher', + 'detached' + ) const [error] = useDetachStateWatcher('error', _error, 'detacher', 'detached') const [fileList] = useDetachStateWatcher( 'fileList', @@ -259,6 +267,12 @@ export const DetachCompileProvider: FC = ({ 'detached', 'detacher' ) + const setPng2pdf = useDetachAction( + 'setPng2pdf', + _setPng2pdf, + 'detached', + 'detacher' + ) const setError = useDetachAction( 'setError', _setError, @@ -410,6 +424,7 @@ export const DetachCompileProvider: FC = ({ compiling, deliveryLatencies, draft, + png2pdf, editedSinceCompileStarted, error, fileList, @@ -431,6 +446,7 @@ export const DetachCompileProvider: FC = ({ setAutoCompile, setCompiling, setDraft, + setPng2pdf, setError, setHasLintingError, setHighlights, @@ -468,6 +484,7 @@ export const DetachCompileProvider: FC = ({ compiling, deliveryLatencies, draft, + png2pdf, editedSinceCompileStarted, error, fileList, @@ -487,6 +504,7 @@ export const DetachCompileProvider: FC = ({ setAutoCompile, setCompiling, setDraft, + setPng2pdf, setError, setHasLintingError, setHighlights, diff --git a/services/web/frontend/js/shared/context/local-compile-context.tsx b/services/web/frontend/js/shared/context/local-compile-context.tsx index 66d82ebd42..a9cb5ffd70 100644 --- a/services/web/frontend/js/shared/context/local-compile-context.tsx +++ b/services/web/frontend/js/shared/context/local-compile-context.tsx @@ -71,6 +71,7 @@ export type CompileContext = { compiling: boolean deliveryLatencies: Record draft: boolean + png2pdf: boolean error?: string fileList?: PdfFileDataList hasChanges: boolean @@ -93,6 +94,7 @@ export type CompileContext = { rawLog?: string setAutoCompile: (value: boolean) => void setDraft: (value: any) => void + setPng2pdf: (value: any) => void setError: (value: any) => void setHasLintingError: (value: boolean) => void // only for storybook setHighlights: (value: any) => void @@ -271,6 +273,15 @@ export const LocalCompileProvider: FC = ({ listen: true, }) + // whether the compile should optimize images (png2pdf conversion) + const [png2pdf, setPng2pdf] = usePersistedState( + `png2pdf:${projectId}`, + false, + { + listen: true, + } + ) + // whether compiling should stop on first error const [stopOnFirstError, setStopOnFirstError] = usePersistedState( `stop_on_first_error:${projectId}`, @@ -348,6 +359,11 @@ export const LocalCompileProvider: FC = ({ compiler.setOption('draft', draft) }, [compiler, draft]) + // keep png2pdf (optimize images) setting in sync with the compiler + useEffect(() => { + compiler.setOption('png2pdf', png2pdf) + }, [compiler, png2pdf]) + // keep stop on first error setting in sync with the compiler useEffect(() => { compiler.setOption('stopOnFirstError', stopOnFirstError) @@ -396,6 +412,7 @@ export const LocalCompileProvider: FC = ({ settingsUpToDate = getRootDocInfo().rootDocId === dataFromCache.rootDocId && dataFromCache.options.draft === draft && + Boolean(dataFromCache.options.png2pdf) === png2pdf && // Allow stopOnFirstError to be enabled in the compile from cache and disabled locally. // Compiles that passed with stopOnFirstError=true will also pass with stopOnFirstError=false. The inverse does not hold, and we need to recompile. !!dataFromCache.options.stopOnFirstError >= stopOnFirstError @@ -427,6 +444,7 @@ export const LocalCompileProvider: FC = ({ imageName, stopOnFirstError, draft, + png2pdf, ]) // always compile the PDF once after opening the project, after the doc has loaded @@ -755,6 +773,7 @@ export const LocalCompileProvider: FC = ({ compiling, deliveryLatencies, draft, + png2pdf, editedSinceCompileStarted, error, fileList, @@ -776,6 +795,7 @@ export const LocalCompileProvider: FC = ({ setAutoCompile, setCompiling, setDraft, + setPng2pdf, setError, setHasLintingError, // only for stories setHighlights, @@ -813,6 +833,7 @@ export const LocalCompileProvider: FC = ({ compiling, deliveryLatencies, draft, + png2pdf, editedSinceCompileStarted, error, fileList, @@ -831,6 +852,7 @@ export const LocalCompileProvider: FC = ({ setAnimateCompileDropdownArrow, setAutoCompile, setDraft, + setPng2pdf, setError, setHasLintingError, // only for stories setHighlights, diff --git a/services/web/locales/en.json b/services/web/locales/en.json index 51b5f5a00c..666ba47a59 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -862,6 +862,7 @@ "faq_how_does_free_trial_works_answer": "You get full access to your chosen __appName__ plan during your __len__-day free trial. There is no obligation to continue beyond the trial. Your card will be charged at the end of your __len__ day trial unless you cancel before then. You can cancel via your subscription settings.", "fast": "Fast", "fast_draft": "Fast [draft]", + "fast_optimize_images": "Fast [optimize images]", "fastest": "Fastest", "feature_enabled_or_disabled": "Feature enabled/disabled", "feature_included": "Feature included", diff --git a/services/web/test/frontend/features/ide-settings/settings/draft-setting.test.tsx b/services/web/test/frontend/features/ide-settings/settings/draft-setting.test.tsx index 3a02f32152..2cf52fe8f1 100644 --- a/services/web/test/frontend/features/ide-settings/settings/draft-setting.test.tsx +++ b/services/web/test/frontend/features/ide-settings/settings/draft-setting.test.tsx @@ -8,18 +8,12 @@ import { import userEvent from '@testing-library/user-event' import DraftSetting from '@/features/ide-settings/components/compiler-settings/draft-setting' -const OPTIONS = [ - { - label: 'Normal', - value: false, - }, - { - label: 'Fast [draft]', - value: true, - }, -] - describe('', function () { + afterEach(function () { + window.metaAttributesCache.delete('ol-splitTestVariants') + localStorage.clear() + }) + it('each option is shown and can be selected', async function () { render( @@ -31,15 +25,45 @@ describe('', function () { const select = screen.getByLabelText('Compile mode') - for (const option of OPTIONS) { + const options = [ + { label: 'Normal', value: 'normal', draft: false }, + { label: 'Fast [draft]', value: 'fast_draft', draft: true }, + ] + for (const option of options) { const optionElement = within(select).getByText(option.label) - expect(optionElement.getAttribute('value')).to.equal( - option.value.toString() - ) + expect(optionElement.getAttribute('value')).to.equal(option.value) await userEvent.selectOptions(select, [optionElement]) expect(!!localStorage.getItem(`draft:${projectDefaults._id}`)).to.equal( - option.value + option.draft + ) + expect(!!localStorage.getItem(`png2pdf:${projectDefaults._id}`)).to.equal( + false ) } }) + + it('offers the optimize-images option behind the png2pdf split test', async function () { + window.metaAttributesCache.set('ol-splitTestVariants', { + png2pdf: 'enabled', + }) + render( + + + + + + ) + + const select = screen.getByLabelText('Compile mode') + const optionElement = within(select).getByText('Fast [optimize images]') + expect(optionElement.getAttribute('value')).to.equal('png2pdf') + + await userEvent.selectOptions(select, [optionElement]) + expect(!!localStorage.getItem(`png2pdf:${projectDefaults._id}`)).to.equal( + true + ) + expect(!!localStorage.getItem(`draft:${projectDefaults._id}`)).to.equal( + false + ) + }) }) diff --git a/services/web/test/frontend/helpers/editor-providers.tsx b/services/web/test/frontend/helpers/editor-providers.tsx index 3607a38c75..4ecfc22588 100644 --- a/services/web/test/frontend/helpers/editor-providers.tsx +++ b/services/web/test/frontend/helpers/editor-providers.tsx @@ -728,10 +728,12 @@ const BASE_COMPILE_CONTEXT_MOCK = { logEntries: undefined, outputFilesArchive: undefined, pdfViewer: 'pdfjs', + png2pdf: false, position: undefined, rawLog: undefined, recompileFromScratch: () => {}, setAnimateCompileDropdownArrow: () => {}, + setPng2pdf: () => {}, setHasLintingError: () => {}, setHighlights: () => {}, setPosition: () => {}, diff --git a/services/web/test/unit/src/Compile/ClsiManager.test.mjs b/services/web/test/unit/src/Compile/ClsiManager.test.mjs index edc9c1ea84..77017de9f4 100644 --- a/services/web/test/unit/src/Compile/ClsiManager.test.mjs +++ b/services/web/test/unit/src/Compile/ClsiManager.test.mjs @@ -1031,6 +1031,29 @@ describe('ClsiManager', function () { }) }) + describe('with the png2pdf option', function () { + beforeEach(async function (ctx) { + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + png2pdf: true, + } + ) + }) + + it('should add the png2pdf option into the request', function (ctx) { + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { compile: { options: { png2pdf: true } } }, + }) + ) + }) + }) + describe('with a failed compile', function () { beforeEach(async function (ctx) { ctx.responseBody.compile.status = 'failure' diff --git a/services/web/types/compile.ts b/services/web/types/compile.ts index c2d24cd699..bc4585c12c 100644 --- a/services/web/types/compile.ts +++ b/services/web/types/compile.ts @@ -80,6 +80,7 @@ export type CompileResponseData = { export type CompileOptions = { draft?: boolean + png2pdf?: boolean stopOnFirstError?: boolean isAutoCompileOnLoad?: boolean isAutoCompileOnChange?: boolean