[clsi] integrate png2pdf (#34796)

* Add png to pdf converter for optimising image loading in pdftex

* refactor

* move fixtures

* refactor and add unit tests

* Remove gamma correction from PNG processing and update related tests

* Add crc32fast dependency and enhance PNG processing tests

* Add fixture test cases for PNG processing

* fix alpha channel to match pdftex

pdftex always uses an 8 bit alpha channel in the output even for 16 bit images

* Refactor PNG processing to use Option for alpha stream and update related logic

* Enhance PDF writing by adding DecodeParms for alpha stream and optimize PNG encoding process

* Add pixel dimensions support in PNG processing and PDF writing

* [copybara] move clsi excludes

* [clsi] move gitignore rules

* [png2pdf] move into dockerfiles folder

* [png2pdf] use atomic writes

* [clsi] integrate png2pdf

* [saas-e2e] add coverage for png2pdf

* [png2pdf] make compatible with seccomp profile in clsi

* [copybara] remove exclude following move of png2pdf

* [copybara] remove spurious excludes

* fix broken detection of malformed chunk headers

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* [project-history] add build script flag for depending on doc-updater types

* [png2pdf] run tests as part of clsi pipeline

* [png2pdf] fix calculation of PDF page size

* [web] put png2pdf behind feature flag and make it opt-in

* [web] fix ClsiManager.promises.sendRequest invocation after merge

* extend is_copy_safe function to follow pdftex

see the conditions in pdftex's writepng.c
https://svn.tug.org:8369/texlive/trunk/Build/source/texk/web2c/pdftexdir/writepng.c?revision=78081&view=markup#l575

the criteria are not an exact match because the png crate does not support all the
png chunk types
https://github.com/image-rs/image-png/issues/116

if the criteria result in a png being converted or skipped the final result should be the same.

in one case the conversion will be done by pdftex (at runtime)
in the other case the conversion will be done by png2pdf -

both conversions should give the same result.

* include tests in png2pdf Dockerfile

* [clsi] explicitly disable png2pdf conversion for project conversion

* [clsi] png2pdf: count converted files

* [png2pdf] bump date in docker image tag

* [web] make draft-setting option wider when png2pdf is available

---------

Co-authored-by: Brian Gough <brian.gough@overleaf.com>
GitOrigin-RevId: d0d952ee009bf93c72bba2ffe763ae0160d396e4
This commit is contained in:
Jakob Ackermann
2026-09-17 11:17:28 +00:00
committed by Copybot
co-authored by Brian Gough
parent 2b05a3c9df
commit 69ccb3c316
24 changed files with 793 additions and 87 deletions
+2 -1
View File
@@ -114,7 +114,8 @@ async function doCompile(request, stats, timings) {
userId,
request,
compileDir,
timings
timings,
stats
))
} else {
// NOTE: resourceList is insecure, it should only be used to exclude files from the output list
+5 -1
View File
@@ -140,6 +140,9 @@ async function convertProjectToDocument(req, res) {
request.project_id = projectId
request.user_id = userId
request.metricsOpts = {}
// Document conversions reuse the history writer but must never run png2pdf:
// the converted PDF bytes would be fed to pandoc instead of the original png.
request.png2pdf = false
const responseFormat = req.query.responseFormat === 'json' ? 'json' : 'stream'
@@ -176,7 +179,8 @@ async function convertProjectToDocument(req, res) {
userId,
request,
conversionDir,
{}
{}, // timings
{} // stats
)
} catch (err) {
if (err instanceof Errors.MissingUpdatesError) {
+125 -24
View File
@@ -25,10 +25,19 @@ import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
import Metrics from '@overleaf/metrics'
import TikzManager from './TikzManager.js'
import DraftModeManager from './DraftModeManager.js'
import Png2Pdf from './Png2Pdf.js'
const gzip = promisify(zlib.gzip)
const gunzip = promisify(zlib.gunzip)
/**
* @param {string} path
* @return {boolean}
*/
function isPng(path) {
return Path.extname(path).toLowerCase() === '.png'
}
export const clearCacheCb = callbackify(clearCache)
/**
@@ -76,7 +85,7 @@ function isENOENT(err) {
* @param {string} cacheKey
* @param {number} remoteBaseVersion
* @param {boolean} populateClsiCache
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>}
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[], png2pdf: boolean}>}
*/
async function loadSnapshot(
projectId,
@@ -137,7 +146,7 @@ async function loadSnapshot(
* @param {string} userId
* @param {string} cacheKey
* @param {number} remoteBaseVersion
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>}
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[], png2pdf: boolean}>}
*/
async function loadSnapshotFromClsiCache(
projectId,
@@ -168,7 +177,7 @@ async function loadSnapshotFromClsiCache(
* @param {string} path
* @param {number} remoteBaseVersion
* @param {boolean} fullSync
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean, dirty: string[]}>}
* @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean, dirty: string[], png2pdf: boolean}>}
*/
async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) {
let blob = await fs.promises.readFile(path)
@@ -178,13 +187,21 @@ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) {
globalBlobs,
localBaseVersion,
dirty = [], // added later, provide a default value.
png2pdf = false, // the png2pdf mode used for the last sync, added later.
} = JSON.parse(blob.toString('utf-8'))
if (localBaseVersion < remoteBaseVersion) {
throw new Errors.MissingUpdatesError('missing updates', {
baseHistoryVersion: localBaseVersion,
})
}
return { rawSnapshot, globalBlobs, localBaseVersion, fullSync, dirty }
return {
rawSnapshot,
globalBlobs,
localBaseVersion,
fullSync,
dirty,
png2pdf,
}
}
/**
@@ -193,6 +210,7 @@ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) {
* @param {number} localBaseVersion
* @param {string[]} globalBlobs
* @param {string[]} dirty
* @param {boolean} png2pdf the png2pdf mode used for this sync
* @return {Promise<void>}
*/
async function saveSnapshot(
@@ -200,7 +218,8 @@ async function saveSnapshot(
snapshot,
localBaseVersion,
globalBlobs,
dirty
dirty,
png2pdf
) {
const { dir, path } = snapshotPath(cacheKey)
await fs.promises.mkdir(dir, { recursive: true })
@@ -213,6 +232,7 @@ async function saveSnapshot(
localBaseVersion,
rawSnapshot: snapshot.toRaw(),
dirty,
png2pdf,
}),
// use cheapest gzip compression level
{ level: 1 }
@@ -363,6 +383,7 @@ function changesFromRawChangeOperations(raw) {
* @param {Object} request
* @param {string} compileDir
* @param {Record<string, number>} timings
* @param {Record<string, number>} stats
* @return {Promise<{baseHistoryVersion: number, resourceList: {path: string}[]}>}
*/
export async function syncResourcesToDisk(
@@ -370,23 +391,36 @@ export async function syncResourcesToDisk(
userId,
request,
compileDir,
timings
timings,
stats
) {
// - logged in user: <project-id>-<user-id>
// - anonymous user: <project-id>
// - conversion job: <uuid>
const cacheKey = Path.basename(compileDir)
const remoteBaseVersion = request.baseHistoryVersion
let rawSnapshot, globalBlobs, localBaseVersion, source, dirty, fullSync
let rawSnapshot,
globalBlobs,
localBaseVersion,
source,
dirty,
fullSync,
lastPng2pdf
try {
;({ rawSnapshot, globalBlobs, fullSync, localBaseVersion, dirty } =
await loadSnapshot(
projectId,
userId,
cacheKey,
remoteBaseVersion,
request.populateClsiCache
))
;({
rawSnapshot,
globalBlobs,
fullSync,
localBaseVersion,
dirty,
png2pdf: lastPng2pdf,
} = await loadSnapshot(
projectId,
userId,
cacheKey,
remoteBaseVersion,
request.populateClsiCache
))
source = fullSync ? 'clsi-cache' : 'local'
logger.debug(
{ projectId, userId, cacheKey, localBaseVersion, remoteBaseVersion },
@@ -431,6 +465,8 @@ export async function syncResourcesToDisk(
const entriesDepthFirst = await discoverExistingEntries(compileDir)
await removeExtraneousEntries(compileDir, snapshot, entriesDepthFirst)
const pngModeChanged = lastPng2pdf !== request.png2pdf
const changedPaths = []
if (fullSync) {
changedPaths.push(...snapshot.getFilePathnames())
@@ -443,6 +479,12 @@ export async function syncResourcesToDisk(
if (request.draft) {
dedupe.add(request.rootResourcePath)
}
if (pngModeChanged) {
// When the png2pdf mode changed since the last sync, the on-disk images are in the wrong variant (optimized vs original). Re-sync them so they are converted (served from the <cachePath>.opt cache) or restored to the original png.
for (const path of snapshot.getFilePathnames()) {
if (isPng(path)) dedupe.add(path)
}
}
for (const change of changes) {
for (const operation of change.getOperations()) {
if (operation instanceof AddFileOperation) {
@@ -490,6 +532,8 @@ export async function syncResourcesToDisk(
const wasDirty = dirty.length > 0
dirty = []
let createCacheFolder
const pngFilesToConvert = []
// Use Promise.allSettled to ensure that all writes have stopped when we exit.
const allDone = await promiseMapSettledWithLimit(
Settings.parallelFileDownloads,
@@ -526,16 +570,33 @@ export async function syncResourcesToDisk(
}
await createCacheFolder
const url = blobStore.getBlobURL(hash).href
const destPath = Path.join(compileDir, path)
try {
const fallbackURL = null // no fallback
const lastModified = new Date(0) // content is static
await UrlCache.promises.downloadUrlToFile(
projectId,
url,
fallbackURL,
Path.join(compileDir, path),
lastModified
)
if (request.png2pdf && Png2Pdf.isEnabled() && isPng(path)) {
// PNG files go through a batch conversion process first.
const toConvert = await UrlCache.promises.downloadUrlToFile(
projectId,
url,
fallbackURL,
destPath,
lastModified,
// Avoid sharing the conversion file between two users.
cacheKey
)
if (toConvert) {
pngFilesToConvert.push(toConvert)
}
} else {
await UrlCache.promises.downloadUrlToFile(
projectId,
url,
fallbackURL,
destPath,
lastModified
)
}
} catch (err) {
logger.err(
{ err, projectId, path, resourceUrl: url },
@@ -551,14 +612,54 @@ export async function syncResourcesToDisk(
const path = changedPaths[idx]
throw OError.tag(result.reason, 'write failed', { path })
}
if (pngFilesToConvert.length) {
const cacheDir = UrlCache.getProjectCacheDir(projectId)
try {
await Png2Pdf.convertPngFilesInCacheDir(
projectId,
cacheDir,
pngFilesToConvert.map(f => Path.relative(cacheDir, f.conversionPath)),
stats,
timings
)
} catch (err) {
logger.warn(
{ err, projectId, userId, count: pngFilesToConvert.length },
'png2pdf conversion failed, using original png(s)'
)
}
for (const f of pngFilesToConvert) {
try {
await UrlCache.promises.commitConversion(
f.conversionPath,
f.cachePath,
f.destPath
)
} catch (err) {
logger.err(
{ err, projectId, userId, path: f.destPath },
'error copying file for resources'
)
Metrics.inc('download-failed')
}
}
}
const baseHistoryVersion = localBaseVersion + changes.length
if (fullSync || changes.length || wasDirty || dirty.length) {
if (
fullSync ||
changes.length ||
wasDirty ||
dirty.length ||
pngModeChanged
) {
await saveSnapshot(
cacheKey,
snapshot,
baseHistoryVersion,
globalBlobs,
dirty
dirty,
request.png2pdf
)
}
if (fullSync) {
+96
View File
@@ -0,0 +1,96 @@
import logger from '@overleaf/logger'
import Settings from '@overleaf/settings'
import Metrics from '@overleaf/metrics'
import OError from '@overleaf/o-error'
import CommandRunner from './CommandRunner.js'
// Convert "slow" PNGs (e.g. with transparency) to PDF ahead of the compile, so
// that pdflatex can include them via the fast native path instead of paying the
// conversion cost on every run. The png2pdf tool overwrites each file in place
// with PDF bytes (pdflatex looks at file content, not the extension); PNGs that
// are already fast-includable are left unchanged.
//
// The conversion runs in an isolated, network-disabled container built FROM
// scratch (see dockerfiles/png2pdf), with the project's cache dir bind-mounted.
/**
* Whether png-to-pdf conversion is available in this deployment. It requires
* sandboxed (docker) compiles, the host path of the cache dir (so the sibling
* container can bind-mount it) and a configured image.
*
* @return {boolean}
*/
export function isEnabled() {
return Boolean(
Settings.enablePng2pdfConversions &&
Settings.clsi?.dockerRunner === true &&
Settings.path.sandboxedCompilesHostDirCache &&
Settings.png2pdfImage
)
}
// The tool prints one "Converted <file> to PDF" line per file it actually
// converts; already-fast PNGs are skipped silently.
const CONVERTED_LINE = /^Converted .+ to PDF$/gm
/**
* Run the png2pdf binary over a list of files living directly inside a project's
* cache dir, converting them in place. Files are processed sequentially by the
* binary in a single container invocation.
*
* @param {string} projectId
* @param {string} cacheProjectDir absolute path to the project's cache dir
* @param {string[]} relativePaths file names within cacheProjectDir to convert
* @param {Record<string, number>} stats
* @param {Record<string, number>} timings
* @return {Promise<void>}
*/
export async function convertPngFilesInCacheDir(
projectId,
cacheProjectDir,
relativePaths,
stats,
timings
) {
if (!isEnabled()) return
const timer = new Metrics.Timer('png2pdf')
try {
const { stdout, stderr, exitCode } = await CommandRunner.promises.run(
projectId,
// `--` terminates option parsing: the cache file names start with a `-`
// (the leading `/` of the blob URL path), which clap would otherwise treat
// as a flag.
['--in-place', '--', ...relativePaths],
cacheProjectDir,
Settings.png2pdfImage,
Settings.conversionTimeoutSeconds * 1000,
{},
'png2pdf',
null
)
if (exitCode !== 0) {
throw new OError('non-zero exit code from png2pdf', {
exitCode,
stdout,
stderr,
})
}
const converted = (stdout.match(CONVERTED_LINE) || []).length
logger.debug(
{ projectId, attempted: relativePaths.length, converted, stdout, stderr },
'png2pdf conversion completed'
)
Metrics.count('png2pdf-converted', converted)
stats.png2pdf = converted
timings.png2pdf = timer.done({ status: 'success' })
} catch (err) {
timings.png2pdf = timer.done({ status: 'error' })
throw OError.tag(err, 'png2pdf conversion failed', {
projectId,
count: relativePaths.length,
})
}
}
export default { isEnabled, convertPngFilesInCacheDir }
+4
View File
@@ -89,6 +89,10 @@ function parse(body, callback) {
default: false,
type: 'boolean',
})
response.png2pdf = _parseAttribute('png2pdf', compile.options.png2pdf, {
default: false,
type: 'boolean',
})
response.stopOnFirstError = _parseAttribute(
'stopOnFirstError',
compile.options.stopOnFirstError,
+91 -25
View File
@@ -48,50 +48,90 @@ async function createProjectDir(projectId) {
await fs.promises.mkdir(getProjectDir(projectId), { recursive: true })
}
async function tryCopyFile(src, dest) {
try {
await fs.promises.copyFile(src, dest)
return true
} catch (err) {
if (err.code === 'ENOENT') return false
throw err
}
}
/**
* Copy a url's file from the cache into destPath, downloading it first if it is
* not cached yet.
*
* When `conversionSuffix` is set, the optimized result is cached alongside the
* original at `<cachePath>.opt`. The file is downloaded to a per-user conversion
* path and NOT copied to destPath: the caller is expected to process it (e.g.
* convert a slow png to pdf) and then commit it with commitConversion (which
* renames it onto `<cachePath>.opt`). In that case a
* `{ conversionPath, cachePath, destPath }` handle is returned for the files that
* were freshly downloaded (a cache hit returns undefined, as the already-final
* file has been copied to destPath).
*
* @param {string} projectId
* @param {string} url
* @param {string|null} fallbackURL
* @param {string} destPath
* @param {Date} lastModified
* @param {string} [conversionSuffix] when set the download is deferred to a
* conversion path for processing + commitConversion instead of being copied
* to destPath
* @return {Promise<{ conversionPath: string, cachePath: string, destPath: string }|undefined>}
*/
async function downloadUrlToFile(
projectId,
url,
fallbackURL,
destPath,
lastModified
lastModified,
conversionSuffix = ''
) {
const cachePath = getCachePath(projectId, url, lastModified)
try {
// When converting, the optimized result is cached alongside the original at
// `<cachePath>.opt`, so switching modes can reuse either variant.
const cachePath =
getCachePath(projectId, url, lastModified) +
(conversionSuffix ? '.opt' : '')
{
const timer = new Metrics.Timer('url_cache', {
status: 'cache-hit',
path: 'copy',
})
try {
await fs.promises.copyFile(cachePath, destPath)
} catch (err) {
if (err.code === 'ENOENT' && fallbackURL) {
const fallbackPath = getCachePath(projectId, fallbackURL, lastModified)
await fs.promises.copyFile(fallbackPath, destPath)
} else {
throw err
}
let copied = await tryCopyFile(cachePath, destPath)
if (!copied && fallbackURL) {
copied = await tryCopyFile(
getCachePath(projectId, fallbackURL, lastModified) +
(conversionSuffix ? '.opt' : ''),
destPath
)
}
// the metric is only updated if the file is present in the cache
timer.done()
return
} catch (e) {
if (e.code !== 'ENOENT') {
throw e
if (copied) {
timer.done()
return
}
}
// time the download
const conversionPath = cachePath + conversionSuffix
{
const timer = new Metrics.Timer('url_cache', {
status: 'cache-miss',
path: 'download',
})
try {
await download(url, fallbackURL, cachePath)
await download(url, fallbackURL, conversionPath)
} finally {
timer.done()
}
}
// time the file copy
if (conversionSuffix) {
return { conversionPath, cachePath, destPath }
}
{
const timer = new Metrics.Timer('url_cache', {
status: 'cache-miss',
@@ -102,8 +142,13 @@ async function downloadUrlToFile(
}
}
async function download(url, fallbackURL, cachePath) {
let pending = PENDING_DOWNLOADS.get(cachePath)
/**
* Download a url to `targetPath`. UrlFetcher writes atomically (to its own
* `targetPath + '~'` then renames onto targetPath). Concurrent downloads of the
* same target are deduplicated.
*/
async function download(url, fallbackURL, targetPath) {
let pending = PENDING_DOWNLOADS.get(targetPath)
if (pending) {
return pending
}
@@ -111,23 +156,44 @@ async function download(url, fallbackURL, cachePath) {
pending = UrlFetcher.promises.pipeUrlToFileWithRetry(
url,
fallbackURL,
cachePath
targetPath
)
PENDING_DOWNLOADS.set(cachePath, pending)
PENDING_DOWNLOADS.set(targetPath, pending)
try {
await pending
} finally {
PENDING_DOWNLOADS.delete(cachePath)
PENDING_DOWNLOADS.delete(targetPath)
}
}
/**
* Commit a converted cache file: atomically rename it onto the cache path, then
* copy it to the destination.
*
* @param {string} conversionPath
* @param {string} cachePath
* @param {string} destPath
* @return {Promise<void>}
*/
async function commitConversion(conversionPath, cachePath, destPath) {
await fs.promises.rename(conversionPath, cachePath)
const timer = new Metrics.Timer('url_cache', {
status: 'cache-miss',
path: 'copy',
})
await fs.promises.copyFile(cachePath, destPath)
timer.done()
}
export default {
clearProject: callbackify(clearProject),
createProjectDir: callbackify(createProjectDir),
downloadUrlToFile: callbackify(downloadUrlToFile),
getProjectCacheDir: getProjectDir,
promises: {
clearProject,
createProjectDir,
downloadUrlToFile,
commitConversion,
},
}
@@ -34,6 +34,9 @@ module.exports = {
pdftocairoImage:
process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
png2pdfImage:
process.env.PNG2PDF_IMAGE || 'quay.io/sharelatex/png2pdf:2026-06-24',
enablePng2pdfConversions: process.env.ENABLE_PNG2PDF_CONVERSIONS === 'true',
maxUploadSize: 50 * 1024 * 1024,
preciousFilePattern: process.env.PRECIOUS_FILE_PATTERN || '',
@@ -168,6 +171,13 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
synctex: { 'HostConfig.AutoRemove': true },
'synctex-output': { 'HostConfig.AutoRemove': true },
conversions: { 'HostConfig.AutoRemove': true },
// The png2pdf image is built FROM scratch and has no 'tex' user, so run
// it as the same uid/gid as this clsi process, which owns the cache files
// it converts in place.
png2pdf: {
'HostConfig.AutoRemove': true,
User: `${process.getuid()}:${process.getgid()}`,
},
}
module.exports.clsi.docker.compileGroupConfig = Object.assign(
defaultCompileGroupConfig,
@@ -235,6 +245,12 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
)
}
// Host path of the clsi cache dir, so that the png2pdf conversion container
// (a sibling container started via the docker socket) can bind-mount a
// project's cache dir and convert downloaded PNGs in place.
module.exports.path.sandboxedCompilesHostDirCache =
process.env.SANDBOXED_COMPILES_HOST_DIR_CACHE
module.exports.path.sandboxedCompilesHostDirOutput =
process.env.SANDBOXED_COMPILES_HOST_DIR_OUTPUT ||
process.env.OUTPUT_HOST_DIR
@@ -49,10 +49,10 @@ describe('ConversionController', function () {
},
}
// HistoryResourceWriter is consumed via a namespace import, so the named
// export lives at the top level (not under `.promises`).
ctx.HistoryResourceWriter = {
promises: {
syncResourcesToDisk: sinon.stub().resolves(),
},
syncResourcesToDisk: sinon.stub().resolves(),
}
ctx.RequestParser = {
@@ -429,6 +429,33 @@ describe('ConversionController', function () {
const uuidDirPattern =
/^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
describe('when compiling from history', function () {
beforeEach(async function (ctx) {
ctx.parsedRequest.isCompileFromHistory = true
// Even if the request asked for png2pdf, conversions must disable it.
ctx.parsedRequest.png2pdf = true
await ctx.ConversionController.convertProjectToDocument(
ctx.req,
ctx.res,
sinon.stub()
)
})
it('should sync resources via the history writer with png2pdf disabled', function (ctx) {
sinon.assert.calledWith(
ctx.HistoryResourceWriter.syncResourcesToDisk,
'test-project-id',
'test-user-id',
sinon.match({ png2pdf: false }),
sinon.match(uuidDirPattern),
sinon.match.object, // timings
sinon.match.object // stats
)
sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk)
})
})
describe('successfully (default streaming response)', function () {
beforeEach(async function (ctx) {
await ctx.ConversionController.convertProjectToDocument(
+156
View File
@@ -0,0 +1,156 @@
import Path from 'node:path'
import sinon from 'sinon'
import { vi, describe, beforeEach, it, expect } from 'vitest'
const MODULE_PATH = Path.join(import.meta.dirname, '../../../app/js/Png2Pdf')
describe('Png2Pdf', function () {
beforeEach(async function (ctx) {
ctx.CommandRunner = {
promises: {
run: sinon.stub().resolves({ stdout: '', stderr: '', exitCode: 0 }),
},
}
ctx.Settings = {
png2pdfImage: 'mock-png2pdf-image',
enablePng2pdfConversions: true,
conversionTimeoutSeconds: 60,
clsi: { dockerRunner: true },
path: { sandboxedCompilesHostDirCache: '/host/cache' },
}
vi.doMock('../../../app/js/CommandRunner', () => ({
default: ctx.CommandRunner,
}))
vi.doMock('@overleaf/settings', () => ({ default: ctx.Settings }))
vi.doMock('@overleaf/logger', () => ({
default: { debug: sinon.stub(), warn: sinon.stub(), err: sinon.stub() },
}))
ctx.Metrics = {
Timer: sinon.stub().returns({ done: sinon.stub().returns(42) }),
count: sinon.stub(),
}
vi.doMock('@overleaf/metrics', () => ({ default: ctx.Metrics }))
ctx.Png2Pdf = (await import(MODULE_PATH)).default
})
describe('isEnabled', function () {
it('is true when enabled, sandboxed, a cache host dir and an image are set', function (ctx) {
expect(ctx.Png2Pdf.isEnabled()).to.equal(true)
})
it('is false when conversions are disabled', function (ctx) {
ctx.Settings.enablePng2pdfConversions = false
expect(ctx.Png2Pdf.isEnabled()).to.equal(false)
})
it('is false without the docker runner', function (ctx) {
ctx.Settings.clsi.dockerRunner = false
expect(ctx.Png2Pdf.isEnabled()).to.equal(false)
})
it('is false without a cache host dir', function (ctx) {
ctx.Settings.path.sandboxedCompilesHostDirCache = undefined
expect(ctx.Png2Pdf.isEnabled()).to.equal(false)
})
it('is false without an image', function (ctx) {
ctx.Settings.png2pdfImage = undefined
expect(ctx.Png2Pdf.isEnabled()).to.equal(false)
})
})
describe('convertPngFilesInCacheDir', function () {
it('runs png2pdf over the files in the cache dir', async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: 'Converted a.tmp to PDF\nConverted b.tmp to PDF\n',
stderr: '',
exitCode: 0,
})
const stats = {}
const timings = {}
await ctx.Png2Pdf.convertPngFilesInCacheDir(
'project-1',
'/cache/dir',
['a.tmp', 'b.tmp'],
stats,
timings
)
sinon.assert.calledWith(
ctx.CommandRunner.promises.run,
'project-1',
['--in-place', '--', 'a.tmp', 'b.tmp'],
'/cache/dir',
'mock-png2pdf-image',
60 * 1000,
{},
'png2pdf',
null
)
expect(stats.png2pdf).to.equal(2)
expect(timings.png2pdf).to.equal(42)
})
it('records only the files actually converted (skipped pngs print nothing)', async function (ctx) {
// The tool converted one of the two attempted files; the other was
// already fast-includable and skipped silently.
ctx.CommandRunner.promises.run.resolves({
stdout: 'Converted a.tmp to PDF\n',
stderr: '',
exitCode: 0,
})
const stats = {}
await ctx.Png2Pdf.convertPngFilesInCacheDir(
'project-1',
'/cache/dir',
['a.tmp', 'b.tmp'],
stats,
{}
)
expect(stats.png2pdf).to.equal(1)
sinon.assert.calledWith(ctx.Metrics.count, 'png2pdf-converted', 1)
})
it('does nothing when disabled', async function (ctx) {
ctx.Settings.clsi.dockerRunner = false
const stats = {}
const timings = {}
await ctx.Png2Pdf.convertPngFilesInCacheDir(
'project-1',
'/cache/dir',
['a.tmp'],
stats,
timings
)
sinon.assert.notCalled(ctx.CommandRunner.promises.run)
expect(stats.png2pdf).to.be.undefined
expect(timings.png2pdf).to.be.undefined
})
it('records the timing but not stats on a non-zero exit code', async function (ctx) {
ctx.CommandRunner.promises.run.resolves({
stdout: '',
stderr: 'boom',
exitCode: 1,
})
const stats = {}
const timings = {}
await expect(
ctx.Png2Pdf.convertPngFilesInCacheDir(
'project-1',
'/cache/dir',
['a.tmp'],
stats,
timings
)
).to.be.rejectedWith('non-zero exit code from png2pdf')
expect(stats.png2pdf).to.be.undefined
expect(timings.png2pdf).to.equal(42)
})
})
})
@@ -31,10 +31,19 @@ describe('UrlCache', () => {
},
}))
vi.doMock('../../../app/js/Png2Pdf', () => ({
default: (ctx.Png2Pdf = {
isEnabled: sinon.stub().returns(false),
convertPngFilesInCacheDir: sinon.stub().resolves(),
}),
}))
ctx.fs = {
promises: {
rm: sinon.stub().resolves(),
copyFile: sinon.stub().resolves(),
rename: sinon.stub().resolves(),
access: sinon.stub().resolves(),
},
}
@@ -136,6 +145,80 @@ describe('UrlCache', () => {
})
})
describe('downloadUrlToFile with a conversion suffix', () => {
beforeEach(ctx => {
ctx.destPath = 'path/to/destination'
ctx.conversionSuffix = '-user-cache-key'
})
it('should serve the optimized .opt variant from the cache', async ctx => {
const result = await ctx.UrlCache.promises.downloadUrlToFile(
ctx.project_id,
ctx.url,
ctx.fallbackURL,
ctx.destPath,
ctx.lastModified,
ctx.conversionSuffix
)
// a cache hit copies the already-final file to destPath and returns nothing
expect(result).to.be.undefined
expect(ctx.fs.promises.copyFile.firstCall.args[0]).to.match(/\.opt$/)
expect(ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.called).to.equal(
false
)
})
it('should return a conversion handle pointing at the .opt path on a cache miss', async ctx => {
const codedError = new Error()
codedError.code = 'ENOENT'
// miss on both the primary and the fallback .opt cache paths
ctx.fs.promises.copyFile.onCall(0).rejects(codedError)
ctx.fs.promises.copyFile.onCall(1).rejects(codedError)
const result = await ctx.UrlCache.promises.downloadUrlToFile(
ctx.project_id,
ctx.url,
ctx.fallbackURL,
ctx.destPath,
ctx.lastModified,
ctx.conversionSuffix
)
expect(result.destPath).to.equal(ctx.destPath)
expect(result.cachePath).to.match(/\.opt$/)
expect(result.conversionPath).to.equal(
result.cachePath + ctx.conversionSuffix
)
// the original is downloaded to the per-user conversion path, not destPath
expect(
ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.calledWith(
ctx.url,
ctx.fallbackURL,
result.conversionPath
)
).to.equal(true)
})
})
describe('commitConversion', () => {
it('should rename the conversion file onto the cache path and copy it to dest', async ctx => {
const conversionPath = '/cache/dir/project/key.opt-user-cache-key'
const cachePath = '/cache/dir/project/key.opt'
const destPath = 'path/to/destination'
await ctx.UrlCache.promises.commitConversion(
conversionPath,
cachePath,
destPath
)
expect(
ctx.fs.promises.rename.calledWith(conversionPath, cachePath)
).to.equal(true)
expect(ctx.fs.promises.copyFile.calledWith(cachePath, destPath)).to.equal(
true
)
})
})
describe('clearProject', () => {
beforeEach(async ctx => {
await ctx.UrlCache.promises.clearProject(ctx.project_id)
@@ -1142,6 +1142,7 @@ function _finaliseRequest(projectId, options, project, docs, files) {
timeout: options.timeout,
imageName: project.imageName,
draft: Boolean(options.draft),
png2pdf: Boolean(options.png2pdf),
stopOnFirstError: Boolean(options.stopOnFirstError),
check: options.check,
syncType: options.syncType,
@@ -202,6 +202,9 @@ const _CompileController = {
if (req.body.draft) {
options.draft = req.body.draft
}
if (req.body.png2pdf) {
options.png2pdf = req.body.png2pdf
}
if (['validate', 'error', 'silent'].includes(req.body.check)) {
options.check = req.body.check
}
@@ -288,6 +291,7 @@ const _CompileController = {
restoredClsiCache: stats?.restoredClsiCache === 1,
stopOnFirstError,
isDraftMode: !!options.draft,
isPng2pdf: !!options.png2pdf,
}
)
}
@@ -449,6 +449,7 @@ const _ProjectController = {
}
const splitTests = [
'png2pdf',
'plugin-dimensions',
'bibtex-visual-editor',
'compile-log-events',
@@ -664,6 +664,7 @@
"fair_usage_policy_applies": "",
"fast": "",
"fast_draft": "",
"fast_optimize_images": "",
"feature_enabled_or_disabled": "",
"features_like_track_changes": "",
"feedback": "",
@@ -1,26 +1,46 @@
import { useTranslation } from 'react-i18next'
import { useDetachCompileContext as useCompileContext } from '@/shared/context/detach-compile-context'
import { useMemo } from 'react'
import { useCallback, useMemo } from 'react'
import DropdownSetting from '../dropdown-setting'
import { useSetCompilationSettingWithEvent } from '@/features/ide-settings/hooks/use-set-compilation-setting'
import { useEditorAnalytics } from '@/shared/hooks/use-editor-analytics'
import { useFeatureFlag } from '@/shared/context/split-test-context'
type CompileMode = 'normal' | 'png2pdf' | 'fast_draft'
export default function DraftSetting() {
const { draft, setDraft } = useCompileContext()
const { draft, setDraft, png2pdf, setPng2pdf } = useCompileContext()
const { t } = useTranslation()
const changeDraft = useSetCompilationSettingWithEvent(
'compile-mode',
setDraft
const { sendEvent } = useEditorAnalytics()
const png2pdfEnabled = useFeatureFlag('png2pdf')
// The three modes are mutually exclusive, derived from the two underlying flags.
const mode: CompileMode = png2pdf
? 'png2pdf'
: draft
? 'fast_draft'
: 'normal'
const changeMode = useCallback(
(value: CompileMode) => {
sendEvent('recompile-setting-changed', {
setting: 'compile-mode',
settingVal: value,
})
setDraft(value === 'fast_draft')
setPng2pdf(value === 'png2pdf')
},
[sendEvent, setDraft, setPng2pdf]
)
const options = useMemo(
() => [
{ label: t('normal'), value: false },
{
label: t('fast_draft'),
value: true,
},
{ label: t('normal'), value: 'normal' as const },
...(png2pdfEnabled
? [{ label: t('fast_optimize_images'), value: 'png2pdf' as const }]
: []),
{ label: t('fast_draft'), value: 'fast_draft' as const },
],
[t]
[t, png2pdfEnabled]
)
return (
@@ -29,8 +49,9 @@ export default function DraftSetting() {
label={t('compile_mode')}
options={options}
description={t('switch_compile_mode_for_faster_draft_compilation')}
value={draft}
onChange={changeDraft}
value={mode}
onChange={changeMode}
width={png2pdfEnabled ? 'wide' : 'default'}
/>
)
}
@@ -18,6 +18,7 @@ import OLButton from '@/shared/components/ol/ol-button'
import OLButtonGroup from '@/shared/components/ol/ol-button-group'
import { useLayoutContext } from '@/shared/context/layout-context'
import { useCommandProvider } from '@/features/ide-react/hooks/use-command-provider'
import { useFeatureFlag } from '@/shared/context/split-test-context'
const modifierKey = /Mac/i.test(navigator.platform) ? 'Cmd' : 'Ctrl'
@@ -39,9 +40,11 @@ function PdfCompileButton() {
autoCompile,
compiling,
draft,
png2pdf,
hasChanges,
setAutoCompile,
setDraft,
setPng2pdf,
setStopOnValidationError,
stopOnFirstError,
stopOnValidationError,
@@ -54,6 +57,22 @@ function PdfCompileButton() {
const { t } = useTranslation()
const png2pdfEnabled = useFeatureFlag('png2pdf')
// The three compile modes (Normal / Fast [optimize images] / Fast [draft]) are
// mutually exclusive, so each selection sets both underlying flags.
const setCompileMode = useCallback(
(mode: 'normal' | 'png2pdf' | 'draft') => {
eventTracking.sendMB('recompile-setting-changed', {
setting: 'compile-mode',
settingVal: mode,
})
setDraft(mode === 'draft')
setPng2pdf(mode === 'png2pdf')
},
[setDraft, setPng2pdf]
)
const { detachRole } = useLayoutContext()
const fromScratchWithEvent = useCallback(() => {
@@ -174,16 +193,28 @@ function PdfCompileButton() {
<li role="none">
<DropdownItem
as="button"
onClick={() => sendEventAndSet(false, setDraft, 'compile-mode')}
trailingIcon={!draft ? 'check' : null}
onClick={() => setCompileMode('normal')}
trailingIcon={!draft && !png2pdf ? 'check' : null}
>
{t('normal')}
</DropdownItem>
</li>
{png2pdfEnabled && (
<li role="none">
<DropdownItem
as="button"
onClick={() => setCompileMode('png2pdf')}
trailingIcon={png2pdf ? 'check' : null}
>
{t('fast')}&nbsp;
<span className="subdued">[optimize images]</span>
</DropdownItem>
</li>
)}
<li role="none">
<DropdownItem
as="button"
onClick={() => sendEventAndSet(true, setDraft, 'compile-mode')}
onClick={() => setCompileMode('draft')}
trailingIcon={draft ? 'check' : null}
>
{t('fast')}&nbsp;<span className="subdued">[draft]</span>
@@ -87,6 +87,7 @@ export default class DocumentCompiler {
this.timer = 0
this.defaultOptions = {
draft: false,
png2pdf: false,
stopOnFirstError: false,
}
@@ -139,6 +140,7 @@ export default class DocumentCompiler {
rootDoc_id: rootDocId,
rootResourcePath,
draft: options.draft,
png2pdf: options.png2pdf,
check: 'silent', // NOTE: 'error' and 'validate' are possible, but unused
// use incremental compile for all users but revert to a full compile
// if there was previously a server error
@@ -28,6 +28,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
compiling: _compiling,
deliveryLatencies: _deliveryLatencies,
draft: _draft,
png2pdf: _png2pdf,
editedSinceCompileStarted: _editedSinceCompileStarted,
error: _error,
fileList: _fileList,
@@ -45,6 +46,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
setAnimateCompileDropdownArrow: _setAnimateCompileDropdownArrow,
setAutoCompile: _setAutoCompile,
setDraft: _setDraft,
setPng2pdf: _setPng2pdf,
setError: _setError,
setHasLintingError: _setHasLintingError,
setHighlights: _setHighlights,
@@ -119,6 +121,12 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
'detached'
)
const [draft] = useDetachStateWatcher('draft', _draft, 'detacher', 'detached')
const [png2pdf] = useDetachStateWatcher(
'png2pdf',
_png2pdf,
'detacher',
'detached'
)
const [error] = useDetachStateWatcher('error', _error, 'detacher', 'detached')
const [fileList] = useDetachStateWatcher(
'fileList',
@@ -259,6 +267,12 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
'detached',
'detacher'
)
const setPng2pdf = useDetachAction(
'setPng2pdf',
_setPng2pdf,
'detached',
'detacher'
)
const setError = useDetachAction(
'setError',
_setError,
@@ -410,6 +424,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
compiling,
deliveryLatencies,
draft,
png2pdf,
editedSinceCompileStarted,
error,
fileList,
@@ -431,6 +446,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
setAutoCompile,
setCompiling,
setDraft,
setPng2pdf,
setError,
setHasLintingError,
setHighlights,
@@ -468,6 +484,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
compiling,
deliveryLatencies,
draft,
png2pdf,
editedSinceCompileStarted,
error,
fileList,
@@ -487,6 +504,7 @@ export const DetachCompileProvider: FC<React.PropsWithChildren> = ({
setAutoCompile,
setCompiling,
setDraft,
setPng2pdf,
setError,
setHasLintingError,
setHighlights,
@@ -71,6 +71,7 @@ export type CompileContext = {
compiling: boolean
deliveryLatencies: Record<string, any>
draft: boolean
png2pdf: boolean
error?: string
fileList?: PdfFileDataList
hasChanges: boolean
@@ -93,6 +94,7 @@ export type CompileContext = {
rawLog?: string
setAutoCompile: (value: boolean) => void
setDraft: (value: any) => void
setPng2pdf: (value: any) => void
setError: (value: any) => void
setHasLintingError: (value: boolean) => void // only for storybook
setHighlights: (value: any) => void
@@ -271,6 +273,15 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
listen: true,
})
// whether the compile should optimize images (png2pdf conversion)
const [png2pdf, setPng2pdf] = usePersistedState(
`png2pdf:${projectId}`,
false,
{
listen: true,
}
)
// whether compiling should stop on first error
const [stopOnFirstError, setStopOnFirstError] = usePersistedState(
`stop_on_first_error:${projectId}`,
@@ -348,6 +359,11 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
compiler.setOption('draft', draft)
}, [compiler, draft])
// keep png2pdf (optimize images) setting in sync with the compiler
useEffect(() => {
compiler.setOption('png2pdf', png2pdf)
}, [compiler, png2pdf])
// keep stop on first error setting in sync with the compiler
useEffect(() => {
compiler.setOption('stopOnFirstError', stopOnFirstError)
@@ -396,6 +412,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
settingsUpToDate =
getRootDocInfo().rootDocId === dataFromCache.rootDocId &&
dataFromCache.options.draft === draft &&
Boolean(dataFromCache.options.png2pdf) === png2pdf &&
// Allow stopOnFirstError to be enabled in the compile from cache and disabled locally.
// Compiles that passed with stopOnFirstError=true will also pass with stopOnFirstError=false. The inverse does not hold, and we need to recompile.
!!dataFromCache.options.stopOnFirstError >= stopOnFirstError
@@ -427,6 +444,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
imageName,
stopOnFirstError,
draft,
png2pdf,
])
// always compile the PDF once after opening the project, after the doc has loaded
@@ -755,6 +773,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
compiling,
deliveryLatencies,
draft,
png2pdf,
editedSinceCompileStarted,
error,
fileList,
@@ -776,6 +795,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
setAutoCompile,
setCompiling,
setDraft,
setPng2pdf,
setError,
setHasLintingError, // only for stories
setHighlights,
@@ -813,6 +833,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
compiling,
deliveryLatencies,
draft,
png2pdf,
editedSinceCompileStarted,
error,
fileList,
@@ -831,6 +852,7 @@ export const LocalCompileProvider: FC<React.PropsWithChildren> = ({
setAnimateCompileDropdownArrow,
setAutoCompile,
setDraft,
setPng2pdf,
setError,
setHasLintingError, // only for stories
setHighlights,
+1
View File
@@ -862,6 +862,7 @@
"faq_how_does_free_trial_works_answer": "You get full access to your chosen __appName__ plan during your __len__-day free trial. There is no obligation to continue beyond the trial. Your card will be charged at the end of your __len__ day trial unless you cancel before then. You can cancel via your subscription settings.",
"fast": "Fast",
"fast_draft": "Fast [draft]",
"fast_optimize_images": "Fast [optimize images]",
"fastest": "Fastest",
"feature_enabled_or_disabled": "Feature enabled/disabled",
"feature_included": "Feature included",
@@ -8,18 +8,12 @@ import {
import userEvent from '@testing-library/user-event'
import DraftSetting from '@/features/ide-settings/components/compiler-settings/draft-setting'
const OPTIONS = [
{
label: 'Normal',
value: false,
},
{
label: 'Fast [draft]',
value: true,
},
]
describe('<DraftSetting />', function () {
afterEach(function () {
window.metaAttributesCache.delete('ol-splitTestVariants')
localStorage.clear()
})
it('each option is shown and can be selected', async function () {
render(
<EditorProviders>
@@ -31,15 +25,45 @@ describe('<DraftSetting />', function () {
const select = screen.getByLabelText('Compile mode')
for (const option of OPTIONS) {
const options = [
{ label: 'Normal', value: 'normal', draft: false },
{ label: 'Fast [draft]', value: 'fast_draft', draft: true },
]
for (const option of options) {
const optionElement = within(select).getByText(option.label)
expect(optionElement.getAttribute('value')).to.equal(
option.value.toString()
)
expect(optionElement.getAttribute('value')).to.equal(option.value)
await userEvent.selectOptions(select, [optionElement])
expect(!!localStorage.getItem(`draft:${projectDefaults._id}`)).to.equal(
option.value
option.draft
)
expect(!!localStorage.getItem(`png2pdf:${projectDefaults._id}`)).to.equal(
false
)
}
})
it('offers the optimize-images option behind the png2pdf split test', async function () {
window.metaAttributesCache.set('ol-splitTestVariants', {
png2pdf: 'enabled',
})
render(
<EditorProviders>
<SettingsModalProvider>
<DraftSetting />
</SettingsModalProvider>
</EditorProviders>
)
const select = screen.getByLabelText('Compile mode')
const optionElement = within(select).getByText('Fast [optimize images]')
expect(optionElement.getAttribute('value')).to.equal('png2pdf')
await userEvent.selectOptions(select, [optionElement])
expect(!!localStorage.getItem(`png2pdf:${projectDefaults._id}`)).to.equal(
true
)
expect(!!localStorage.getItem(`draft:${projectDefaults._id}`)).to.equal(
false
)
})
})
@@ -728,10 +728,12 @@ const BASE_COMPILE_CONTEXT_MOCK = {
logEntries: undefined,
outputFilesArchive: undefined,
pdfViewer: 'pdfjs',
png2pdf: false,
position: undefined,
rawLog: undefined,
recompileFromScratch: () => {},
setAnimateCompileDropdownArrow: () => {},
setPng2pdf: () => {},
setHasLintingError: () => {},
setHighlights: () => {},
setPosition: () => {},
@@ -1031,6 +1031,29 @@ describe('ClsiManager', function () {
})
})
describe('with the png2pdf option', function () {
beforeEach(async function (ctx) {
await ctx.ClsiManager.promises.sendRequest(
null,
ctx.project._id,
ctx.user_id,
{
timeout: 100,
png2pdf: true,
}
)
})
it('should add the png2pdf option into the request', function (ctx) {
expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
sinon.match.any,
sinon.match({
json: { compile: { options: { png2pdf: true } } },
})
)
})
})
describe('with a failed compile', function () {
beforeEach(async function (ctx) {
ctx.responseBody.compile.status = 'failure'
+1
View File
@@ -80,6 +80,7 @@ export type CompileResponseData = {
export type CompileOptions = {
draft?: boolean
png2pdf?: boolean
stopOnFirstError?: boolean
isAutoCompileOnLoad?: boolean
isAutoCompileOnChange?: boolean