diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 57770bb290..9d4aa91423 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -170,7 +170,7 @@ async function doCompile(request, stats, timings) { ) // set up environment variables for chktex - const env = { + let env = { OVERLEAF_PROJECT_ID: request.project_id, } if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') { @@ -194,10 +194,11 @@ async function doCompile(request, stats, timings) { } } - // Pass through checkpoint setting - if (request.enableCheckpoint) { - env.ENABLE_CHECKPOINT = '1' - } + // the requested compile options may need a variant of the requested image, + // which brings its own environment + const imageVariantSettings = _getImageVariantSettings(request) + request.imageName = imageVariantSettings.imageName + env = { ...env, ...imageVariantSettings.env } const compileStart = Date.now() @@ -769,10 +770,45 @@ function _parseWordcountFromOutput(output) { return results } +function _getAllowedImages() { + return Settings.clsi?.docker?.allowedImages +} + function _isImageNameAllowed(imageName) { - const ALLOWED_IMAGES = - Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages - return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName) + const allowedImages = _getAllowedImages() + return !allowedImages || allowedImages.includes(imageName) +} + +// Variants are images built on top of a base image, tagged with a suffix, eg +// texlive-full:2026.1.checkpointing is the checkpointing variant of +// texlive-full:2026.1. Returns the settings needed to compile the request on +// the variant its options call for: the image to run and the environment that +// image needs. Falls back to the requested image when no variant applies, or +// when the variant is not available on this host. +function _getImageVariantSettings(request) { + const { imageName } = request + + // checkpointing compiles run on the checkpointing variant of the image + if (request.enableCheckpoint) { + const checkpointingImageName = `${imageName}.checkpointing` + if (imageName && _getAllowedImages()?.includes(checkpointingImageName)) { + return { + imageName: checkpointingImageName, + env: { ENABLE_CHECKPOINT: '1' }, + } + } + // variant either doesnt exist when it should, or user was able to make a request they shouldnt be able to + logger.error( + { + projectId: request.project_id, + userId: request.user_id, + imageName, + }, + 'no checkpointing variant available for image, compiling without it' + ) + } + + return { imageName, env: {} } } function _emitMetrics(request, status, stats, timings) { diff --git a/services/clsi/test/unit/js/CompileManager.test.js b/services/clsi/test/unit/js/CompileManager.test.js index 19829f9b9f..930d84e73c 100644 --- a/services/clsi/test/unit/js/CompileManager.test.js +++ b/services/clsi/test/unit/js/CompileManager.test.js @@ -186,6 +186,16 @@ describe('CompileManager', () => { default: ctx.Settings, })) + ctx.logger = { + debug: sinon.stub(), + info: sinon.stub(), + warn: sinon.stub(), + error: sinon.stub(), + } + vi.doMock('@overleaf/logger', () => ({ + default: ctx.logger, + })) + vi.doMock('@overleaf/metrics', () => ({ default: { inc: sinon.stub(), @@ -390,6 +400,77 @@ describe('CompileManager', () => { }) }) + describe('with the enableCheckpoint option', () => { + beforeEach(ctx => { + ctx.baseImage = 'example.com/texlive-full:2025.1' + ctx.checkpointingImage = `${ctx.baseImage}.checkpointing` + ctx.request.imageName = ctx.baseImage + ctx.request.enableCheckpoint = true + ctx.Settings.clsi.docker.allowedImages = [ + ctx.baseImage, + ctx.checkpointingImage, + ] + }) + + it('should compile on the checkpointing variant of the image', async ctx => { + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + const { image, environment } = + ctx.LatexRunner.promises.runLatex.args[0][1] + expect(image).to.equal(ctx.checkpointingImage) + expect(environment).to.have.property('ENABLE_CHECKPOINT', '1') + }) + + it('should log an error and compile on the base image when there is no checkpointing variant', async ctx => { + ctx.Settings.clsi.docker.allowedImages = [ctx.baseImage] + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + const { image, environment } = + ctx.LatexRunner.promises.runLatex.args[0][1] + expect(image).to.equal(ctx.baseImage) + expect(environment).to.not.have.property('ENABLE_CHECKPOINT') + expect(ctx.logger.error).to.have.been.calledWith( + sinon.match({ imageName: ctx.baseImage }) + ) + }) + + it('should log an error and compile on the base image when there is no allow list', async ctx => { + delete ctx.Settings.clsi.docker.allowedImages + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + const { image, environment } = + ctx.LatexRunner.promises.runLatex.args[0][1] + expect(image).to.equal(ctx.baseImage) + expect(environment).to.not.have.property('ENABLE_CHECKPOINT') + expect(ctx.logger.error).to.have.been.called + }) + + it('should log an error and compile without an image when none is requested', async ctx => { + delete ctx.request.imageName + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + const { image, environment } = + ctx.LatexRunner.promises.runLatex.args[0][1] + expect(image).to.be.undefined + expect(environment).to.not.have.property('ENABLE_CHECKPOINT') + expect(ctx.logger.error).to.have.been.called + }) + }) + + describe('without the enableCheckpoint option', () => { + beforeEach(async ctx => { + ctx.request.imageName = 'example.com/texlive-full:2025.1' + ctx.Settings.clsi.docker.allowedImages = [ + 'example.com/texlive-full:2025.1', + 'example.com/texlive-full:2025.1.checkpointing', + ] + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + }) + + it('should not enable checkpointing or change the image', ctx => { + const { image, environment } = + ctx.LatexRunner.promises.runLatex.args[0][1] + expect(image).to.equal('example.com/texlive-full:2025.1') + expect(environment).to.not.have.property('ENABLE_CHECKPOINT') + }) + }) + describe('with a check option', () => { beforeEach(async ctx => { ctx.request.check = 'error' diff --git a/services/clsi/test/unit/js/RequestParser.test.js b/services/clsi/test/unit/js/RequestParser.test.js index b08a6acb09..bae771ed7b 100644 --- a/services/clsi/test/unit/js/RequestParser.test.js +++ b/services/clsi/test/unit/js/RequestParser.test.js @@ -194,6 +194,39 @@ describe('RequestParser', () => { }) }) + describe('with enableCheckpoint set', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.validRequest.compile.options.enableCheckpoint = true + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the enableCheckpoint attribute', ctx => { + expect(ctx.data.enableCheckpoint).to.equal(true) + }) + }) + + describe('with enableCheckpoint not specified', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should default the enableCheckpoint attribute to false', ctx => { + expect(ctx.data.enableCheckpoint).to.equal(false) + }) + }) + describe('without a timeout specified', () => { beforeEach(async ctx => { await new Promise((resolve, reject) => { diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index 5500aa29f9..7df8ea49d5 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1,3 +1,4 @@ +import Path from 'node:path' import { callbackify } from 'node:util' import { callbackifyMultiResult } from '@overleaf/promise-utils' import { @@ -1184,7 +1185,6 @@ function _finaliseRequest(projectId, options, project, docs, files) { const hasPremiumCompiles = ['alpha', 'priority'].includes( options.compileGroup ) - return { compile: { options: { @@ -1197,6 +1197,11 @@ function _finaliseRequest(projectId, options, project, docs, files) { draft: Boolean(options.draft), // enable for premium compiles only png2pdf: Boolean(options.png2pdf) && hasPremiumCompiles, + // enable for premium compiles on an image that has a checkpointing build + enableCheckpoint: + Boolean(options.checkpointing) && + hasPremiumCompiles && + _imageHasCheckpointing(project.imageName), stopOnFirstError: Boolean(options.stopOnFirstError), check: options.check, syncType: options.syncType, @@ -1213,7 +1218,6 @@ function _finaliseRequest(projectId, options, project, docs, files) { enablePdfCaching: (Settings.enablePdfCaching && options.enablePdfCaching) || false, pdfCachingMinChunkSize: options.pdfCachingMinChunkSize, - enableCheckpoint: Boolean(options.enableCheckpoint), flags, metricsMethod: options.compileGroup, metricsPath: options.metricsPath, @@ -1228,6 +1232,21 @@ function _finaliseRequest(projectId, options, project, docs, files) { } } +// checkpointing compiles run on a checkpointing build of the project's image, +// which is only available for some TeX Live years +function _imageHasCheckpointing(imageName) { + if (!imageName) { + return false + } + // allowedImageNames holds bare image names, so drop the hosting URL first + const bareImageName = Path.basename(imageName) + return Boolean( + Settings.allowedImageNames?.find( + allowedImage => allowedImage.imageName === bareImageName + )?.hasCheckpointing + ) +} + async function buildDocumentConversionRequest(projectId, userId, options) { return await _buildRequest(null, projectId, userId, { ...options, diff --git a/services/web/app/src/Features/Compile/CompileController.mjs b/services/web/app/src/Features/Compile/CompileController.mjs index f1f858f10a..16663c4455 100644 --- a/services/web/app/src/Features/Compile/CompileController.mjs +++ b/services/web/app/src/Features/Compile/CompileController.mjs @@ -81,13 +81,6 @@ async function _getSplitTestOptions(req, res) { { enablePdfCaching: false } : { enablePdfCaching, pdfCachingMinChunkSize } - const enableCheckpoint = await SplitTestHandler.promises.featureFlagEnabled( - req, - res, - 'compile-with-checkpoint', - { includeReferer: true } - ) - const enablePng2Pdf = await SplitTestHandler.promises.featureFlagEnabled( req, res, @@ -100,12 +93,24 @@ async function _getSplitTestOptions(req, res) { ? { enablePng2Pdf: false } : { enablePng2Pdf } + const checkpointingEnabled = + await SplitTestHandler.promises.featureFlagEnabled( + req, + res, + 'compile-with-checkpoint', + { includeReferer: true } + ) + + const checkpointCompilesOptions = { + enableCheckpointCompiles: checkpointingEnabled, + } + return { compileFromHistory, pdfDownloadDomain, ...pdfCachingOptions, - enableCheckpoint, ...png2PdfOptions, + ...checkpointCompilesOptions, } } @@ -454,19 +459,19 @@ const _CompileController = { pdfCachingMinChunkSize, pdfDownloadDomain, compileFromHistory, - enableCheckpoint, enablePng2Pdf, + enableCheckpointCompiles, } = await _getSplitTestOptions(req, res) if (Features.hasFeature('saas')) { options.compileFromClsiCache = true options.populateClsiCache = true options.compileFromHistory = compileFromHistory - if (enableCheckpoint) { - options.enableCheckpoint = enableCheckpoint - } if (enablePng2Pdf) { options.png2pdf = enablePng2Pdf } + if (enableCheckpointCompiles) { + options.checkpointing = true + } } options.enablePdfCaching = enablePdfCaching if (enablePdfCaching) { diff --git a/services/web/app/src/Features/Project/ProjectHelper.mjs b/services/web/app/src/Features/Project/ProjectHelper.mjs index 4b3dadc651..1f2ce88f8c 100644 --- a/services/web/app/src/Features/Project/ProjectHelper.mjs +++ b/services/web/app/src/Features/Project/ProjectHelper.mjs @@ -189,13 +189,13 @@ function _imageAllowed( } async function getAllowedImagesForUser(req, res, user) { - let images = Settings.allowedImageNames || [] + const images = Settings.allowedImageNames || [] const alphaImagesAllowed = Boolean(user?.alphaProgram) const monthlyExperimentalImagesAllowed = await _monthlyExperimentalImageAllowed(req, res) - images = images.map(image => { + return images.map(image => { return { ...image, allowed: _imageAllowed( @@ -206,6 +206,4 @@ async function getAllowedImagesForUser(req, res, user) { rolling: image.monthlyExperimental, } }) - - return images } diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 67443b25e2..c4b747df09 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -1013,6 +1013,8 @@ "increase_indent": "", "increased_ai_allowance": "", "increased_compile_timeout": "", + "incremental_compiles_are_on": "", + "incremental_compiles_reusing_last_compile": "", "info": "", "inline": "", "inline_math": "", diff --git a/services/web/frontend/js/features/pdf-preview/components/checkpoint-compiles-enabled-reminder.tsx b/services/web/frontend/js/features/pdf-preview/components/checkpoint-compiles-enabled-reminder.tsx new file mode 100644 index 0000000000..788c7f14e8 --- /dev/null +++ b/services/web/frontend/js/features/pdf-preview/components/checkpoint-compiles-enabled-reminder.tsx @@ -0,0 +1,32 @@ +import Notification from '@/shared/components/notification' +import { useTranslation, Trans } from 'react-i18next' +import { isSplitTestEnabled } from '@/utils/splitTestUtils' + +const CheckpointCompilesEnabledReminder = () => { + const { t } = useTranslation() + if (!isSplitTestEnabled('compile-with-checkpoint')) { + return null + } + + const content = ( + , // eslint-disable-line react/jsx-key, jsx-a11y/anchor-has-content + ]} + /> + ) + + return ( +
+ +
+ ) +} + +export default CheckpointCompilesEnabledReminder diff --git a/services/web/frontend/js/features/pdf-preview/components/error-logs.tsx b/services/web/frontend/js/features/pdf-preview/components/error-logs.tsx index e8c6b78ae4..24615af2e5 100644 --- a/services/web/frontend/js/features/pdf-preview/components/error-logs.tsx +++ b/services/web/frontend/js/features/pdf-preview/components/error-logs.tsx @@ -16,6 +16,7 @@ import getMeta from '@/utils/meta' import PdfClearCacheButton from '@/features/pdf-preview/components/pdf-clear-cache-button' import PdfDownloadFilesButton from '@/features/pdf-preview/components/pdf-download-files-button' import RollingBuildSelectedReminder from './rolling-build-selected-reminder' +import CheckpointCompilesEnabledReminder from './checkpoint-compiles-enabled-reminder' import ErrorAssistantAiPaywallNotification from './error-assistant-ai-paywall-notification' type ErrorLogTab = { @@ -78,6 +79,7 @@ function ErrorLogs({
+ {stoppedOnFirstError && includeErrors && } {loadingError && ( diff --git a/services/web/frontend/js/features/pdf-preview/components/pdf-preview-hybrid-toolbar.tsx b/services/web/frontend/js/features/pdf-preview/components/pdf-preview-hybrid-toolbar.tsx index a4da4f9767..b2c140453c 100644 --- a/services/web/frontend/js/features/pdf-preview/components/pdf-preview-hybrid-toolbar.tsx +++ b/services/web/frontend/js/features/pdf-preview/components/pdf-preview-hybrid-toolbar.tsx @@ -38,7 +38,9 @@ function PdfPreviewHybridToolbar() { function PdfPreviewHybridToolbarInner() { const { focusMode } = useLayoutContext() - const showCompileTimer = useFeatureFlag('compile-with-checkpoint') + const debugTimer = + new URLSearchParams(window.location.search).get('compile-timer') === 'true' + const showCompileTimer = useFeatureFlag('show-compile-timer') || debugTimer return ( <> diff --git a/services/web/locales/en.json b/services/web/locales/en.json index bbf4fbb511..556d099de9 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -1285,6 +1285,8 @@ "increased_ai_allowance": "Increased AI allowance", "increased_ai_allowance_tooltip": "10 AI uses per day across all AI tools.", "increased_compile_timeout": "Increased compile timeout", + "incremental_compiles_are_on": "Faster recompiles are on", + "incremental_compiles_reusing_last_compile": "We’re reusing part of your last compile to speed things up. It’s experimental, so if your PDF or logs look wrong, turn it off by <0>leaving the Labs experiment, then recompile.", "individuals": "Individuals", "info": "Info", "initiator": "Initiator", diff --git a/services/web/test/unit/src/Compile/ClsiManager.test.mjs b/services/web/test/unit/src/Compile/ClsiManager.test.mjs index 5b37ab17ed..14833e9a05 100644 --- a/services/web/test/unit/src/Compile/ClsiManager.test.mjs +++ b/services/web/test/unit/src/Compile/ClsiManager.test.mjs @@ -198,6 +198,10 @@ describe('ClsiManager', function () { clsiCookie: { key: 'clsiserver' }, safeCompilers: ['pdflatex', 'latex', 'xelatex', 'lualatex'], defaultLatexCompiler: 'pdflatex', + allowedImageNames: [ + { imageName: 'mock-image-name', hasCheckpointing: true }, + { imageName: 'mock-image-name-no-checkpointing' }, + ], } ctx.ClsiCacheHandler = { clearCache: sinon.stub().resolves(), @@ -1117,6 +1121,152 @@ describe('ClsiManager', function () { }) }) + describe('with the checkpointing option', function () { + it('should ask the clsi to enable checkpointing, leaving the image alone', async function (ctx) { + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'priority', + checkpointing: true, + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { + options: { + imageName: ctx.project.imageName, + enableCheckpoint: true, + }, + }, + }, + }) + ) + }) + + it('should not enable checkpointing for a standard compileGroup', async function (ctx) { + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'standard', + checkpointing: true, + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { options: { enableCheckpoint: false } }, + }, + }) + ) + }) + + it('should not enable checkpointing without the option', async function (ctx) { + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'priority', + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { + options: { + imageName: ctx.project.imageName, + enableCheckpoint: false, + }, + }, + }, + }) + ) + }) + + it('should not enable checkpointing when the image has no checkpointing build', async function (ctx) { + ctx.project.imageName = 'mock-image-name-no-checkpointing' + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'priority', + checkpointing: true, + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { options: { enableCheckpoint: false } }, + }, + }) + ) + }) + + it('should not enable checkpointing when the project has no image', async function (ctx) { + delete ctx.project.imageName + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'priority', + checkpointing: true, + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { options: { enableCheckpoint: false } }, + }, + }) + ) + }) + + it('should look up the image ignoring the registry host', async function (ctx) { + ctx.project.imageName = 'quay.io/sharelatex/mock-image-name' + await ctx.ClsiManager.promises.sendRequest( + null, + ctx.project._id, + ctx.user_id, + { + timeout: 100, + compileGroup: 'priority', + checkpointing: true, + } + ) + + expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith( + sinon.match.any, + sinon.match({ + json: { + compile: { options: { enableCheckpoint: true } }, + }, + }) + ) + }) + }) + describe('with the png2pdf option', function () { beforeEach(async function (ctx) { await ctx.ClsiManager.promises.sendRequest(