Adding 2025 and 2026 incremental compiles options (#35850)

* feat: adding 2025 and 2026 incremental compiles options

* feat: remove dmtcp from compiler list and autoselect checkpointing version with labs inclusion

* feat: updating seccomp profile for checkpointing

* feat: disable non-premium dmtcp compiles, and align checkpoint version detection between clsi and web

* feat: adding in reminder of checkpoint experiment, and using session based split test retrieval

* Increase Packer build.sh timeout from 1h15m to 1h30m as build takes approx 1h

* fix: moving checkpointing check from tag name based to hasCheckpointing flag set as part of allowed_images, and removing uneeded seccomp rule

* feat: move variant/ image selection to CLSI, and have web control only what options to set for a compile. removing uneeded extra variants from webs list of allowed images

* feat: adding compile timer as a query string as well

---------

Co-authored-by: Tim Gibbon <tim.gibbon@overleaf.com>
GitOrigin-RevId: cdbdf76a62dcd5a585925f18d64b6cb9cd6ed373

services/web/app/src/Features/Compile/ClsiManager.mjs

@@ -1,3 +1,4 @@
+import Path from 'node:path'
 import { callbackify } from 'node:util'
 import { callbackifyMultiResult } from '@overleaf/promise-utils'
 import {
@@ -1184,7 +1185,6 @@ function _finaliseRequest(projectId, options, project, docs, files) {
   const hasPremiumCompiles = ['alpha', 'priority'].includes(
     options.compileGroup
   )
-
   return {
     compile: {
       options: {
@@ -1197,6 +1197,11 @@ function _finaliseRequest(projectId, options, project, docs, files) {
         draft: Boolean(options.draft),
         // enable for premium compiles only
         png2pdf: Boolean(options.png2pdf) && hasPremiumCompiles,
+        // enable for premium compiles on an image that has a checkpointing build
+        enableCheckpoint:
+          Boolean(options.checkpointing) &&
+          hasPremiumCompiles &&
+          _imageHasCheckpointing(project.imageName),
         stopOnFirstError: Boolean(options.stopOnFirstError),
         check: options.check,
         syncType: options.syncType,
@@ -1213,7 +1218,6 @@ function _finaliseRequest(projectId, options, project, docs, files) {
         enablePdfCaching:
           (Settings.enablePdfCaching && options.enablePdfCaching) || false,
         pdfCachingMinChunkSize: options.pdfCachingMinChunkSize,
-        enableCheckpoint: Boolean(options.enableCheckpoint),
         flags,
         metricsMethod: options.compileGroup,
         metricsPath: options.metricsPath,
@@ -1228,6 +1232,21 @@ function _finaliseRequest(projectId, options, project, docs, files) {
   }
 }
 
+// checkpointing compiles run on a checkpointing build of the project's image,
+// which is only available for some TeX Live years
+function _imageHasCheckpointing(imageName) {
+  if (!imageName) {
+    return false
+  }
+  // allowedImageNames holds bare image names, so drop the hosting URL first
+  const bareImageName = Path.basename(imageName)
+  return Boolean(
+    Settings.allowedImageNames?.find(
+      allowedImage => allowedImage.imageName === bareImageName
+    )?.hasCheckpointing
+  )
+}
+
 async function buildDocumentConversionRequest(projectId, userId, options) {
   return await _buildRequest(null, projectId, userId, {
     ...options,

services/web/app/src/Features/Compile/CompileController.mjs

@@ -81,13 +81,6 @@ async function _getSplitTestOptions(req, res) {
         { enablePdfCaching: false }
       : { enablePdfCaching, pdfCachingMinChunkSize }
 
-  const enableCheckpoint = await SplitTestHandler.promises.featureFlagEnabled(
-    req,
-    res,
-    'compile-with-checkpoint',
-    { includeReferer: true }
-  )
-
   const enablePng2Pdf = await SplitTestHandler.promises.featureFlagEnabled(
     req,
     res,
@@ -100,12 +93,24 @@ async function _getSplitTestOptions(req, res) {
       ? { enablePng2Pdf: false }
       : { enablePng2Pdf }
 
+  const checkpointingEnabled =
+    await SplitTestHandler.promises.featureFlagEnabled(
+      req,
+      res,
+      'compile-with-checkpoint',
+      { includeReferer: true }
+    )
+
+  const checkpointCompilesOptions = {
+    enableCheckpointCompiles: checkpointingEnabled,
+  }
+
   return {
     compileFromHistory,
     pdfDownloadDomain,
     ...pdfCachingOptions,
-    enableCheckpoint,
     ...png2PdfOptions,
+    ...checkpointCompilesOptions,
   }
 }
 
@@ -454,19 +459,19 @@ const _CompileController = {
       pdfCachingMinChunkSize,
       pdfDownloadDomain,
       compileFromHistory,
-      enableCheckpoint,
       enablePng2Pdf,
+      enableCheckpointCompiles,
     } = await _getSplitTestOptions(req, res)
     if (Features.hasFeature('saas')) {
       options.compileFromClsiCache = true
       options.populateClsiCache = true
       options.compileFromHistory = compileFromHistory
-      if (enableCheckpoint) {
-        options.enableCheckpoint = enableCheckpoint
-      }
       if (enablePng2Pdf) {
         options.png2pdf = enablePng2Pdf
       }
+      if (enableCheckpointCompiles) {
+        options.checkpointing = true
+      }
     }
     options.enablePdfCaching = enablePdfCaching
     if (enablePdfCaching) {

services/web/app/src/Features/Project/ProjectHelper.mjs

@@ -189,13 +189,13 @@ function _imageAllowed(
 }
 
 async function getAllowedImagesForUser(req, res, user) {
-  let images = Settings.allowedImageNames || []
+  const images = Settings.allowedImageNames || []
 
   const alphaImagesAllowed = Boolean(user?.alphaProgram)
   const monthlyExperimentalImagesAllowed =
     await _monthlyExperimentalImageAllowed(req, res)
 
-  images = images.map(image => {
+  return images.map(image => {
     return {
       ...image,
       allowed: _imageAllowed(
@@ -206,6 +206,4 @@ async function getAllowedImagesForUser(req, res, user) {
       rolling: image.monthlyExperimental,
     }
   })
-
-  return images
 }

services/web/frontend/extracted-translations.json

@@ -1013,6 +1013,8 @@
   "increase_indent": "",
   "increased_ai_allowance": "",
   "increased_compile_timeout": "",
+  "incremental_compiles_are_on": "",
+  "incremental_compiles_reusing_last_compile": "",
   "info": "",
   "inline": "",
   "inline_math": "",

services/web/frontend/js/features/pdf-preview/components/checkpoint-compiles-enabled-reminder.tsx

@@ -0,0 +1,32 @@
+import Notification from '@/shared/components/notification'
+import { useTranslation, Trans } from 'react-i18next'
+import { isSplitTestEnabled } from '@/utils/splitTestUtils'
+
+const CheckpointCompilesEnabledReminder = () => {
+  const { t } = useTranslation()
+  if (!isSplitTestEnabled('compile-with-checkpoint')) {
+    return null
+  }
+
+  const content = (
+    <Trans
+      i18nKey="incremental_compiles_reusing_last_compile"
+      components={[
+        <a href="/labs/participate" target="_blank" rel="noreferrer" />, // eslint-disable-line react/jsx-key, jsx-a11y/anchor-has-content
+      ]}
+    />
+  )
+
+  return (
+    <div className="notification-list">
+      <Notification
+        title={t('incremental_compiles_are_on')}
+        content={content}
+        type="info"
+        className="mb-0"
+      />
+    </div>
+  )
+}
+
+export default CheckpointCompilesEnabledReminder

services/web/frontend/js/features/pdf-preview/components/error-logs.tsx

@@ -16,6 +16,7 @@ import getMeta from '@/utils/meta'
 import PdfClearCacheButton from '@/features/pdf-preview/components/pdf-clear-cache-button'
 import PdfDownloadFilesButton from '@/features/pdf-preview/components/pdf-download-files-button'
 import RollingBuildSelectedReminder from './rolling-build-selected-reminder'
+import CheckpointCompilesEnabledReminder from './checkpoint-compiles-enabled-reminder'
 import ErrorAssistantAiPaywallNotification from './error-assistant-ai-paywall-notification'
 
 type ErrorLogTab = {
@@ -78,6 +79,7 @@ function ErrorLogs({
       <TabContent className="error-logs new-error-logs">
         <div className="logs-pane-content">
           <RollingBuildSelectedReminder />
+          <CheckpointCompilesEnabledReminder />
           {stoppedOnFirstError && includeErrors && <StopOnFirstErrorPrompt />}
 
           {loadingError && (

services/web/frontend/js/features/pdf-preview/components/pdf-preview-hybrid-toolbar.tsx

@@ -38,7 +38,9 @@ function PdfPreviewHybridToolbar() {
 
 function PdfPreviewHybridToolbarInner() {
   const { focusMode } = useLayoutContext()
-  const showCompileTimer = useFeatureFlag('compile-with-checkpoint')
+  const debugTimer =
+    new URLSearchParams(window.location.search).get('compile-timer') === 'true'
+  const showCompileTimer = useFeatureFlag('show-compile-timer') || debugTimer
 
   return (
     <>

services/web/locales/en.json

@@ -1285,6 +1285,8 @@
   "increased_ai_allowance": "Increased AI allowance",
   "increased_ai_allowance_tooltip": "10 AI uses per day across all AI tools.",
   "increased_compile_timeout": "Increased compile timeout",
+  "incremental_compiles_are_on": "Faster recompiles are on",
+  "incremental_compiles_reusing_last_compile": "We’re reusing part of your last compile to speed things up. It’s experimental, so if your PDF or logs look wrong, turn it off by <0>leaving the Labs experiment</0>, then recompile.",
   "individuals": "Individuals",
   "info": "Info",
   "initiator": "Initiator",

services/web/test/unit/src/Compile/ClsiManager.test.mjs

@@ -198,6 +198,10 @@ describe('ClsiManager', function () {
       clsiCookie: { key: 'clsiserver' },
       safeCompilers: ['pdflatex', 'latex', 'xelatex', 'lualatex'],
       defaultLatexCompiler: 'pdflatex',
+      allowedImageNames: [
+        { imageName: 'mock-image-name', hasCheckpointing: true },
+        { imageName: 'mock-image-name-no-checkpointing' },
+      ],
     }
     ctx.ClsiCacheHandler = {
       clearCache: sinon.stub().resolves(),
@@ -1117,6 +1121,152 @@ describe('ClsiManager', function () {
       })
     })
 
+    describe('with the checkpointing option', function () {
+      it('should ask the clsi to enable checkpointing, leaving the image alone', async function (ctx) {
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'priority',
+            checkpointing: true,
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: {
+                options: {
+                  imageName: ctx.project.imageName,
+                  enableCheckpoint: true,
+                },
+              },
+            },
+          })
+        )
+      })
+
+      it('should not enable checkpointing for a standard compileGroup', async function (ctx) {
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'standard',
+            checkpointing: true,
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: { options: { enableCheckpoint: false } },
+            },
+          })
+        )
+      })
+
+      it('should not enable checkpointing without the option', async function (ctx) {
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'priority',
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: {
+                options: {
+                  imageName: ctx.project.imageName,
+                  enableCheckpoint: false,
+                },
+              },
+            },
+          })
+        )
+      })
+
+      it('should not enable checkpointing when the image has no checkpointing build', async function (ctx) {
+        ctx.project.imageName = 'mock-image-name-no-checkpointing'
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'priority',
+            checkpointing: true,
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: { options: { enableCheckpoint: false } },
+            },
+          })
+        )
+      })
+
+      it('should not enable checkpointing when the project has no image', async function (ctx) {
+        delete ctx.project.imageName
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'priority',
+            checkpointing: true,
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: { options: { enableCheckpoint: false } },
+            },
+          })
+        )
+      })
+
+      it('should look up the image ignoring the registry host', async function (ctx) {
+        ctx.project.imageName = 'quay.io/sharelatex/mock-image-name'
+        await ctx.ClsiManager.promises.sendRequest(
+          null,
+          ctx.project._id,
+          ctx.user_id,
+          {
+            timeout: 100,
+            compileGroup: 'priority',
+            checkpointing: true,
+          }
+        )
+
+        expect(ctx.FetchUtils.fetchStringWithResponse).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match({
+            json: {
+              compile: { options: { enableCheckpoint: true } },
+            },
+          })
+        )
+      })
+    })
+
     describe('with the png2pdf option', function () {
       beforeEach(async function (ctx) {
         await ctx.ClsiManager.promises.sendRequest(