[clsi] handle draft mode and tikzexternalize as part of sync phase (#32516)

* [clsi] handle draft mode and tikzexternalize as part of sync phase

* [clsi] emit empty string from SafeReader on ENOENT

* [clsi] persist history state after clearing dirty state without changes

GitOrigin-RevId: d9dcd2e6887017f7935b5e95bdbdc6e11a3b18f5

services/clsi/app/js/CompileManager.js

@@ -122,6 +122,25 @@ async function doCompile(request, stats, timings) {
         request,
         compileDir
       )
+
+      // apply a series of file modifications/creations for draft mode and tikz
+      if (request.draft) {
+        await DraftModeManager.promises.injectDraftMode(
+          Path.join(compileDir, request.rootResourcePath)
+        )
+      }
+
+      const needsMainFile = await TikzManager.promises.checkMainFile(
+        compileDir,
+        request.rootResourcePath,
+        resourceList
+      )
+      if (needsMainFile) {
+        await TikzManager.promises.injectOutputFile(
+          compileDir,
+          request.rootResourcePath
+        )
+      }
     }
   } catch (error) {
     if (error instanceof Errors.FilesOutOfSyncError) {
@@ -173,25 +192,6 @@ async function doCompile(request, stats, timings) {
     }
   }
 
-  // apply a series of file modifications/creations for draft mode and tikz
-  if (request.draft) {
-    await DraftModeManager.promises.injectDraftMode(
-      Path.join(compileDir, request.rootResourcePath)
-    )
-  }
-
-  const needsMainFile = await TikzManager.promises.checkMainFile(
-    compileDir,
-    request.rootResourcePath,
-    resourceList
-  )
-  if (needsMainFile) {
-    await TikzManager.promises.injectOutputFile(
-      compileDir,
-      request.rootResourcePath
-    )
-  }
-
   const compileStart = Date.now()
 
   const compileName = getCompileName(request.project_id, request.user_id)

services/clsi/app/js/DraftModeManager.js

@@ -2,11 +2,12 @@ import fsPromises from 'node:fs/promises'
 import { callbackify } from 'node:util'
 import logger from '@overleaf/logger'
 
+const PREFIX =
+  '\\PassOptionsToPackage{draft}{graphicx}\\PassOptionsToPackage{draft}{graphics}'
+
 async function injectDraftMode(filename) {
   const content = await fsPromises.readFile(filename, { encoding: 'utf8' })
-  const modifiedContent =
-    '\\PassOptionsToPackage{draft}{graphicx}\\PassOptionsToPackage{draft}{graphics}' +
-    content
+  const modifiedContent = PREFIX + content
   logger.debug(
     {
       content: content.slice(0, 1024), // \documentclass is normally v near the top
@@ -19,6 +20,7 @@ async function injectDraftMode(filename) {
 }
 
 export default {
+  PREFIX,
   injectDraftMode: callbackify(injectDraftMode),
   promises: { injectDraftMode },
 }

services/clsi/app/js/HistoryResourceWriter.js

@@ -23,6 +23,8 @@ import OError from '@overleaf/o-error'
 import ClsiMetrics from './Metrics.js'
 import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
 import Metrics from '@overleaf/metrics'
+import TikzManager from './TikzManager.js'
+import DraftModeManager from './DraftModeManager.js'
 
 const gzip = promisify(zlib.gzip)
 const gunzip = promisify(zlib.gunzip)
@@ -76,7 +78,7 @@ function isENOENT(err) {
  * @param {string} userId
  * @param {number} remoteBaseVersion
  * @param {boolean} populateClsiCache
- * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number}>}
+ * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>}
  */
 async function loadSnapshot(
   projectId,
@@ -134,7 +136,7 @@ async function loadSnapshot(
  * @param {string} projectId
  * @param {string} userId
  * @param {number} remoteBaseVersion
- * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number}>}
+ * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], fullSync: boolean,localBaseVersion: number, dirty: string[]}>}
  */
 async function loadSnapshotFromClsiCache(projectId, userId, remoteBaseVersion) {
   const { dir, resyncPath } = snapshotPath(projectId, userId)
@@ -160,20 +162,23 @@ async function loadSnapshotFromClsiCache(projectId, userId, remoteBaseVersion) {
  * @param {string} path
  * @param {number} remoteBaseVersion
  * @param {boolean} fullSync
- * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean}>}
+ * @return {Promise<{rawSnapshot: import('overleaf-editor-core/lib/types.js').RawSnapshot, globalBlobs: string[], localBaseVersion: number, fullSync: boolean, dirty: string[]}>}
  */
 async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) {
   let blob = await fs.promises.readFile(path)
   blob = await gunzip(blob)
-  const { rawSnapshot, globalBlobs, localBaseVersion } = JSON.parse(
-    blob.toString('utf-8')
-  )
+  const {
+    rawSnapshot,
+    globalBlobs,
+    localBaseVersion,
+    dirty = [], // added later, provide a default value.
+  } = JSON.parse(blob.toString('utf-8'))
   if (localBaseVersion < remoteBaseVersion) {
     throw new Errors.MissingUpdatesError('missing updates', {
       baseHistoryVersion: localBaseVersion,
     })
   }
-  return { rawSnapshot, globalBlobs, localBaseVersion, fullSync }
+  return { rawSnapshot, globalBlobs, localBaseVersion, fullSync, dirty }
 }
 
 /**
@@ -182,6 +187,7 @@ async function loadSnapshotFromFile(path, remoteBaseVersion, fullSync) {
  * @param {Snapshot} snapshot
  * @param {number} localBaseVersion
  * @param {string[]} globalBlobs
+ * @param {string[]} dirty
  * @return {Promise<void>}
  */
 async function saveSnapshot(
@@ -189,7 +195,8 @@ async function saveSnapshot(
   userId,
   snapshot,
   localBaseVersion,
-  globalBlobs
+  globalBlobs,
+  dirty
 ) {
   const { dir, path } = snapshotPath(projectId, userId)
   await fs.promises.mkdir(dir, { recursive: true })
@@ -201,6 +208,7 @@ async function saveSnapshot(
         globalBlobs,
         localBaseVersion,
         rawSnapshot: snapshot.toRaw(),
+        dirty,
       }),
       // use cheapest gzip compression level
       { level: 1 }
@@ -360,10 +368,9 @@ export async function syncResourcesToDisk(
   timings
 ) {
   const remoteBaseVersion = request.baseHistoryVersion
-  let rawSnapshot, globalBlobs, localBaseVersion, source
-  let fullSync = true
+  let rawSnapshot, globalBlobs, localBaseVersion, source, dirty, fullSync
   try {
-    ;({ rawSnapshot, globalBlobs, fullSync, localBaseVersion } =
+    ;({ rawSnapshot, globalBlobs, fullSync, localBaseVersion, dirty } =
       await loadSnapshot(
         projectId,
         userId,
@@ -391,6 +398,8 @@ export async function syncResourcesToDisk(
     localBaseVersion = remoteBaseVersion
     rawSnapshot = request.rawSnapshot
     globalBlobs = []
+    dirty = []
+    fullSync = true
   }
   globalBlobs = Array.from(new Set(globalBlobs.concat(request.globalBlobs)))
 
@@ -417,7 +426,10 @@ export async function syncResourcesToDisk(
     changedPaths.push(...snapshot.getFilePathnames())
     logger.debug({ projectId, userId }, 'compile from cache: full sync')
   } else {
-    const dedupe = new Set()
+    const dedupe = new Set(dirty)
+    if (request.draft) {
+      dedupe.add(request.rootResourcePath)
+    }
     for (const change of changes) {
       for (const operation of change.getOperations()) {
         if (operation instanceof AddFileOperation) {
@@ -462,6 +474,8 @@ export async function syncResourcesToDisk(
     await ensureHasParentFolder(compileDir, path, entriesDepthFirst)
   }
 
+  const wasDirty = dirty.length > 0
+  dirty = []
   let createCacheFolder
   // Use Promise.allSettled to ensure that all writes have stopped when we exit.
   const allDone = await promiseMapSettledWithLimit(
@@ -471,8 +485,19 @@ export async function syncResourcesToDisk(
       const file = snapshot.getFile(path)
       if (!file) return // deleted, handled by removeExtraneousEntries
 
-      const content = file.getContent({ filterTrackedDeletes: true })
+      let content = file.getContent({ filterTrackedDeletes: true })
       if (typeof content === 'string') {
+        if (path === request.rootResourcePath) {
+          if (request.draft) {
+            content = DraftModeManager.PREFIX + content
+            dirty.push(path)
+          }
+          await TikzManager.writeOutputFileIfNeeded(
+            compileDir,
+            snapshot,
+            content
+          )
+        }
         await fs.promises.writeFile(
           Path.join(compileDir, path),
           content,
@@ -514,13 +539,14 @@ export async function syncResourcesToDisk(
     throw OError.tag(result.reason, 'write failed', { path })
   }
   const baseHistoryVersion = localBaseVersion + changes.length
-  if (fullSync || changes.length) {
+  if (fullSync || changes.length || wasDirty || dirty.length) {
     await saveSnapshot(
       projectId,
       userId,
       snapshot,
       baseHistoryVersion,
-      globalBlobs
+      globalBlobs,
+      dirty
     )
   }
   if (fullSync) {

services/clsi/app/js/SafeReader.js

@@ -26,7 +26,7 @@ export default SafeReader = {
     }
     return fs.open(file, 'r', function (err, fd) {
       if (err != null && err.code === 'ENOENT') {
-        return callback()
+        return callback(null, '', 0)
       }
       if (err != null) {
         return callback(err)

services/clsi/app/js/TikzManager.js

@@ -23,13 +23,39 @@ let TikzManager
 // copy of the main file as 'output.tex'.
 
 export default TikzManager = {
+  OUTPUT_TEX: 'output.tex',
+
+  /**
+   * @param {string} content
+   * @return {boolean}
+   */
+  usesTikzExternalize(content) {
+    return content.includes('\\tikzexternalize') || content.includes('{pstool}')
+  },
+
+  /**
+   * @param {string} compileDir
+   * @param {import('overleaf-editor-core').Snapshot} snapshot
+   * @param {string} content
+   * @return {Promise<void>}
+   */
+  async writeOutputFileIfNeeded(compileDir, snapshot, content) {
+    if (snapshot.getFile(TikzManager.OUTPUT_TEX)) return
+    if (!TikzManager.usesTikzExternalize(content)) return
+    await fs.promises.writeFile(
+      Path.join(compileDir, TikzManager.OUTPUT_TEX),
+      content,
+      'utf-8'
+    )
+  },
+
   checkMainFile(compileDir, mainFile, resources, callback) {
     // if there's already an output.tex file, we don't want to touch it
     if (callback == null) {
       callback = function () {}
     }
     for (const resource of Array.from(resources)) {
-      if (resource.path === 'output.tex') {
+      if (resource.path === TikzManager.OUTPUT_TEX) {
         logger.debug(
           { compileDir, mainFile },
           'output.tex already in resources'
@@ -53,17 +79,11 @@ export default TikzManager = {
             if (error != null) {
               return callback(error)
             }
-            const usesTikzExternalize =
-              (content != null
-                ? content.indexOf('\\tikzexternalize')
-                : undefined) >= 0
-            const usesPsTool =
-              (content != null ? content.indexOf('{pstool}') : undefined) >= 0
+            const needsMainFile = TikzManager.usesTikzExternalize(content)
             logger.debug(
-              { compileDir, mainFile, usesTikzExternalize, usesPsTool },
+              { compileDir, mainFile, needsMainFile },
               'checked for packages needing main file as output.tex'
             )
-            const needsMainFile = usesTikzExternalize || usesPsTool
             return callback(null, needsMainFile)
           }
         )
@@ -92,7 +112,7 @@ export default TikzManager = {
           )
           // use wx flag to ensure that output file does not already exist
           return fs.writeFile(
-            Path.join(compileDir, 'output.tex'),
+            Path.join(compileDir, TikzManager.OUTPUT_TEX),
             content,
             { flag: 'wx' },
             callback