Merge pull request #36664 from overleaf/lg-wordcount-sync-resources

[clsi] sync project resources for word count GitOrigin-RevId: fe96c936e7aba22ab94416a102dcb7e4835e1256
2026-11-15 14:43:46 +01:00 · 2026-09-17 12:03:09 +00:00
parent 285bcd4315
commit 538d04ddb8
16 changed files with 659 additions and 45 deletions
@@ -57,6 +57,11 @@ app.delete('/project/:project_id', CompileController.clearCache)
 app.get('/project/:project_id/sync/code', CompileController.syncFromCode)
 app.get('/project/:project_id/sync/pdf', CompileController.syncFromPdf)
 app.get('/project/:project_id/wordcount', CompileController.wordcount)
+app.post(
+  '/project/:project_id/wordcount',
+  express.json({ limit: Settings.compileSizeLimit }),
+  CompileController.wordcountWithSync
+)
 app.get('/project/:project_id/status', CompileController.status)
 app.post('/project/:project_id/status', CompileController.status)

@@ -84,6 +89,11 @@ app.get(
  '/project/:project_id/user/:user_id/wordcount',
  CompileController.wordcount
 )
+app.post(
+  '/project/:project_id/user/:user_id/wordcount',
+  express.json({ limit: Settings.compileSizeLimit }),
+  CompileController.wordcountWithSync
+)

 // This needs to be before GET /project/:project_id/build/:build_id/output/*
 app.get(
@@ -401,6 +401,7 @@ function wordcount(req, res, next) {
    userId,
    file,
    image,
+    null,
    function (error, result) {
      if (error) {
        return next(error)
@@ -412,6 +413,67 @@ function wordcount(req, res, next) {
  )
 }

+const wordcountWithSyncSchema = z.object({
+  params: projectOrUserParamsSchema,
+  query: wordcountSchema.shape.query,
+  body: compileRequestBodySchema,
+})
+
+// Same as wordcount, but carrying the project state as a compile request body.
+// texcount reads the sources from the compile dir, which only a previous
+// compile on this clsi populates -- and there may not have been one, e.g. when
+// the editor served the PDF from clsi-cache, which stores output files only.
+function wordcountWithSync(req, res, next) {
+  const { params, query, body } = parseReq(req, wordcountWithSyncSchema)
+  const { file, image } = query
+  const { project_id: projectId, user_id: userId } = params
+  logger.debug({ image, file, projectId }, 'word count request with sync')
+
+  RequestParser.parse(body, function (error, request) {
+    if (error) {
+      return next(error)
+    }
+    request.project_id = projectId
+    if (userId != null) {
+      request.user_id = userId
+    }
+    ProjectPersistenceManager.markProjectAsJustAccessed(
+      projectId,
+      function (error) {
+        if (error) {
+          return next(error)
+        }
+        CompileManager.wordcount(
+          projectId,
+          userId,
+          file,
+          image,
+          request,
+          function (error, result) {
+            if (error instanceof Errors.MissingUpdatesError) {
+              return res.status(409).json({
+                baseHistoryVersion: error.info.baseHistoryVersion,
+              })
+            }
+            if (error instanceof Errors.AlreadyCompilingError) {
+              return res.status(423).send('compile in progress') // Http 423 Locked
+            }
+            if (error instanceof Errors.TooManyCompileRequestsError) {
+              return res.status(503).send('too many concurrent requests')
+            }
+            if (error) {
+              return next(error)
+            }
+            res.json({
+              texcount: result,
+            })
+          }
+        )
+      }
+    )
+  })
+}
+
 function status(req, res, next) {
  res.send('OK')
 }
@@ -423,6 +485,7 @@ export default {
  syncFromCode,
  syncFromPdf,
  wordcount,
+  wordcountWithSync,
  status,
  timeSinceLastSuccessfulCompile,
 }
@@ -4,6 +4,7 @@ import Path from 'node:path'
 import { callbackify } from 'node:util'
 import Settings from '@overleaf/settings'
 import logger from '@overleaf/logger'
+import Metrics from '@overleaf/metrics'
 import OError from '@overleaf/o-error'
 import ResourceWriter from './ResourceWriter.js'
 import LatexRunner from './LatexRunner.js'
@@ -569,6 +570,7 @@ async function _checkFileExists(dir, filename) {
    if (error.code === 'ENOENT') {
      throw new Errors.NotFoundError('no output file')
    }
+    throw error
  }
  if (!stats.isFile()) {
    throw new Error('not a file')
@@ -669,7 +671,43 @@ async function _runSynctex(projectId, userId, command, opts) {
  )
 }

-async function wordcount(projectId, userId, filename, image) {
+async function _syncResourcesForWordcount(
+  projectId,
+  userId,
+  filename,
+  compileDir,
+  request
+) {
+  // Always write, rather than skipping when the root file is already there:
+  // texcount reads every included file, so they all have to be current.
+  const lock = LockManager.acquire(compileDir)
+  try {
+    Metrics.inc('wordcount_sync_resources')
+    if (request.isCompileFromHistory) {
+      await HistoryResourceWriter.syncResourcesToDisk(
+        projectId,
+        userId,
+        request,
+        compileDir,
+        {}, // timings
+        {} // stats
+      )
+    } else {
+      await ResourceWriter.promises.syncResourcesToDisk(request, compileDir)
+    }
+  } catch (err) {
+    if (err instanceof Errors.MissingUpdatesError) throw err
+    throw OError.tag(err, 'error syncing resources for wordcount', {
+      projectId,
+      userId,
+      filename,
+    })
+  } finally {
+    lock.release()
+  }
+}
+
+async function wordcount(projectId, userId, filename, image, request) {
  logger.debug({ projectId, userId, filename, image }, 'running wordcount')
  const filePath = `$COMPILE_DIR/${filename}`
  const command = ['texcount', '-nocol', '-inc', filePath]
@@ -682,8 +720,10 @@ async function wordcount(projectId, userId, filename, image) {
    throw new Errors.InvalidParameter('invalid image')
  }

+  let isNewCompileDir
  try {
-    await fsPromises.mkdir(compileDir, { recursive: true })
+    isNewCompileDir =
+      (await fsPromises.mkdir(compileDir, { recursive: true })) === compileDir
  } catch (err) {
    throw OError.tag(err, 'error ensuring dir for wordcount', {
      projectId,
@@ -692,6 +732,29 @@ async function wordcount(projectId, userId, filename, image) {
    })
  }

+  if (isNewCompileDir && request?.compileFromClsiCache) {
+    // We are bootstrapping the compile dir on this clsi. Restore the cached
+    // outputs too, so the next compile does not start from scratch.
+    try {
+      await downloadLatestCompileCache(projectId, userId, compileDir)
+    } catch (err) {
+      logger.warn(
+        { err, projectId, userId },
+        'failed to populate compile dir from cache'
+      )
+    }
+  }
+
+  if (request) {
+    await _syncResourcesForWordcount(
+      projectId,
+      userId,
+      filename,
+      compileDir,
+      request
+    )
+  }
+
  try {
    const { stdout } = await CommandRunner.promises.run(
      compileName,
@@ -4,6 +4,19 @@ import path from 'node:path'
 import fs from 'node:fs'
 import ClsiApp from './helpers/ClsiApp.js'

+const EXPECTED_TEXCOUNT = {
+  encode: 'utf8',
+  textWords: 2281,
+  headWords: 2,
+  outside: 0,
+  headers: 2,
+  elements: 0,
+  mathInline: 6,
+  mathDisplay: 0,
+  errors: 0,
+  messages: '',
+}
+
 describe('Syncing', function () {
  before(async function () {
    this.request = {
@@ -26,19 +39,63 @@ describe('Syncing', function () {
    it('should return wordcount info', async function () {
      const result = await Client.wordcount(this.project_id, 'main.tex')
      expect(result).to.deep.equal({
-        texcount: {
-          encode: 'utf8',
-          textWords: 2281,
-          headWords: 2,
-          outside: 0,
-          headers: 2,
-          elements: 0,
-          mathInline: 6,
-          mathDisplay: 0,
-          errors: 0,
-          messages: '',
-        },
+        texcount: EXPECTED_TEXCOUNT,
      })
    })
  })
 })
+
+describe('Wordcount without a compile', function () {
+  before(async function () {
+    this.request = {
+      rootResourcePath: 'main.tex',
+      resources: [
+        {
+          path: 'main.tex',
+          content: fs.readFileSync(
+            path.join(import.meta.dirname, '../fixtures/naugty_strings.txt'),
+            'utf-8'
+          ),
+        },
+        {
+          path: 'other.tex',
+          content: '\\section{Hello}\nOne two three four five.\n',
+        },
+      ],
+    }
+    this.project_id = Client.randomId()
+    await ClsiApp.ensureRunning()
+    // deliberately no Client.compile(): this clsi has never seen the project,
+    // as when the editor served the PDF straight from clsi-cache
+  })
+
+  it('should sync the resources and return wordcount info', async function () {
+    const result = await Client.wordcount(
+      this.project_id,
+      'main.tex',
+      this.request
+    )
+    expect(result).to.deep.equal({ texcount: EXPECTED_TEXCOUNT })
+  })
+
+  it('should leave the whole project on disk for later requests', async function () {
+    // no request body this time: the resources synced above are still there
+    const result = await Client.wordcount(this.project_id, 'other.tex')
+    expect(result.texcount.messages).to.equal('')
+    expect(result.texcount.textWords).to.equal(5)
+    expect(result.texcount.headers).to.equal(1)
+  })
+})
+
+describe('Wordcount without a compile or a request body', function () {
+  before(async function () {
+    this.project_id = Client.randomId()
+    await ClsiApp.ensureRunning()
+  })
+
+  it('should report that the file is missing', async function () {
+    const result = await Client.wordcount(this.project_id, 'main.tex')
+    expect(result.texcount.textWords).to.equal(0)
+    expect(result.texcount.messages).to.match(/File not found/)
+  })
+})
@@ -169,18 +169,24 @@ function syncFromPdfWithImage(projectId, page, h, v, imageName) {
  return fetchJson(url)
 }

-function wordcount(projectId, file) {
+function wordcount(projectId, file, compileRequest) {
  const image = undefined
-  return wordcountWithImage(projectId, file, image)
+  return wordcountWithImage(projectId, file, image, compileRequest)
 }

-async function wordcountWithImage(projectId, file, image) {
+async function wordcountWithImage(projectId, file, image, compileRequest) {
  const url = new URL(`${host}/project/${projectId}/wordcount`)
  if (image) {
    url.searchParams.append('image', image)
  }
  url.searchParams.append('file', file)
-  return await fetchJson(url)
+  if (!compileRequest) {
+    return await fetchJson(url)
+  }
+  return await fetchJson(url, {
+    method: 'POST',
+    json: { compile: compileRequest },
+  })
 }

 async function compileDirectory(projectId, baseDirectory, directory) {
@@ -532,13 +532,13 @@ describe('CompileController', () => {

      ctx.CompileManager.wordcount = sinon
        .stub()
-        .callsArgWith(4, null, (ctx.texcount = ['mock-texcount']))
+        .callsArgWith(5, null, (ctx.texcount = ['mock-texcount']))
    })

    it('should return the word count of a file', ctx => {
      ctx.CompileController.wordcount(ctx.req, ctx.res, ctx.next)
      ctx.CompileManager.wordcount
-        .calledWith(ctx.project_id, undefined, ctx.file, ctx.image)
+        .calledWith(ctx.project_id, undefined, ctx.file, ctx.image, null)
        .should.equal(true)
    })

@@ -551,4 +551,66 @@ describe('CompileController', () => {
        .should.equal(true)
    })
  })
+
+  describe('wordcountWithSync', () => {
+    beforeEach(ctx => {
+      ctx.file = 'main.tex'
+      ctx.project_id = '60a8c1d1e5e0a10123456789'
+      ctx.req.params = { project_id: ctx.project_id }
+      ctx.req.query = {
+        file: ctx.file,
+        image: (ctx.image = 'example.com/image'),
+      }
+      ctx.req.body = {
+        compile: {
+          options: {},
+          rootResourcePath: ctx.file,
+          resources: [{ path: ctx.file, content: 'hello' }],
+        },
+      }
+      ctx.res.json = sinon.stub()
+      ctx.res.status = sinon.stub().returns({ send: sinon.stub() })
+
+      ctx.request = { resources: ctx.req.body.compile.resources }
+      ctx.RequestParser.parse = sinon.stub().callsArgWith(1, null, ctx.request)
+      ctx.ProjectPersistenceManager.markProjectAsJustAccessed = sinon
+        .stub()
+        .callsArgWith(1, null)
+      ctx.CompileManager.wordcount = sinon
+        .stub()
+        .callsArgWith(5, null, (ctx.texcount = ['mock-texcount']))
+    })
+
+    it('should pass the parsed request through to the compile manager', ctx => {
+      ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next)
+      ctx.CompileManager.wordcount
+        .calledWith(ctx.project_id, undefined, ctx.file, ctx.image, ctx.request)
+        .should.equal(true)
+      ctx.request.project_id.should.equal(ctx.project_id)
+    })
+
+    it('should mark the project as just accessed', ctx => {
+      ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next)
+      ctx.ProjectPersistenceManager.markProjectAsJustAccessed
+        .calledWith(ctx.project_id)
+        .should.equal(true)
+    })
+
+    it('should return the texcount info', ctx => {
+      ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next)
+      ctx.res.json
+        .calledWith({
+          texcount: ctx.texcount,
+        })
+        .should.equal(true)
+    })
+
+    it('should return a 423 when a compile is in progress', ctx => {
+      ctx.CompileManager.wordcount = sinon
+        .stub()
+        .callsArgWith(5, new Errors.AlreadyCompilingError('locked'))
+      ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next)
+      ctx.res.status.calledWith(423).should.equal(true)
+    })
+  })
 })
@@ -913,5 +913,139 @@ describe('CompileManager', () => {
        messages: '',
      })
    })
+
+    it('should not sync resources without a request', async ctx => {
+      expect(ctx.ResourceWriter.promises.syncResourcesToDisk).not.to.have.been
+        .called
+      expect(ctx.LockManager.acquire).not.to.have.been.called
+    })
+  })
+
+  describe('wordcount with a request', () => {
+    beforeEach(ctx => {
+      ctx.filename = 'main.tex'
+      ctx.image = 'example.com/image'
+      ctx.filePath = Path.join(ctx.compileDir, ctx.filename)
+      ctx.request = { project_id: ctx.projectId, resources: ctx.resources }
+    })
+
+    describe('normally', () => {
+      beforeEach(async ctx => {
+        await ctx.CompileManager.promises.wordcount(
+          ctx.projectId,
+          ctx.userId,
+          ctx.filename,
+          ctx.image,
+          ctx.request
+        )
+      })
+
+      it('should sync the resources to the compile dir', ctx => {
+        expect(
+          ctx.ResourceWriter.promises.syncResourcesToDisk
+        ).to.have.been.calledWith(ctx.request, ctx.compileDir)
+      })
+
+      it('should hold the compile dir lock while syncing', ctx => {
+        expect(ctx.LockManager.acquire).to.have.been.calledWith(ctx.compileDir)
+        expect(ctx.lock.release).to.have.been.called
+      })
+
+      it('should run the texcount command afterwards', ctx => {
+        expect(ctx.CommandRunner.promises.run).to.have.been.called
+      })
+
+      it('should not restore the clsi cache for an existing compile dir', ctx => {
+        expect(ctx.CLSICacheHandler.downloadLatestCompileCache).not.to.have.been
+          .called
+      })
+    })
+
+    describe('when this request creates the compile dir', () => {
+      beforeEach(async ctx => {
+        // fsPromises.mkdir resolves the path only when it created the directory
+        ctx.fsPromises.mkdir.withArgs(ctx.compileDir).resolves(ctx.compileDir)
+        ctx.request.compileFromClsiCache = true
+
+        await ctx.CompileManager.promises.wordcount(
+          ctx.projectId,
+          ctx.userId,
+          ctx.filename,
+          ctx.image,
+          ctx.request
+        )
+      })
+
+      it('should restore the cached outputs before syncing', ctx => {
+        expect(
+          ctx.CLSICacheHandler.downloadLatestCompileCache
+        ).to.have.been.calledWith(ctx.projectId, ctx.userId, ctx.compileDir)
+        expect(
+          ctx.CLSICacheHandler.downloadLatestCompileCache
+        ).to.have.been.calledBefore(
+          ctx.ResourceWriter.promises.syncResourcesToDisk
+        )
+      })
+
+      it('should still count when the cache restore fails', async ctx => {
+        ctx.CLSICacheHandler.downloadLatestCompileCache.rejects(
+          new Error('clsi-cache is down')
+        )
+        await ctx.CompileManager.promises.wordcount(
+          ctx.projectId,
+          ctx.userId,
+          ctx.filename,
+          ctx.image,
+          ctx.request
+        )
+        expect(ctx.CommandRunner.promises.run).to.have.been.called
+      })
+    })
+
+    describe('when the request is a compile from history', () => {
+      beforeEach(async ctx => {
+        ctx.request.isCompileFromHistory = true
+
+        await ctx.CompileManager.promises.wordcount(
+          ctx.projectId,
+          ctx.userId,
+          ctx.filename,
+          ctx.image,
+          ctx.request
+        )
+      })
+
+      it('should sync via the history resource writer', ctx => {
+        expect(
+          ctx.HistoryResourceWriter.syncResourcesToDisk
+        ).to.have.been.calledWith(
+          ctx.projectId,
+          ctx.userId,
+          ctx.request,
+          ctx.compileDir
+        )
+        expect(ctx.ResourceWriter.promises.syncResourcesToDisk).not.to.have.been
+          .called
+      })
+    })
+
+    describe('when a compile is in progress', () => {
+      beforeEach(ctx => {
+        ctx.LockManager.acquire.throws(new Error('compile in progress'))
+      })
+
+      it('should not run the texcount command', async ctx => {
+        await expect(
+          ctx.CompileManager.promises.wordcount(
+            ctx.projectId,
+            ctx.userId,
+            ctx.filename,
+            ctx.image,
+            ctx.request
+          )
+        ).to.be.rejectedWith('compile in progress')
+        expect(ctx.CommandRunner.promises.run).not.to.have.been.called
+      })
+    })
  })
 })
@@ -1257,9 +1257,29 @@ async function buildDocumentConversionRequest(projectId, userId, options) {
  })
 }

-async function wordCount(projectId, userId, file, limits, clsiserverid) {
+async function wordCount(
+  projectId,
+  userId,
+  file,
+  limits,
+  clsiserverid,
+  { rootResourcePath, baseHistoryVersion } = {}
+) {
  const { compileBackendClass, compileGroup } = limits
-  const req = await _buildRequest(null, projectId, userId, limits)
+  // texcount reads the sources from the compile dir on the clsi, which only a
+  // previous compile on that same clsi populates. Send the project state along
+  // with the request so the clsi can write it out itself when it has none --
+  // e.g. when the editor served the PDF from clsi-cache without compiling.
+  const req = await _buildRequest(null, projectId, userId, {
+    ...limits,
+    compileFromHistory: true,
+    // let the clsi restore the cached outputs when this request is what
+    // bootstraps the compile dir, so the next compile is not slower for it
+    compileFromClsiCache: true,
+    rootResourcePath,
+    baseHistoryVersion,
+    metricsPath: 'wordcount',
+  })
  const filename = file || req.compile.rootResourcePath
  const url = _getCompilerUrl(
    compileBackendClass,
@@ -1271,19 +1291,46 @@ async function wordCount(projectId, userId, file, limits, clsiserverid) {
  url.searchParams.set('file', filename)
  url.searchParams.set('image', req.compile.options.imageName)

-  const opts = {
-    method: 'GET',
+  const requestWordCount = async opts => {
+    const { body } = await _makeRequestWithClsiServerId(
+      projectId,
+      userId,
+      compileGroup,
+      compileBackendClass,
+      url,
+      opts,
+      clsiserverid
+    )
+    return body
+  }
+
+  try {
+    return await requestWordCount({ method: 'POST', json: req })
+  } catch (err) {
+    if (!(err instanceof RequestFailedError)) throw err
+    const { status } = err.response
+    if (status === 409 && baseHistoryVersion === undefined) {
+      // The clsi could not apply the history changes onto the snapshot it
+      // holds. Retry once from the version it asked for.
+      let retryFrom = -1
+      try {
+        ;({ baseHistoryVersion: retryFrom } = JSON.parse(err.body))
+      } catch {}
+      return await wordCount(projectId, userId, file, limits, clsiserverid, {
+        rootResourcePath,
+        baseHistoryVersion: retryFrom,
+      })
+    }
+    if (status === 404 || status === 413 || status === 423) {
+      // 404: the clsi predates the POST route (deploy or rollback window).
+      // 413: the project is over the clsi's compileSizeLimit.
+      // 423: a compile holds the compile dir lock. Compiles can run for a
+      //      while, so count what is on disk rather than failing the request.
+      Metrics.inc('clsi_wordcount_get_fallback', 1, { status })
+      return await requestWordCount({ method: 'GET' })
+    }
+    throw err
  }
-  const { body } = await _makeRequestWithClsiServerId(
-    projectId,
-    userId,
-    compileGroup,
-    compileBackendClass,
-    url,
-    opts,
-    clsiserverid
-  )
-  return body
 }

 async function syncTeX(
@@ -226,6 +226,7 @@ const wordCountSchema = z.object({
  query: z.object({
    clsiserverid: zz.clsiServerId().optional(),
    file: z.string().optional(),
+    rootResourcePath: zz.filepath().optional(),
  }),
 })

@@ -238,6 +239,7 @@ const wordCountFallbackSchema = z.object({
  query: z.object({
    clsiserverid: zz.clsiServerId().optional(),
    file: z.string().optional(),
+    rootResourcePath: z.string().optional(),
  }),
 })

@@ -901,14 +903,15 @@ const _CompileController = {
    })
    const projectId = params.Project_id
    const file = query.file || false
-    const { clsiserverid } = query
+    const { clsiserverid, rootResourcePath } = query
    const userId = CompileController._getUserIdForCompile(req)

    const body = await CompileManager.promises.wordCount(
      projectId,
      userId,
      file,
-      clsiserverid
+      clsiserverid,
+      rootResourcePath
    )
    res.json(body)
  },
@@ -186,7 +186,13 @@ async function _getUserCompileLimits(userId) {
  return limits
 }

-async function wordCount(projectId, userId, file, clsiserverid) {
+async function wordCount(
+  projectId,
+  userId,
+  file,
+  clsiserverid,
+  rootResourcePath
+) {
  const limits =
    await CompileManager.promises.getProjectCompileLimits(projectId)
  return await ClsiManager.promises.wordCount(
@@ -194,7 +200,8 @@ async function wordCount(projectId, userId, file, clsiserverid) {
    userId,
    file,
    limits,
-    clsiserverid
+    clsiserverid,
+    { rootResourcePath }
  )
 }

@@ -3,6 +3,7 @@ import { ServerWordCountData } from '@/features/word-count-modal/components/word
 import { WordCountError } from '@/features/word-count-modal/components/word-count-error'
 import { useProjectContext } from '@/shared/context/project-context'
 import { useLocalCompileContext } from '@/shared/context/local-compile-context'
+import { useRootDoc } from '@/shared/hooks/use-root-doc'
 import useAbortController from '@/shared/hooks/use-abort-controller'
 import { getJSON } from '@/infrastructure/fetch-json'
 import { debugConsole } from '@/utils/debugging'
@@ -12,6 +13,7 @@ import LoadingSpinner from '@/shared/components/loading-spinner'
 export const WordCountServer: FC = () => {
  const { projectId } = useProjectContext()
  const { clsiServerId } = useLocalCompileContext()
+  const getRootDocInfo = useRootDoc()

  const [loading, setLoading] = useState(true)
  const [error, setError] = useState(false)
@@ -24,6 +26,7 @@ export const WordCountServer: FC = () => {
    if (clsiServerId) {
      url.searchParams.set('clsiserverid', clsiServerId)
    }
+    url.searchParams.set('rootResourcePath', getRootDocInfo().rootResourcePath)

    getJSON(url.toString(), { signal })
      .then(data => {
@@ -36,7 +39,7 @@ export const WordCountServer: FC = () => {
      .finally(() => {
        setLoading(false)
      })
-  }, [projectId, clsiServerId, signal])
+  }, [projectId, clsiServerId, getRootDocInfo, signal])

  return (
    <>
@@ -2,6 +2,8 @@ import { expect } from 'chai'
 import UserHelper from './helpers/User.mjs'
 import request from './helpers/request.js'
 import settings from '@overleaf/settings'
+import MockClsiApi from './mocks/MockClsiApi.mjs'
+import MockV1HistoryApi from './mocks/MockV1HistoryApi.mjs'
 import { expectValidationErrorRaw } from '@overleaf/validation-tools/testUtils.js'

 const User = UserHelper.promises
@@ -93,6 +95,33 @@ describe('CompileController', function () {
        'Project_id'
      )
    })
+
+    async function requestWordCount() {
+      const { response, body } = await owner.doRequest('get', {
+        url: `/project/${projectId}/wordcount?rootResourcePath=main.tex`,
+        json: true,
+      })
+      expect(response.statusCode).to.equal(200)
+      expect(body.texcount.textWords).to.equal(12)
+      return MockClsiApi.instance().lastWordcountRequestBody || {}
+    }
+
+    it('should send the project history snapshot to the clsi', async function () {
+      const clsiRequestBody = await requestWordCount()
+      expect(clsiRequestBody.compile.rootResourcePath).to.equal('main.tex')
+      expect(clsiRequestBody.compile.options.syncType).to.equal('history-full')
+      expect(clsiRequestBody.compile.rawSnapshot).to.exist
+    })
+
+    it('should fall back to the resource list when history is unavailable', async function () {
+      MockV1HistoryApi.instance().latestHistoryUnavailable = true
+      const clsiRequestBody = await requestWordCount()
+      expect(clsiRequestBody.compile.rootResourcePath).to.equal('main.tex')
+      expect(clsiRequestBody.compile.rawSnapshot).not.to.exist
+      expect(clsiRequestBody.compile.resources.map(r => r.path)).to.include(
+        'main.tex'
+      )
+    })
  })

  describe('GET /download/project/:Project_id/build/:build_id/output/output.pdf', function () {
@@ -11,6 +11,14 @@ const compileParamsSchema = z.object({
  }),
 })

+// The wordcount request carries the same {compile: {...}} payload as a compile
+// request; the tests only assert on the resources web sent, so keep the body
+// loose here and let clsi's own schema be the strict one.
+const wordcountSchema = z.object({
+  params: compileParamsSchema.shape.params,
+  body: z.object({ compile: z.looseObject({}) }),
+})
+
 // Mirrors services/clsi/app/js/OutputController.js's createOutputZipSchema
 // (this mock stands in for clsi's own output.zip archive generation, as
 // opposed to clsi-nginx's static file serving handled by
@@ -48,6 +56,25 @@ class MockClsiApi extends AbstractMockApi {
    })
  }

+  static wordcount(req, res) {
+    const { body } = parseReq(req, wordcountSchema)
+    MockClsiApi.instance().lastWordcountRequestBody = body
+    res.json({
+      texcount: {
+        encode: 'utf8',
+        textWords: 12,
+        headWords: 1,
+        outside: 0,
+        headers: 1,
+        elements: 0,
+        mathInline: 0,
+        mathDisplay: 0,
+        errors: 0,
+        messages: '',
+      },
+    })
+  }
+
  static outputZip(req, res) {
    const { params } = parseReq(req, outputZipParamsSchema)
    zipAttachment(res, `mock-zip: ${params.build_id}`, 'output.zip')
@@ -60,6 +87,12 @@ class MockClsiApi extends AbstractMockApi {
      MockClsiApi.compile
    )

+    this.app.post('/project/:project_id/wordcount', MockClsiApi.wordcount)
+    this.app.post(
+      '/project/:project_id/user/:user_id/wordcount',
+      MockClsiApi.wordcount
+    )
+
    this.app.get('/project/:project_id/status', (req, res) => {
      res.sendStatus(200)
    })
@@ -77,6 +77,9 @@ class MockV1HistoryApi extends AbstractMockApi {
    this.sentChunks = 0
    this.events = new EventEmitter()
    this.blobs = {}
+    // when true, /latest/history 404s, so compile-from-history requests cannot
+    // be composed and web falls back to building them from mongo
+    this.latestHistoryUnavailable = false
    this.chunks = {}
  }

@@ -265,8 +268,13 @@ class MockV1HistoryApi extends AbstractMockApi {
      '/api/projects/:project_id/latest/history',
      (req, res, next) => {
        const { params } = parseReq(req, latestHistoryParamsSchema)
-        const chunk = this.chunks[params.project_id]
-        if (!chunk) return res.sendStatus(404)
+        if (this.latestHistoryUnavailable) {
+          return res.sendStatus(404)
+        }
+        const chunk = this.chunks[params.project_id] || {
+          history: { snapshot: { files: {} }, changes: [] },
+          startVersion: 0,
+        }
        res.json({ chunk })
      }
    )
@@ -1797,7 +1797,8 @@ describe('ClsiManager', function () {
          ctx.user_id,
          false,
          { compileBackendClass: 'free', compileGroup: 'standard' },
-          'node-1'
+          'node-1',
+          { rootResourcePath: 'main.tex' }
        )
      })

@@ -1811,12 +1812,83 @@ describe('ClsiManager', function () {
        )
      })

+      it('should post the project state as a history payload', function (ctx) {
+        expect(ctx.FetchUtils.fetchString).to.have.been.calledWith(
+          sinon.match.any,
+          sinon.match(
+            opts =>
+              opts.method === 'POST' &&
+              opts.json.compile.rawSnapshot != null &&
+              opts.json.compile.options.syncType === 'history-full'
+          )
+        )
+      })
+
      it('should not persist a cookie on response', function (ctx) {
        expect(ctx.ClsiCookieManager.promises.setServerId).not.to.have.been
          .called
      })
    })

+    describe('when the clsi does not support the POST route', function () {
+      beforeEach(async function (ctx) {
+        ctx.FetchUtils.fetchString
+          .onFirstCall()
+          .rejects(
+            new RequestFailedError(
+              'http://clsi.example.com',
+              { method: 'POST' },
+              { status: 404 }
+            )
+          )
+        await ctx.ClsiManager.promises.wordCount(
+          ctx.project._id,
+          ctx.user_id,
+          false,
+          { compileBackendClass: 'free', compileGroup: 'standard' },
+          'node-1',
+          { rootResourcePath: 'main.tex' }
+        )
+      })
+
+      it('should retry with a GET', function (ctx) {
+        expect(ctx.FetchUtils.fetchString).to.have.been.calledTwice
+        expect(ctx.FetchUtils.fetchString.secondCall.args[1]).to.deep.equal({
+          method: 'GET',
+        })
+      })
+    })
+
+    describe('when a compile holds the compile dir lock', function () {
+      beforeEach(async function (ctx) {
+        ctx.FetchUtils.fetchString
+          .onFirstCall()
+          .rejects(
+            new RequestFailedError(
+              'http://clsi.example.com',
+              { method: 'POST' },
+              { status: 423 }
+            )
+          )
+        ctx.result = await ctx.ClsiManager.promises.wordCount(
+          ctx.project._id,
+          ctx.user_id,
+          false,
+          { compileBackendClass: 'free', compileGroup: 'standard' },
+          'node-1',
+          { rootResourcePath: 'main.tex' }
+        )
+      })
+
+      it('should count what is on disk rather than failing', function (ctx) {
+        expect(ctx.FetchUtils.fetchString).to.have.been.calledTwice
+        expect(ctx.FetchUtils.fetchString.secondCall.args[1]).to.deep.equal({
+          method: 'GET',
+        })
+        expect(ctx.result).to.exist
+      })
+    })
+
    describe('with param file', function () {
      beforeEach(async function (ctx) {
        await ctx.ClsiManager.promises.wordCount(
@@ -1824,7 +1896,8 @@ describe('ClsiManager', function () {
          ctx.user_id,
          'other.tex',
          { compileBackendClass: 'free', compileGroup: 'standard' },
-          'node-2'
+          'node-2',
+          { rootResourcePath: 'main.tex' }
        )
      })

@@ -1858,7 +1931,8 @@ describe('ClsiManager', function () {
          ctx.user_id,
          false,
          { compileBackendClass: 'premium', compileGroup: 'priority' },
-          'node-1'
+          'node-1',
+          { rootResourcePath: 'main.tex' }
        )
        // wait for the background task to finish
        await setTimeout(0)
@@ -518,7 +518,9 @@ describe('CompileManager', function () {
      result = await ctx.CompileManager.promises.wordCount(
        ctx.project_id,
        ctx.user_id,
-        false
+        false,
+        undefined,
+        'main.tex'
      )
    })

@@ -534,6 +536,19 @@ describe('CompileManager', function () {
        .should.equal(true)
    })

+    it('should pass the root doc path from the client through', function (ctx) {
+      expect(ctx.ClsiManager.promises.wordCount.lastCall.args[5]).to.deep.equal(
+        {
+          rootResourcePath: 'main.tex',
+        }
+      )
+    })
+
+    it('should not look the root doc up itself', function (ctx) {
+      expect(ctx.ProjectRootDocManager.promises.ensureRootDocumentIsValid).not
+        .to.exist
+    })
+
    it('should resolve with the wordCount from the ClsiManager', function () {
      expect(result).to.equal(wordCount)
    })