From 538d04ddb8268e3b3f7300fc831cd4c9ff3c528c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 21 Aug 2026 16:33:47 +0100 Subject: [PATCH] Merge pull request #36664 from overleaf/lg-wordcount-sync-resources [clsi] sync project resources for word count GitOrigin-RevId: fe96c936e7aba22ab94416a102dcb7e4835e1256 --- services/clsi/app.js | 10 ++ services/clsi/app/js/CompileController.js | 63 ++++++++ services/clsi/app/js/CompileManager.js | 67 ++++++++- .../clsi/test/acceptance/js/WordcountTests.js | 81 +++++++++-- .../clsi/test/acceptance/js/helpers/Client.js | 14 +- .../test/unit/js/CompileController.test.js | 66 ++++++++- .../clsi/test/unit/js/CompileManager.test.js | 134 ++++++++++++++++++ .../app/src/Features/Compile/ClsiManager.mjs | 75 ++++++++-- .../Features/Compile/CompileController.mjs | 7 +- .../src/Features/Compile/CompileManager.mjs | 11 +- .../components/word-count-server.tsx | 5 +- .../acceptance/src/CompileControllerTests.mjs | 29 ++++ .../test/acceptance/src/mocks/MockClsiApi.mjs | 33 +++++ .../acceptance/src/mocks/MockV1HistoryApi.mjs | 12 +- .../unit/src/Compile/ClsiManager.test.mjs | 80 ++++++++++- .../unit/src/Compile/CompileManager.test.mjs | 17 ++- 16 files changed, 659 insertions(+), 45 deletions(-) diff --git a/services/clsi/app.js b/services/clsi/app.js index 91cb8701ce..2668eefa55 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -57,6 +57,11 @@ app.delete('/project/:project_id', CompileController.clearCache) app.get('/project/:project_id/sync/code', CompileController.syncFromCode) app.get('/project/:project_id/sync/pdf', CompileController.syncFromPdf) app.get('/project/:project_id/wordcount', CompileController.wordcount) +app.post( + '/project/:project_id/wordcount', + express.json({ limit: Settings.compileSizeLimit }), + CompileController.wordcountWithSync +) app.get('/project/:project_id/status', CompileController.status) app.post('/project/:project_id/status', CompileController.status) @@ -84,6 +89,11 @@ app.get( '/project/:project_id/user/:user_id/wordcount', CompileController.wordcount ) +app.post( + '/project/:project_id/user/:user_id/wordcount', + express.json({ limit: Settings.compileSizeLimit }), + CompileController.wordcountWithSync +) // This needs to be before GET /project/:project_id/build/:build_id/output/* app.get( diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js index 186a8b14bb..a64dc76551 100644 --- a/services/clsi/app/js/CompileController.js +++ b/services/clsi/app/js/CompileController.js @@ -401,6 +401,7 @@ function wordcount(req, res, next) { userId, file, image, + null, function (error, result) { if (error) { return next(error) @@ -412,6 +413,67 @@ function wordcount(req, res, next) { ) } +const wordcountWithSyncSchema = z.object({ + params: projectOrUserParamsSchema, + query: wordcountSchema.shape.query, + body: compileRequestBodySchema, +}) + +// Same as wordcount, but carrying the project state as a compile request body. +// texcount reads the sources from the compile dir, which only a previous +// compile on this clsi populates -- and there may not have been one, e.g. when +// the editor served the PDF from clsi-cache, which stores output files only. +function wordcountWithSync(req, res, next) { + const { params, query, body } = parseReq(req, wordcountWithSyncSchema) + const { file, image } = query + const { project_id: projectId, user_id: userId } = params + logger.debug({ image, file, projectId }, 'word count request with sync') + + RequestParser.parse(body, function (error, request) { + if (error) { + return next(error) + } + request.project_id = projectId + if (userId != null) { + request.user_id = userId + } + ProjectPersistenceManager.markProjectAsJustAccessed( + projectId, + function (error) { + if (error) { + return next(error) + } + CompileManager.wordcount( + projectId, + userId, + file, + image, + request, + function (error, result) { + if (error instanceof Errors.MissingUpdatesError) { + return res.status(409).json({ + baseHistoryVersion: error.info.baseHistoryVersion, + }) + } + if (error instanceof Errors.AlreadyCompilingError) { + return res.status(423).send('compile in progress') // Http 423 Locked + } + if (error instanceof Errors.TooManyCompileRequestsError) { + return res.status(503).send('too many concurrent requests') + } + if (error) { + return next(error) + } + res.json({ + texcount: result, + }) + } + ) + } + ) + }) +} + function status(req, res, next) { res.send('OK') } @@ -423,6 +485,7 @@ export default { syncFromCode, syncFromPdf, wordcount, + wordcountWithSync, status, timeSinceLastSuccessfulCompile, } diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 9d4aa91423..093cac535f 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -4,6 +4,7 @@ import Path from 'node:path' import { callbackify } from 'node:util' import Settings from '@overleaf/settings' import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' import OError from '@overleaf/o-error' import ResourceWriter from './ResourceWriter.js' import LatexRunner from './LatexRunner.js' @@ -569,6 +570,7 @@ async function _checkFileExists(dir, filename) { if (error.code === 'ENOENT') { throw new Errors.NotFoundError('no output file') } + throw error } if (!stats.isFile()) { throw new Error('not a file') @@ -669,7 +671,43 @@ async function _runSynctex(projectId, userId, command, opts) { ) } -async function wordcount(projectId, userId, filename, image) { +async function _syncResourcesForWordcount( + projectId, + userId, + filename, + compileDir, + request +) { + // Always write, rather than skipping when the root file is already there: + // texcount reads every included file, so they all have to be current. + const lock = LockManager.acquire(compileDir) + try { + Metrics.inc('wordcount_sync_resources') + if (request.isCompileFromHistory) { + await HistoryResourceWriter.syncResourcesToDisk( + projectId, + userId, + request, + compileDir, + {}, // timings + {} // stats + ) + } else { + await ResourceWriter.promises.syncResourcesToDisk(request, compileDir) + } + } catch (err) { + if (err instanceof Errors.MissingUpdatesError) throw err + throw OError.tag(err, 'error syncing resources for wordcount', { + projectId, + userId, + filename, + }) + } finally { + lock.release() + } +} + +async function wordcount(projectId, userId, filename, image, request) { logger.debug({ projectId, userId, filename, image }, 'running wordcount') const filePath = `$COMPILE_DIR/${filename}` const command = ['texcount', '-nocol', '-inc', filePath] @@ -682,8 +720,10 @@ async function wordcount(projectId, userId, filename, image) { throw new Errors.InvalidParameter('invalid image') } + let isNewCompileDir try { - await fsPromises.mkdir(compileDir, { recursive: true }) + isNewCompileDir = + (await fsPromises.mkdir(compileDir, { recursive: true })) === compileDir } catch (err) { throw OError.tag(err, 'error ensuring dir for wordcount', { projectId, @@ -692,6 +732,29 @@ async function wordcount(projectId, userId, filename, image) { }) } + if (isNewCompileDir && request?.compileFromClsiCache) { + // We are bootstrapping the compile dir on this clsi. Restore the cached + // outputs too, so the next compile does not start from scratch. + try { + await downloadLatestCompileCache(projectId, userId, compileDir) + } catch (err) { + logger.warn( + { err, projectId, userId }, + 'failed to populate compile dir from cache' + ) + } + } + + if (request) { + await _syncResourcesForWordcount( + projectId, + userId, + filename, + compileDir, + request + ) + } + try { const { stdout } = await CommandRunner.promises.run( compileName, diff --git a/services/clsi/test/acceptance/js/WordcountTests.js b/services/clsi/test/acceptance/js/WordcountTests.js index eabf6bd972..4fd4025ef0 100644 --- a/services/clsi/test/acceptance/js/WordcountTests.js +++ b/services/clsi/test/acceptance/js/WordcountTests.js @@ -4,6 +4,19 @@ import path from 'node:path' import fs from 'node:fs' import ClsiApp from './helpers/ClsiApp.js' +const EXPECTED_TEXCOUNT = { + encode: 'utf8', + textWords: 2281, + headWords: 2, + outside: 0, + headers: 2, + elements: 0, + mathInline: 6, + mathDisplay: 0, + errors: 0, + messages: '', +} + describe('Syncing', function () { before(async function () { this.request = { @@ -26,19 +39,63 @@ describe('Syncing', function () { it('should return wordcount info', async function () { const result = await Client.wordcount(this.project_id, 'main.tex') expect(result).to.deep.equal({ - texcount: { - encode: 'utf8', - textWords: 2281, - headWords: 2, - outside: 0, - headers: 2, - elements: 0, - mathInline: 6, - mathDisplay: 0, - errors: 0, - messages: '', - }, + texcount: EXPECTED_TEXCOUNT, }) }) }) }) + +describe('Wordcount without a compile', function () { + before(async function () { + this.request = { + rootResourcePath: 'main.tex', + resources: [ + { + path: 'main.tex', + content: fs.readFileSync( + path.join(import.meta.dirname, '../fixtures/naugty_strings.txt'), + 'utf-8' + ), + }, + { + path: 'other.tex', + content: '\\section{Hello}\nOne two three four five.\n', + }, + ], + } + this.project_id = Client.randomId() + await ClsiApp.ensureRunning() + // deliberately no Client.compile(): this clsi has never seen the project, + // as when the editor served the PDF straight from clsi-cache + }) + + it('should sync the resources and return wordcount info', async function () { + const result = await Client.wordcount( + this.project_id, + 'main.tex', + this.request + ) + expect(result).to.deep.equal({ texcount: EXPECTED_TEXCOUNT }) + }) + + it('should leave the whole project on disk for later requests', async function () { + // no request body this time: the resources synced above are still there + const result = await Client.wordcount(this.project_id, 'other.tex') + expect(result.texcount.messages).to.equal('') + expect(result.texcount.textWords).to.equal(5) + expect(result.texcount.headers).to.equal(1) + }) +}) + +describe('Wordcount without a compile or a request body', function () { + before(async function () { + this.project_id = Client.randomId() + await ClsiApp.ensureRunning() + }) + + it('should report that the file is missing', async function () { + const result = await Client.wordcount(this.project_id, 'main.tex') + expect(result.texcount.textWords).to.equal(0) + expect(result.texcount.messages).to.match(/File not found/) + }) +}) diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js index fd2b331ba1..7c00245f04 100644 --- a/services/clsi/test/acceptance/js/helpers/Client.js +++ b/services/clsi/test/acceptance/js/helpers/Client.js @@ -169,18 +169,24 @@ function syncFromPdfWithImage(projectId, page, h, v, imageName) { return fetchJson(url) } -function wordcount(projectId, file) { +function wordcount(projectId, file, compileRequest) { const image = undefined - return wordcountWithImage(projectId, file, image) + return wordcountWithImage(projectId, file, image, compileRequest) } -async function wordcountWithImage(projectId, file, image) { +async function wordcountWithImage(projectId, file, image, compileRequest) { const url = new URL(`${host}/project/${projectId}/wordcount`) if (image) { url.searchParams.append('image', image) } url.searchParams.append('file', file) - return await fetchJson(url) + if (!compileRequest) { + return await fetchJson(url) + } + return await fetchJson(url, { + method: 'POST', + json: { compile: compileRequest }, + }) } async function compileDirectory(projectId, baseDirectory, directory) { diff --git a/services/clsi/test/unit/js/CompileController.test.js b/services/clsi/test/unit/js/CompileController.test.js index 32e3e77894..c27244c6f8 100644 --- a/services/clsi/test/unit/js/CompileController.test.js +++ b/services/clsi/test/unit/js/CompileController.test.js @@ -532,13 +532,13 @@ describe('CompileController', () => { ctx.CompileManager.wordcount = sinon .stub() - .callsArgWith(4, null, (ctx.texcount = ['mock-texcount'])) + .callsArgWith(5, null, (ctx.texcount = ['mock-texcount'])) }) it('should return the word count of a file', ctx => { ctx.CompileController.wordcount(ctx.req, ctx.res, ctx.next) ctx.CompileManager.wordcount - .calledWith(ctx.project_id, undefined, ctx.file, ctx.image) + .calledWith(ctx.project_id, undefined, ctx.file, ctx.image, null) .should.equal(true) }) @@ -551,4 +551,66 @@ describe('CompileController', () => { .should.equal(true) }) }) + + describe('wordcountWithSync', () => { + beforeEach(ctx => { + ctx.file = 'main.tex' + ctx.project_id = '60a8c1d1e5e0a10123456789' + ctx.req.params = { project_id: ctx.project_id } + ctx.req.query = { + file: ctx.file, + image: (ctx.image = 'example.com/image'), + } + ctx.req.body = { + compile: { + options: {}, + rootResourcePath: ctx.file, + resources: [{ path: ctx.file, content: 'hello' }], + }, + } + ctx.res.json = sinon.stub() + ctx.res.status = sinon.stub().returns({ send: sinon.stub() }) + + ctx.request = { resources: ctx.req.body.compile.resources } + ctx.RequestParser.parse = sinon.stub().callsArgWith(1, null, ctx.request) + ctx.ProjectPersistenceManager.markProjectAsJustAccessed = sinon + .stub() + .callsArgWith(1, null) + ctx.CompileManager.wordcount = sinon + .stub() + .callsArgWith(5, null, (ctx.texcount = ['mock-texcount'])) + }) + + it('should pass the parsed request through to the compile manager', ctx => { + ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next) + ctx.CompileManager.wordcount + .calledWith(ctx.project_id, undefined, ctx.file, ctx.image, ctx.request) + .should.equal(true) + ctx.request.project_id.should.equal(ctx.project_id) + }) + + it('should mark the project as just accessed', ctx => { + ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next) + ctx.ProjectPersistenceManager.markProjectAsJustAccessed + .calledWith(ctx.project_id) + .should.equal(true) + }) + + it('should return the texcount info', ctx => { + ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next) + ctx.res.json + .calledWith({ + texcount: ctx.texcount, + }) + .should.equal(true) + }) + + it('should return a 423 when a compile is in progress', ctx => { + ctx.CompileManager.wordcount = sinon + .stub() + .callsArgWith(5, new Errors.AlreadyCompilingError('locked')) + ctx.CompileController.wordcountWithSync(ctx.req, ctx.res, ctx.next) + ctx.res.status.calledWith(423).should.equal(true) + }) + }) }) diff --git a/services/clsi/test/unit/js/CompileManager.test.js b/services/clsi/test/unit/js/CompileManager.test.js index 930d84e73c..94f668dfa0 100644 --- a/services/clsi/test/unit/js/CompileManager.test.js +++ b/services/clsi/test/unit/js/CompileManager.test.js @@ -913,5 +913,139 @@ describe('CompileManager', () => { messages: '', }) }) + + it('should not sync resources without a request', async ctx => { + expect(ctx.ResourceWriter.promises.syncResourcesToDisk).not.to.have.been + .called + expect(ctx.LockManager.acquire).not.to.have.been.called + }) + }) + + describe('wordcount with a request', () => { + beforeEach(ctx => { + ctx.filename = 'main.tex' + ctx.image = 'example.com/image' + ctx.filePath = Path.join(ctx.compileDir, ctx.filename) + ctx.request = { project_id: ctx.projectId, resources: ctx.resources } + }) + + describe('normally', () => { + beforeEach(async ctx => { + await ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image, + ctx.request + ) + }) + + it('should sync the resources to the compile dir', ctx => { + expect( + ctx.ResourceWriter.promises.syncResourcesToDisk + ).to.have.been.calledWith(ctx.request, ctx.compileDir) + }) + + it('should hold the compile dir lock while syncing', ctx => { + expect(ctx.LockManager.acquire).to.have.been.calledWith(ctx.compileDir) + expect(ctx.lock.release).to.have.been.called + }) + + it('should run the texcount command afterwards', ctx => { + expect(ctx.CommandRunner.promises.run).to.have.been.called + }) + + it('should not restore the clsi cache for an existing compile dir', ctx => { + expect(ctx.CLSICacheHandler.downloadLatestCompileCache).not.to.have.been + .called + }) + }) + + describe('when this request creates the compile dir', () => { + beforeEach(async ctx => { + // fsPromises.mkdir resolves the path only when it created the directory + ctx.fsPromises.mkdir.withArgs(ctx.compileDir).resolves(ctx.compileDir) + ctx.request.compileFromClsiCache = true + + await ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image, + ctx.request + ) + }) + + it('should restore the cached outputs before syncing', ctx => { + expect( + ctx.CLSICacheHandler.downloadLatestCompileCache + ).to.have.been.calledWith(ctx.projectId, ctx.userId, ctx.compileDir) + expect( + ctx.CLSICacheHandler.downloadLatestCompileCache + ).to.have.been.calledBefore( + ctx.ResourceWriter.promises.syncResourcesToDisk + ) + }) + + it('should still count when the cache restore fails', async ctx => { + ctx.CLSICacheHandler.downloadLatestCompileCache.rejects( + new Error('clsi-cache is down') + ) + await ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image, + ctx.request + ) + expect(ctx.CommandRunner.promises.run).to.have.been.called + }) + }) + + describe('when the request is a compile from history', () => { + beforeEach(async ctx => { + ctx.request.isCompileFromHistory = true + + await ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image, + ctx.request + ) + }) + + it('should sync via the history resource writer', ctx => { + expect( + ctx.HistoryResourceWriter.syncResourcesToDisk + ).to.have.been.calledWith( + ctx.projectId, + ctx.userId, + ctx.request, + ctx.compileDir + ) + expect(ctx.ResourceWriter.promises.syncResourcesToDisk).not.to.have.been + .called + }) + }) + + describe('when a compile is in progress', () => { + beforeEach(ctx => { + ctx.LockManager.acquire.throws(new Error('compile in progress')) + }) + + it('should not run the texcount command', async ctx => { + await expect( + ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image, + ctx.request + ) + ).to.be.rejectedWith('compile in progress') + expect(ctx.CommandRunner.promises.run).not.to.have.been.called + }) + }) }) }) diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index 7df8ea49d5..8b9a9ed6e7 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1257,9 +1257,29 @@ async function buildDocumentConversionRequest(projectId, userId, options) { }) } -async function wordCount(projectId, userId, file, limits, clsiserverid) { +async function wordCount( + projectId, + userId, + file, + limits, + clsiserverid, + { rootResourcePath, baseHistoryVersion } = {} +) { const { compileBackendClass, compileGroup } = limits - const req = await _buildRequest(null, projectId, userId, limits) + // texcount reads the sources from the compile dir on the clsi, which only a + // previous compile on that same clsi populates. Send the project state along + // with the request so the clsi can write it out itself when it has none -- + // e.g. when the editor served the PDF from clsi-cache without compiling. + const req = await _buildRequest(null, projectId, userId, { + ...limits, + compileFromHistory: true, + // let the clsi restore the cached outputs when this request is what + // bootstraps the compile dir, so the next compile is not slower for it + compileFromClsiCache: true, + rootResourcePath, + baseHistoryVersion, + metricsPath: 'wordcount', + }) const filename = file || req.compile.rootResourcePath const url = _getCompilerUrl( compileBackendClass, @@ -1271,19 +1291,46 @@ async function wordCount(projectId, userId, file, limits, clsiserverid) { url.searchParams.set('file', filename) url.searchParams.set('image', req.compile.options.imageName) - const opts = { - method: 'GET', + const requestWordCount = async opts => { + const { body } = await _makeRequestWithClsiServerId( + projectId, + userId, + compileGroup, + compileBackendClass, + url, + opts, + clsiserverid + ) + return body + } + + try { + return await requestWordCount({ method: 'POST', json: req }) + } catch (err) { + if (!(err instanceof RequestFailedError)) throw err + const { status } = err.response + if (status === 409 && baseHistoryVersion === undefined) { + // The clsi could not apply the history changes onto the snapshot it + // holds. Retry once from the version it asked for. + let retryFrom = -1 + try { + ;({ baseHistoryVersion: retryFrom } = JSON.parse(err.body)) + } catch {} + return await wordCount(projectId, userId, file, limits, clsiserverid, { + rootResourcePath, + baseHistoryVersion: retryFrom, + }) + } + if (status === 404 || status === 413 || status === 423) { + // 404: the clsi predates the POST route (deploy or rollback window). + // 413: the project is over the clsi's compileSizeLimit. + // 423: a compile holds the compile dir lock. Compiles can run for a + // while, so count what is on disk rather than failing the request. + Metrics.inc('clsi_wordcount_get_fallback', 1, { status }) + return await requestWordCount({ method: 'GET' }) + } + throw err } - const { body } = await _makeRequestWithClsiServerId( - projectId, - userId, - compileGroup, - compileBackendClass, - url, - opts, - clsiserverid - ) - return body } async function syncTeX( diff --git a/services/web/app/src/Features/Compile/CompileController.mjs b/services/web/app/src/Features/Compile/CompileController.mjs index f002a6aaab..a5f9a03cf4 100644 --- a/services/web/app/src/Features/Compile/CompileController.mjs +++ b/services/web/app/src/Features/Compile/CompileController.mjs @@ -226,6 +226,7 @@ const wordCountSchema = z.object({ query: z.object({ clsiserverid: zz.clsiServerId().optional(), file: z.string().optional(), + rootResourcePath: zz.filepath().optional(), }), }) @@ -238,6 +239,7 @@ const wordCountFallbackSchema = z.object({ query: z.object({ clsiserverid: zz.clsiServerId().optional(), file: z.string().optional(), + rootResourcePath: z.string().optional(), }), }) @@ -901,14 +903,15 @@ const _CompileController = { }) const projectId = params.Project_id const file = query.file || false - const { clsiserverid } = query + const { clsiserverid, rootResourcePath } = query const userId = CompileController._getUserIdForCompile(req) const body = await CompileManager.promises.wordCount( projectId, userId, file, - clsiserverid + clsiserverid, + rootResourcePath ) res.json(body) }, diff --git a/services/web/app/src/Features/Compile/CompileManager.mjs b/services/web/app/src/Features/Compile/CompileManager.mjs index b6cfd45b4d..f3867bcbb5 100644 --- a/services/web/app/src/Features/Compile/CompileManager.mjs +++ b/services/web/app/src/Features/Compile/CompileManager.mjs @@ -186,7 +186,13 @@ async function _getUserCompileLimits(userId) { return limits } -async function wordCount(projectId, userId, file, clsiserverid) { +async function wordCount( + projectId, + userId, + file, + clsiserverid, + rootResourcePath +) { const limits = await CompileManager.promises.getProjectCompileLimits(projectId) return await ClsiManager.promises.wordCount( @@ -194,7 +200,8 @@ async function wordCount(projectId, userId, file, clsiserverid) { userId, file, limits, - clsiserverid + clsiserverid, + { rootResourcePath } ) } diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx index 168ae2775f..0b8024886c 100644 --- a/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx @@ -3,6 +3,7 @@ import { ServerWordCountData } from '@/features/word-count-modal/components/word import { WordCountError } from '@/features/word-count-modal/components/word-count-error' import { useProjectContext } from '@/shared/context/project-context' import { useLocalCompileContext } from '@/shared/context/local-compile-context' +import { useRootDoc } from '@/shared/hooks/use-root-doc' import useAbortController from '@/shared/hooks/use-abort-controller' import { getJSON } from '@/infrastructure/fetch-json' import { debugConsole } from '@/utils/debugging' @@ -12,6 +13,7 @@ import LoadingSpinner from '@/shared/components/loading-spinner' export const WordCountServer: FC = () => { const { projectId } = useProjectContext() const { clsiServerId } = useLocalCompileContext() + const getRootDocInfo = useRootDoc() const [loading, setLoading] = useState(true) const [error, setError] = useState(false) @@ -24,6 +26,7 @@ export const WordCountServer: FC = () => { if (clsiServerId) { url.searchParams.set('clsiserverid', clsiServerId) } + url.searchParams.set('rootResourcePath', getRootDocInfo().rootResourcePath) getJSON(url.toString(), { signal }) .then(data => { @@ -36,7 +39,7 @@ export const WordCountServer: FC = () => { .finally(() => { setLoading(false) }) - }, [projectId, clsiServerId, signal]) + }, [projectId, clsiServerId, getRootDocInfo, signal]) return ( <> diff --git a/services/web/test/acceptance/src/CompileControllerTests.mjs b/services/web/test/acceptance/src/CompileControllerTests.mjs index 1295ebcb1f..880499b6a1 100644 --- a/services/web/test/acceptance/src/CompileControllerTests.mjs +++ b/services/web/test/acceptance/src/CompileControllerTests.mjs @@ -2,6 +2,8 @@ import { expect } from 'chai' import UserHelper from './helpers/User.mjs' import request from './helpers/request.js' import settings from '@overleaf/settings' +import MockClsiApi from './mocks/MockClsiApi.mjs' +import MockV1HistoryApi from './mocks/MockV1HistoryApi.mjs' import { expectValidationErrorRaw } from '@overleaf/validation-tools/testUtils.js' const User = UserHelper.promises @@ -93,6 +95,33 @@ describe('CompileController', function () { 'Project_id' ) }) + + async function requestWordCount() { + const { response, body } = await owner.doRequest('get', { + url: `/project/${projectId}/wordcount?rootResourcePath=main.tex`, + json: true, + }) + expect(response.statusCode).to.equal(200) + expect(body.texcount.textWords).to.equal(12) + return MockClsiApi.instance().lastWordcountRequestBody || {} + } + + it('should send the project history snapshot to the clsi', async function () { + const clsiRequestBody = await requestWordCount() + expect(clsiRequestBody.compile.rootResourcePath).to.equal('main.tex') + expect(clsiRequestBody.compile.options.syncType).to.equal('history-full') + expect(clsiRequestBody.compile.rawSnapshot).to.exist + }) + + it('should fall back to the resource list when history is unavailable', async function () { + MockV1HistoryApi.instance().latestHistoryUnavailable = true + const clsiRequestBody = await requestWordCount() + expect(clsiRequestBody.compile.rootResourcePath).to.equal('main.tex') + expect(clsiRequestBody.compile.rawSnapshot).not.to.exist + expect(clsiRequestBody.compile.resources.map(r => r.path)).to.include( + 'main.tex' + ) + }) }) describe('GET /download/project/:Project_id/build/:build_id/output/output.pdf', function () { diff --git a/services/web/test/acceptance/src/mocks/MockClsiApi.mjs b/services/web/test/acceptance/src/mocks/MockClsiApi.mjs index ad826679bc..bb9c4ebe26 100644 --- a/services/web/test/acceptance/src/mocks/MockClsiApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockClsiApi.mjs @@ -11,6 +11,14 @@ const compileParamsSchema = z.object({ }), }) +// The wordcount request carries the same {compile: {...}} payload as a compile +// request; the tests only assert on the resources web sent, so keep the body +// loose here and let clsi's own schema be the strict one. +const wordcountSchema = z.object({ + params: compileParamsSchema.shape.params, + body: z.object({ compile: z.looseObject({}) }), +}) + // Mirrors services/clsi/app/js/OutputController.js's createOutputZipSchema // (this mock stands in for clsi's own output.zip archive generation, as // opposed to clsi-nginx's static file serving handled by @@ -48,6 +56,25 @@ class MockClsiApi extends AbstractMockApi { }) } + static wordcount(req, res) { + const { body } = parseReq(req, wordcountSchema) + MockClsiApi.instance().lastWordcountRequestBody = body + res.json({ + texcount: { + encode: 'utf8', + textWords: 12, + headWords: 1, + outside: 0, + headers: 1, + elements: 0, + mathInline: 0, + mathDisplay: 0, + errors: 0, + messages: '', + }, + }) + } + static outputZip(req, res) { const { params } = parseReq(req, outputZipParamsSchema) zipAttachment(res, `mock-zip: ${params.build_id}`, 'output.zip') @@ -60,6 +87,12 @@ class MockClsiApi extends AbstractMockApi { MockClsiApi.compile ) + this.app.post('/project/:project_id/wordcount', MockClsiApi.wordcount) + this.app.post( + '/project/:project_id/user/:user_id/wordcount', + MockClsiApi.wordcount + ) + this.app.get('/project/:project_id/status', (req, res) => { res.sendStatus(200) }) diff --git a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs index af21be8fac..2a4ba41e8f 100644 --- a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs @@ -77,6 +77,9 @@ class MockV1HistoryApi extends AbstractMockApi { this.sentChunks = 0 this.events = new EventEmitter() this.blobs = {} + // when true, /latest/history 404s, so compile-from-history requests cannot + // be composed and web falls back to building them from mongo + this.latestHistoryUnavailable = false this.chunks = {} } @@ -265,8 +268,13 @@ class MockV1HistoryApi extends AbstractMockApi { '/api/projects/:project_id/latest/history', (req, res, next) => { const { params } = parseReq(req, latestHistoryParamsSchema) - const chunk = this.chunks[params.project_id] - if (!chunk) return res.sendStatus(404) + if (this.latestHistoryUnavailable) { + return res.sendStatus(404) + } + const chunk = this.chunks[params.project_id] || { + history: { snapshot: { files: {} }, changes: [] }, + startVersion: 0, + } res.json({ chunk }) } ) diff --git a/services/web/test/unit/src/Compile/ClsiManager.test.mjs b/services/web/test/unit/src/Compile/ClsiManager.test.mjs index 14833e9a05..d0f1c57dc6 100644 --- a/services/web/test/unit/src/Compile/ClsiManager.test.mjs +++ b/services/web/test/unit/src/Compile/ClsiManager.test.mjs @@ -1797,7 +1797,8 @@ describe('ClsiManager', function () { ctx.user_id, false, { compileBackendClass: 'free', compileGroup: 'standard' }, - 'node-1' + 'node-1', + { rootResourcePath: 'main.tex' } ) }) @@ -1811,12 +1812,83 @@ describe('ClsiManager', function () { ) }) + it('should post the project state as a history payload', function (ctx) { + expect(ctx.FetchUtils.fetchString).to.have.been.calledWith( + sinon.match.any, + sinon.match( + opts => + opts.method === 'POST' && + opts.json.compile.rawSnapshot != null && + opts.json.compile.options.syncType === 'history-full' + ) + ) + }) + it('should not persist a cookie on response', function (ctx) { expect(ctx.ClsiCookieManager.promises.setServerId).not.to.have.been .called }) }) + describe('when the clsi does not support the POST route', function () { + beforeEach(async function (ctx) { + ctx.FetchUtils.fetchString + .onFirstCall() + .rejects( + new RequestFailedError( + 'http://clsi.example.com', + { method: 'POST' }, + { status: 404 } + ) + ) + await ctx.ClsiManager.promises.wordCount( + ctx.project._id, + ctx.user_id, + false, + { compileBackendClass: 'free', compileGroup: 'standard' }, + 'node-1', + { rootResourcePath: 'main.tex' } + ) + }) + + it('should retry with a GET', function (ctx) { + expect(ctx.FetchUtils.fetchString).to.have.been.calledTwice + expect(ctx.FetchUtils.fetchString.secondCall.args[1]).to.deep.equal({ + method: 'GET', + }) + }) + }) + + describe('when a compile holds the compile dir lock', function () { + beforeEach(async function (ctx) { + ctx.FetchUtils.fetchString + .onFirstCall() + .rejects( + new RequestFailedError( + 'http://clsi.example.com', + { method: 'POST' }, + { status: 423 } + ) + ) + ctx.result = await ctx.ClsiManager.promises.wordCount( + ctx.project._id, + ctx.user_id, + false, + { compileBackendClass: 'free', compileGroup: 'standard' }, + 'node-1', + { rootResourcePath: 'main.tex' } + ) + }) + + it('should count what is on disk rather than failing', function (ctx) { + expect(ctx.FetchUtils.fetchString).to.have.been.calledTwice + expect(ctx.FetchUtils.fetchString.secondCall.args[1]).to.deep.equal({ + method: 'GET', + }) + expect(ctx.result).to.exist + }) + }) + describe('with param file', function () { beforeEach(async function (ctx) { await ctx.ClsiManager.promises.wordCount( @@ -1824,7 +1896,8 @@ describe('ClsiManager', function () { ctx.user_id, 'other.tex', { compileBackendClass: 'free', compileGroup: 'standard' }, - 'node-2' + 'node-2', + { rootResourcePath: 'main.tex' } ) }) @@ -1858,7 +1931,8 @@ describe('ClsiManager', function () { ctx.user_id, false, { compileBackendClass: 'premium', compileGroup: 'priority' }, - 'node-1' + 'node-1', + { rootResourcePath: 'main.tex' } ) // wait for the background task to finish await setTimeout(0) diff --git a/services/web/test/unit/src/Compile/CompileManager.test.mjs b/services/web/test/unit/src/Compile/CompileManager.test.mjs index 03866d8efa..122f792855 100644 --- a/services/web/test/unit/src/Compile/CompileManager.test.mjs +++ b/services/web/test/unit/src/Compile/CompileManager.test.mjs @@ -518,7 +518,9 @@ describe('CompileManager', function () { result = await ctx.CompileManager.promises.wordCount( ctx.project_id, ctx.user_id, - false + false, + undefined, + 'main.tex' ) }) @@ -534,6 +536,19 @@ describe('CompileManager', function () { .should.equal(true) }) + it('should pass the root doc path from the client through', function (ctx) { + expect(ctx.ClsiManager.promises.wordCount.lastCall.args[5]).to.deep.equal( + { + rootResourcePath: 'main.tex', + } + ) + }) + + it('should not look the root doc up itself', function (ctx) { + expect(ctx.ProjectRootDocManager.promises.ensureRootDocumentIsValid).not + .to.exist + }) + it('should resolve with the wordCount from the ClsiManager', function () { expect(result).to.equal(wordCount) })