diff --git a/package-lock.json b/package-lock.json index 688a9e9f9a..4bff2d1793 100644 --- a/package-lock.json +++ b/package-lock.json @@ -54920,6 +54920,7 @@ "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", + "@overleaf/object-persistor": "*", "@overleaf/promise-utils": "*", "@overleaf/settings": "*", "@overleaf/stream-utils": "*", @@ -54927,7 +54928,8 @@ "body-parser": "^1.20.3", "bunyan": "^1.8.15", "express": "^4.21.2", - "p-limit": "^3.1.0" + "p-limit": "^3.1.0", + "tar-fs": "^3.1.1" }, "devDependencies": { "chai": "^4.3.6", diff --git a/services/web/app/src/Features/Compile/ClsiCacheHandler.mjs b/services/web/app/src/Features/Compile/ClsiCacheHandler.mjs index 6c5f20c0d2..dc9545220e 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheHandler.mjs +++ b/services/web/app/src/Features/Compile/ClsiCacheHandler.mjs @@ -9,6 +9,7 @@ import Settings from '@overleaf/settings' import OError from '@overleaf/o-error' import { NotFoundError, InvalidNameError } from '../Errors/Errors.js' import Features from '../../infrastructure/Features.mjs' +import Path from 'node:path' const TIMEOUT = 4_000 @@ -228,8 +229,8 @@ async function getRedirectWithFallback( * @param projectId * @param userId * @param sourceProjectId - * @param templateId * @param templateVersionId + * @param imageName * @param lastUpdated * @param shard * @param signal @@ -238,8 +239,9 @@ async function getRedirectWithFallback( async function prepareCacheSource( projectId, userId, - { sourceProjectId, templateId, templateVersionId, lastUpdated, shard, signal } + { sourceProjectId, templateVersionId, imageName, lastUpdated, shard, signal } ) { + imageName = Path.basename(imageName) const url = new URL( `/project/${projectId}/user/${userId}/import-from`, Settings.apis.clsiCache.instances.find(i => i.shard === shard).url @@ -250,8 +252,8 @@ async function prepareCacheSource( json: { sourceProjectId, lastUpdated, - templateId, templateVersionId, + imageName, }, signal, }) @@ -263,6 +265,46 @@ async function prepareCacheSource( } } +/** + * Populate the clsi-cache for a template using a submission build + * + * @param clsiCacheShard + * @param submissionId + * @param buildId + * @param templateVersionId + * @param imageName + * @return {Promise} + */ +async function exportSubmissionAsTemplate( + clsiCacheShard, + submissionId, + buildId, + templateVersionId, + imageName +) { + imageName = Path.basename(imageName) + const url = new URL( + `/submission/${submissionId}/build/${buildId}/export-as-template`, + Settings.apis.clsiCache.instances.find(i => i.shard === clsiCacheShard).url + ) + try { + await fetchNothing(url, { + method: 'POST', + json: { + templateVersionId, + imageName, + }, + // clsi-cache will poll up-to 15s for the output to be copied from clsi. + signal: AbortSignal.timeout(30_000), + }) + } catch (err) { + if (err instanceof RequestFailedError && err.response.status === 404) { + throw new NotFoundError() + } + throw err + } +} + export default { TIMEOUT, getEgressLabel, @@ -270,4 +312,5 @@ export default { getOutputFile, getLatestOutputFile, prepareCacheSource, + exportSubmissionAsTemplate, } diff --git a/services/web/app/src/Features/Compile/ClsiCacheManager.mjs b/services/web/app/src/Features/Compile/ClsiCacheManager.mjs index d61645fef7..40a7b35938 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiCacheManager.mjs @@ -5,9 +5,18 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj import ProjectGetter from '../Project/ProjectGetter.mjs' import UserGetter from '../User/UserGetter.mjs' import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' import { fetchJson, RequestFailedError } from '@overleaf/fetch-utils' import Metrics from '@overleaf/metrics' import Features from '../../infrastructure/Features.mjs' +import ClsiManager from './ClsiManager.mjs' +import Crypto from 'node:crypto' +import ClsiCookieManagerFactory from './ClsiCookieManager.mjs' +import { ObjectId } from '../../infrastructure/mongodb.mjs' + +const ClsiCookieManager = ClsiCookieManagerFactory( + Settings.apis.clsi?.backendGroupName +) /** * Get the most recent build and metadata @@ -167,21 +176,24 @@ async function tryGetLatestCompileResult(projectId, userId, signal) { /** * Collect metadata and prepare the clsi-cache for the given project. * + * Returns true when downloaded; false when download failed; undefined when + * disabled for env/user; + * * @param projectId * @param userId * @param sourceProjectId - * @param templateId * @param templateVersionId - * @return {Promise} + * @param imageName + * @return {Promise} */ async function prepareClsiCache( projectId, userId, - { sourceProjectId, templateId, templateVersionId } + { sourceProjectId, templateVersionId, imageName } ) { - if (!Features.hasFeature('saas')) return + if (!Features.hasFeature('saas')) return undefined const features = await UserGetter.promises.getUserFeatures(userId) - if (features.compileGroup !== 'priority') return + if (features.compileGroup !== 'priority') return undefined const signal = AbortSignal.timeout(ClsiCacheHandler.TIMEOUT) let lastUpdated @@ -197,27 +209,106 @@ async function prepareClsiCache( signal )) } catch (err) { - if (err instanceof NotFoundError) return // nothing cached yet + if (err instanceof NotFoundError) return false // nothing cached yet throw err } } try { await ClsiCacheHandler.prepareCacheSource(projectId, userId, { sourceProjectId, - templateId, templateVersionId, + imageName, shard, lastUpdated, signal, }) } catch (err) { - if (err instanceof NotFoundError) return // nothing cached yet/expired. + if (err instanceof NotFoundError) return false // nothing cached yet/expired. throw err } + return true +} + +async function createTemplateClsiCache({ + templateVersionId, + project, + fileEntries, + docEntries, +}) { + const compileGroup = Settings.defaultFeatures.compileGroup + const compileBackendClass = Settings.apis.clsi.submissionBackendClass + const submissionId = new ObjectId().toString() + const editorId = Crypto.randomUUID() + const options = { + editorId, + compileGroup, + compileBackendClass, + timeout: 60, + syncType: 'full', + compileFromClsiCache: false, + populateClsiCache: true, + enablePdfCaching: false, + pdfCachingMinChunkSize: 0, + metricsPath: 'clsi-cache-template', + } + const req = ClsiManager._finaliseRequest( + submissionId, + options, + project, + Object.fromEntries( + docEntries.map(doc => [ + doc.path, + { _id: doc.doc._id, lines: doc.docLines.split('\n') }, + ]) + ), + Object.fromEntries(fileEntries.map(file => [file.path, file.file])) + ) + let clsiServerId = await ClsiCookieManager.promises.getServerId( + submissionId, + undefined, + compileGroup, + compileBackendClass + ) + const { imageName } = project + try { + let status, buildId, clsiCacheShard + ;({ status, buildId, clsiCacheShard, clsiServerId } = + await ClsiManager.promises.sendExternalRequest( + submissionId, + req, + options + )) + if (status !== 'success') { + logger.warn( + { status, templateVersionId, imageName }, + 'compiling template failed' + ) + return + } + if (!clsiCacheShard) { + // The circuit breaker tripped for all clsi -> clsi-cache shards. Try again later. + return + } + await ClsiCacheHandler.exportSubmissionAsTemplate( + clsiCacheShard, + submissionId, + editorId + '-' + buildId, + templateVersionId, + imageName + ) + } finally { + await ClsiManager.promises.deleteAuxFiles( + submissionId, + null, + options, + clsiServerId + ) + } } export default { getLatestBuildFromCache, getLatestCompileResult, prepareClsiCache, + createTemplateClsiCache, } diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index 02f76ece42..1ae5318aa3 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -904,7 +904,8 @@ function _finaliseRequest(projectId, options, project, docs, files) { ['alpha', 'priority'].includes(options.compileGroup) && options.compileFromClsiCache, populateClsiCache: - ['alpha', 'priority'].includes(options.compileGroup) && + (['alpha', 'priority'].includes(options.compileGroup) || + options.metricsPath === 'clsi-cache-template') && options.populateClsiCache, enablePdfCaching: (Settings.enablePdfCaching && options.enablePdfCaching) || false, @@ -1010,6 +1011,7 @@ function _getClsiServerIdFromResponse(response) { } export default { + _finaliseRequest, sendRequest: callbackifyMultiResult(sendRequest, [ 'status', 'outputFiles', diff --git a/services/web/app/src/Features/Templates/TemplatesManager.mjs b/services/web/app/src/Features/Templates/TemplatesManager.mjs index 06490aa94c..aac5732505 100644 --- a/services/web/app/src/Features/Templates/TemplatesManager.mjs +++ b/services/web/app/src/Features/Templates/TemplatesManager.mjs @@ -65,7 +65,7 @@ const TemplatesManager = { ) throw new Error(`get zip failed: ${zipReq.response.status}`) } - const project = + const { fileEntries, docEntries, project } = await ProjectUploadManager.promises.createProjectFromZipArchiveWithName( userId, projectName, @@ -76,17 +76,31 @@ const TemplatesManager = { const prepareClsiCacheInBackground = ClsiCacheManager.prepareClsiCache( project._id, userId, - { templateId, templateVersionId } + { templateVersionId, imageName } ).catch(err => { logger.warn( - { err, templateId, templateVersionId, projectId: project._id }, + { err, templateVersionId, projectId: project._id }, 'failed to prepare clsi-cache from template' ) + return undefined }) await TemplatesManager._setMainFile(project, mainFile) - await prepareClsiCacheInBackground + const found = await prepareClsiCacheInBackground + if (found === false && project.rootDoc_id) { + ClsiCacheManager.createTemplateClsiCache({ + templateVersionId, + project, + fileEntries, + docEntries, + }).catch(err => { + logger.error( + { err, templateVersionId }, + 'failed to create template clsi-cache' + ) + }) + } return project } finally { diff --git a/services/web/app/src/Features/Uploads/ProjectUploadManager.mjs b/services/web/app/src/Features/Uploads/ProjectUploadManager.mjs index 1936a0c28b..c91c67c142 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadManager.mjs +++ b/services/web/app/src/Features/Uploads/ProjectUploadManager.mjs @@ -86,12 +86,14 @@ async function createProjectFromZipArchiveWithName( ) try { - await _initializeProjectWithZipContents(ownerId, project, contentsPath) + const { fileEntries, docEntries } = + await _initializeProjectWithZipContents(ownerId, project, contentsPath) const rootDocId = await ProjectRootDocManager.promises.setRootDocAutomatically( project._id ) if (rootDocId) project.rootDoc_id = rootDocId + return { fileEntries, docEntries, project } } catch (err) { // no need to wait for the cleanup here ProjectDeleter.promises @@ -104,7 +106,6 @@ async function createProjectFromZipArchiveWithName( ) throw err } - return project } finally { await fs.promises.rm(contentsPath, { recursive: true, force: true }) } @@ -159,6 +160,7 @@ async function _initializeProjectWithZipContents( newProject: { version: projectVersion }, }) await TpdsProjectFlusher.promises.flushProjectToTpds(project._id) + return { fileEntries, docEntries } } async function _createEntriesFromImports(project, importEntries) { diff --git a/services/web/test/unit/src/Templates/TemplatesManager.test.mjs b/services/web/test/unit/src/Templates/TemplatesManager.test.mjs index 276165af57..92bf929253 100644 --- a/services/web/test/unit/src/Templates/TemplatesManager.test.mjs +++ b/services/web/test/unit/src/Templates/TemplatesManager.test.mjs @@ -37,9 +37,11 @@ describe('TemplatesManager', function () { } ctx.ProjectUploadManager = { promises: { - createProjectFromZipArchiveWithName: sinon - .stub() - .resolves({ _id: ctx.project_id }), + createProjectFromZipArchiveWithName: sinon.stub().resolves({ + project: { _id: ctx.project_id }, + fileEntries: [], + docEntries: [], + }), }, } ctx.ProjectOptionsHandler = {