diff --git a/services/clsi/app/js/CLSICacheHandler.js b/services/clsi/app/js/CLSICacheHandler.js index de6f512987..b9415ae3ec 100644 --- a/services/clsi/app/js/CLSICacheHandler.js +++ b/services/clsi/app/js/CLSICacheHandler.js @@ -20,6 +20,19 @@ const TIMING_BUCKETS = [ 0, 10, 100, 1000, 2000, 5000, 10000, 15000, 20000, 30000, ] const MAX_ENTRIES_IN_OUTPUT_TAR = 100 +const OBJECT_ID_REGEX = /^[0-9a-f]{24}$/ + +/** + * @param {string} projectId + * @return {{shard: string, url: string}} + */ +function getShard(projectId) { + // [timestamp 4bytes][random per machine 5bytes][counter 3bytes] + // [32bit 4bytes] + const last4Bytes = Buffer.from(projectId, 'hex').subarray(8, 12) + const idx = last4Bytes.readUInt32BE() % Settings.apis.clsiCache.shards.length + return Settings.apis.clsiCache.shards[idx] +} /** * @param {string} projectId @@ -29,6 +42,7 @@ const MAX_ENTRIES_IN_OUTPUT_TAR = 100 * @param {[{path: string}]} outputFiles * @param {string} compileGroup * @param {Record} options + * @return {string | undefined} */ function notifyCLSICacheAboutBuild({ projectId, @@ -39,14 +53,16 @@ function notifyCLSICacheAboutBuild({ compileGroup, options, }) { - if (!Settings.apis.clsiCache.enabled) return + if (!Settings.apis.clsiCache.enabled) return undefined + if (!OBJECT_ID_REGEX.test(projectId)) return undefined + const { url, shard } = getShard(projectId) /** * @param {[{path: string}]} files */ const enqueue = files => { Metrics.count('clsi_cache_enqueue_files', files.length) - fetchNothing(`${Settings.apis.clsiCache.url}/enqueue`, { + fetchNothing(`${url}/enqueue`, { method: 'POST', json: { projectId, @@ -97,6 +113,8 @@ function notifyCLSICacheAboutBuild({ 'build output.tar.gz for clsi cache failed' ) }) + + return shard } /** @@ -155,6 +173,7 @@ async function downloadOutputDotSynctexFromCompileCache( outputDir ) { if (!Settings.apis.clsiCache.enabled) return false + if (!OBJECT_ID_REGEX.test(projectId)) return false const timer = new Metrics.Timer( 'clsi_cache_download', @@ -165,7 +184,7 @@ async function downloadOutputDotSynctexFromCompileCache( let stream try { stream = await fetchStream( - `${Settings.apis.clsiCache.url}/project/${projectId}/${ + `${getShard(projectId).url}/project/${projectId}/${ userId ? `user/${userId}/` : '' }build/${editorId}-${buildId}/search/output/output.synctex.gz`, { @@ -205,8 +224,9 @@ async function downloadOutputDotSynctexFromCompileCache( */ async function downloadLatestCompileCache(projectId, userId, compileDir) { if (!Settings.apis.clsiCache.enabled) return false + if (!OBJECT_ID_REGEX.test(projectId)) return false - const url = `${Settings.apis.clsiCache.url}/project/${projectId}/${ + const url = `${getShard(projectId).url}/project/${projectId}/${ userId ? `user/${userId}/` : '' }latest/output/output.tar.gz` const timer = new Metrics.Timer( diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js index 87a7db6ec2..c698ee2b75 100644 --- a/services/clsi/app/js/CompileController.js +++ b/services/clsi/app/js/CompileController.js @@ -112,12 +112,13 @@ function compile(req, res, next) { buildId = error.buildId } + let clsiCacheShard if ( status === 'success' && request.editorId && request.populateClsiCache ) { - notifyCLSICacheAboutBuild({ + clsiCacheShard = notifyCLSICacheAboutBuild({ projectId: request.project_id, userId: request.user_id, buildId: outputFiles[0].build, @@ -144,6 +145,7 @@ function compile(req, res, next) { stats, timings, buildId, + clsiCacheShard, outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix, outputFiles: outputFiles.map(file => ({ url: diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index 6f16e01a89..17042498db 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -60,9 +60,15 @@ module.exports = { }`, }, clsiCache: { - enabled: !!process.env.CLSI_CACHE_HOST, - url: `http://${process.env.CLSI_CACHE_HOST}:3044`, - downloadURL: `http://${process.env.CLSI_CACHE_NGINX_HOST || process.env.CLSI_CACHE_HOST}:8080`, + enabled: !!(process.env.CLSI_CACHE_SHARDS || process.env.CLSI_CACHE_HOST), + shards: process.env.CLSI_CACHE_SHARDS + ? JSON.parse(process.env.CLSI_CACHE_SHARDS) + : [ + { + url: `http://${process.env.CLSI_CACHE_HOST}:3044`, + shard: 'cache', + }, + ], }, }, diff --git a/services/clsi/test/unit/js/CompileControllerTests.js b/services/clsi/test/unit/js/CompileControllerTests.js index e6d21aed9f..506b5f02dd 100644 --- a/services/clsi/test/unit/js/CompileControllerTests.js +++ b/services/clsi/test/unit/js/CompileControllerTests.js @@ -129,6 +129,7 @@ describe('CompileController', function () { url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, ...file, })), + clsiCacheShard: undefined, }, }) .should.equal(true) @@ -156,6 +157,7 @@ describe('CompileController', function () { url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, ...file, })), + clsiCacheShard: undefined, }, }) .should.equal(true) @@ -203,6 +205,7 @@ describe('CompileController', function () { url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, ...file, })), + clsiCacheShard: undefined, }, }) }) @@ -250,6 +253,7 @@ describe('CompileController', function () { url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, ...file, })), + clsiCacheShard: undefined, }, }) }) @@ -281,6 +285,7 @@ describe('CompileController', function () { buildId: this.buildId, stats: this.stats, timings: this.timings, + clsiCacheShard: undefined, }, }) .should.equal(true) @@ -315,6 +320,7 @@ describe('CompileController', function () { timings: this.timings, // JSON.stringify will omit these undefined values buildId: undefined, + clsiCacheShard: undefined, }, }) .should.equal(true) @@ -348,6 +354,7 @@ describe('CompileController', function () { timings: this.timings, // JSON.stringify will omit these undefined values buildId: undefined, + clsiCacheShard: undefined, }, }) .should.equal(true) @@ -379,6 +386,7 @@ describe('CompileController', function () { timings: this.timings, // JSON.stringify will omit these undefined values buildId: undefined, + clsiCacheShard: undefined, }, }) .should.equal(true) diff --git a/services/web/app/src/Features/Compile/ClsiCacheController.js b/services/web/app/src/Features/Compile/ClsiCacheController.js index 9795fd3ef2..42f037985d 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheController.js +++ b/services/web/app/src/Features/Compile/ClsiCacheController.js @@ -110,8 +110,8 @@ async function getLatestBuildFromCache(req, res) { const userId = CompileController._getUserIdForCompile(req) try { const { - internal: { location: metaLocation, zone }, - external: { isUpToDate, allFiles }, + internal: { location: metaLocation }, + external: { isUpToDate, allFiles, zone, shard }, } = await ClsiCacheManager.getLatestBuildFromCache( projectId, userId, @@ -153,7 +153,7 @@ async function getLatestBuildFromCache(req, res) { size, editorId, }) - if (clsiServerId !== 'cache') { + if (clsiServerId !== shard) { // Enable PDF caching and attempt to download from VM first. // (clsi VMs do not have the editorId in the path on disk, omit it). Object.assign(f, { @@ -174,6 +174,7 @@ async function getLatestBuildFromCache(req, res) { outputFiles, compileGroup, clsiServerId, + clsiCacheShard: shard, pdfDownloadDomain, pdfCachingMinChunkSize, options, diff --git a/services/web/app/src/Features/Compile/ClsiCacheHandler.js b/services/web/app/src/Features/Compile/ClsiCacheHandler.js index 54ebd9e259..76b5d50f12 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheHandler.js +++ b/services/web/app/src/Features/Compile/ClsiCacheHandler.js @@ -41,7 +41,7 @@ async function clearCache(projectId, userId) { path += '/output' await Promise.all( - Settings.apis.clsiCache.instances.map(async ({ url, zone }) => { + Settings.apis.clsiCache.instances.map(async ({ url, shard }) => { const u = new URL(url) u.pathname = path try { @@ -50,7 +50,7 @@ async function clearCache(projectId, userId) { signal: AbortSignal.timeout(15_000), }) } catch (err) { - throw OError.tag(err, 'clear clsi-cache', { url, zone }) + throw OError.tag(err, 'clear clsi-cache', { url, shard }) } }) ) @@ -64,7 +64,7 @@ async function clearCache(projectId, userId) { * @param buildId * @param filename * @param signal - * @return {Promise<{size: number, zone: string, location: string, lastModified: Date, allFiles: string[]}>} + * @return {Promise<{size: number, zone: string, shard: string, location: string, lastModified: Date, allFiles: string[]}>} */ async function getOutputFile( projectId, @@ -93,7 +93,7 @@ async function getOutputFile( * @param userId * @param filename * @param signal - * @return {Promise<{size: number, zone: string, location: string, lastModified: Date, allFiles: string[]}>} + * @return {Promise<{size: number, zone: string, shard: string, location: string, lastModified: Date, allFiles: string[]}>} */ async function getLatestOutputFile( projectId, @@ -125,7 +125,7 @@ async function getLatestOutputFile( * @param userId * @param path * @param signal - * @return {Promise<{size: number, zone: string, location: string, lastModified: Date, allFiles: string[]}>} + * @return {Promise<{size: number, zone: string, shard: string, location: string, lastModified: Date, allFiles: string[]}>} */ async function getRedirectWithFallback( projectId, @@ -135,7 +135,7 @@ async function getRedirectWithFallback( ) { // Avoid hitting the same instance first all the time. const instances = _.shuffle(Settings.apis.clsiCache.instances) - for (const { url, zone } of instances) { + for (const { url, shard } of instances) { const u = new URL(url) u.pathname = path try { @@ -149,6 +149,7 @@ async function getRedirectWithFallback( return { location, zone: headers.get('X-Zone'), + shard: headers.get('X-Shard') || 'cache', lastModified: new Date(headers.get('X-Last-Modified')), size: parseInt(headers.get('X-Content-Length'), 10), allFiles: JSON.parse(headers.get('X-All-Files')), @@ -158,7 +159,7 @@ async function getRedirectWithFallback( break // No clsi-cache instance has cached something for this project/user. } logger.warn( - { err, projectId, userId, url, zone }, + { err, projectId, userId, url, shard }, 'getLatestOutputFile from clsi-cache failed' ) // This clsi-cache instance is down, try the next backend. @@ -178,18 +179,18 @@ async function getRedirectWithFallback( * @param templateId * @param templateVersionId * @param lastUpdated - * @param zone + * @param shard * @param signal * @return {Promise} */ async function prepareCacheSource( projectId, userId, - { sourceProjectId, templateId, templateVersionId, lastUpdated, zone, signal } + { sourceProjectId, templateId, templateVersionId, lastUpdated, shard, signal } ) { const url = new URL( `/project/${projectId}/user/${userId}/import-from`, - Settings.apis.clsiCache.instances.find(i => i.zone === zone).url + Settings.apis.clsiCache.instances.find(i => i.shard === shard).url ) try { await fetchNothing(url, { diff --git a/services/web/app/src/Features/Compile/ClsiCacheManager.js b/services/web/app/src/Features/Compile/ClsiCacheManager.js index 3fe4b987c5..cf0665af56 100644 --- a/services/web/app/src/Features/Compile/ClsiCacheManager.js +++ b/services/web/app/src/Features/Compile/ClsiCacheManager.js @@ -1,9 +1,11 @@ +const _ = require('lodash') const { NotFoundError } = require('../Errors/Errors') const ClsiCacheHandler = require('./ClsiCacheHandler') const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler') const ProjectGetter = require('../Project/ProjectGetter') const SplitTestHandler = require('../SplitTests/SplitTestHandler') const UserGetter = require('../User/UserGetter') +const Settings = require('@overleaf/settings') /** * Get the most recent build and metadata @@ -14,11 +16,11 @@ const UserGetter = require('../User/UserGetter') * @param userId * @param filename * @param signal - * @return {Promise<{internal: {zone: string, location: string}, external: {isUpToDate: boolean, lastUpdated: Date, size: number, allFiles: string[]}}>} + * @return {Promise<{internal: {location: string}, external: {zone: string, shard: string, isUpToDate: boolean, lastUpdated: Date, size: number, allFiles: string[]}}>} */ async function getLatestBuildFromCache(projectId, userId, filename, signal) { const [ - { location, lastModified: lastCompiled, zone, size, allFiles }, + { location, lastModified: lastCompiled, zone, shard, size, allFiles }, lastUpdatedInRedis, { lastUpdated: lastUpdatedInMongo }, ] = await Promise.all([ @@ -36,13 +38,14 @@ async function getLatestBuildFromCache(projectId, userId, filename, signal) { return { internal: { location, - zone, }, external: { isUpToDate, lastUpdated, size, allFiles, + shard, + zone, }, } } @@ -73,12 +76,11 @@ async function prepareClsiCache( const signal = AbortSignal.timeout(5_000) let lastUpdated - let zone = 'b' // populate template data on zone b + let shard = _.shuffle(Settings.apis.clsiCache.instances)[0].shard if (sourceProjectId) { try { ;({ - internal: { zone }, - external: { lastUpdated }, + external: { lastUpdated, shard }, } = await getLatestBuildFromCache( sourceProjectId, userId, @@ -95,7 +97,7 @@ async function prepareClsiCache( sourceProjectId, templateId, templateVersionId, - zone, + shard, lastUpdated, signal, }) diff --git a/services/web/app/src/Features/Compile/ClsiManager.js b/services/web/app/src/Features/Compile/ClsiManager.js index 2e32aa9622..6f11297248 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.js +++ b/services/web/app/src/Features/Compile/ClsiManager.js @@ -207,6 +207,7 @@ async function _sendBuiltRequest(projectId, userId, req, options, callback) { stats: compile.stats, timings: compile.timings, outputUrlPrefix: compile.outputUrlPrefix, + clsiCacheShard: compile.clsiCacheShard, } } @@ -853,6 +854,7 @@ module.exports = { 'timings', 'outputUrlPrefix', 'buildId', + 'clsiCacheShard', ]), sendExternalRequest: callbackifyMultiResult(sendExternalRequest, [ 'status', diff --git a/services/web/app/src/Features/Compile/CompileController.js b/services/web/app/src/Features/Compile/CompileController.js index 5d2bbcda3e..9fb9d502a4 100644 --- a/services/web/app/src/Features/Compile/CompileController.js +++ b/services/web/app/src/Features/Compile/CompileController.js @@ -192,7 +192,8 @@ module.exports = CompileController = { stats, timings, outputUrlPrefix, - buildId + buildId, + clsiCacheShard ) => { if (error) { Metrics.inc('compile-error') @@ -236,6 +237,7 @@ module.exports = CompileController = { outputFilesArchive, compileGroup: limits?.compileGroup, clsiServerId, + clsiCacheShard, validationProblems, stats, timings, diff --git a/services/web/app/src/Features/Compile/CompileManager.js b/services/web/app/src/Features/Compile/CompileManager.js index 9b5404865a..974f573815 100644 --- a/services/web/app/src/Features/Compile/CompileManager.js +++ b/services/web/app/src/Features/Compile/CompileManager.js @@ -86,6 +86,7 @@ async function compile(projectId, userId, options = {}) { timings, outputUrlPrefix, buildId, + clsiCacheShard, } = await ClsiManager.promises.sendRequest(projectId, compileAsUser, options) return { @@ -98,6 +99,7 @@ async function compile(projectId, userId, options = {}) { timings, outputUrlPrefix, buildId, + clsiCacheShard, } } @@ -184,6 +186,7 @@ module.exports = CompileManager = { 'timings', 'outputUrlPrefix', 'buildId', + 'clsiCacheShard', ]), stopCompile: callbackify(stopCompile), diff --git a/services/web/cypress/support/shared/commands/compile.ts b/services/web/cypress/support/shared/commands/compile.ts index 9f7273c403..44ee9c0805 100644 --- a/services/web/cypress/support/shared/commands/compile.ts +++ b/services/web/cypress/support/shared/commands/compile.ts @@ -48,6 +48,7 @@ const compileFromCacheResponse = () => { fromCache: true, status: 'success', clsiServerId: 'foo', + clsiCacheShard: 'clsi-cache-zone-b-shard-1', compileGroup: 'priority', pdfDownloadDomain: 'https://clsi.test-overleaf.com', outputFiles: outputFiles(), @@ -166,10 +167,10 @@ export const waitForCompileOutput = ({ } = {}) => { cy.wait(`@${prefix}-log`) .its('request.query.clsiserverid') - .should('eq', cached ? 'cache' : 'foo') // straight from cache if cached + .should('eq', cached ? 'clsi-cache-zone-b-shard-1' : 'foo') // straight from cache if cached cy.wait(`@${prefix}-blg`) .its('request.query.clsiserverid') - .should('eq', cached ? 'cache' : 'foo') // straight from cache if cached + .should('eq', cached ? 'clsi-cache-zone-b-shard-1' : 'foo') // straight from cache if cached if (pdf) { cy.wait(`@${prefix}-pdf`) .its('request.query.clsiserverid') diff --git a/services/web/frontend/js/features/pdf-preview/util/file-list.ts b/services/web/frontend/js/features/pdf-preview/util/file-list.ts index 310fbb55fb..a8a37a9e2b 100644 --- a/services/web/frontend/js/features/pdf-preview/util/file-list.ts +++ b/services/web/frontend/js/features/pdf-preview/util/file-list.ts @@ -13,6 +13,7 @@ export function buildFileList( outputFiles: Map, { clsiServerId, + clsiCacheShard, compileGroup, outputFilesArchive, fromCache = false, @@ -24,7 +25,7 @@ export function buildFileList( const params = new URLSearchParams() if (fromCache) { - params.set('clsiserverid', 'cache') + params.set('clsiserverid', clsiCacheShard || 'cache') } else if (clsiServerId) { params.set('clsiserverid', clsiServerId) } diff --git a/services/web/frontend/js/features/pdf-preview/util/output-files.js b/services/web/frontend/js/features/pdf-preview/util/output-files.js index 6c93a02368..3ee0dc1180 100644 --- a/services/web/frontend/js/features/pdf-preview/util/output-files.js +++ b/services/web/frontend/js/features/pdf-preview/util/output-files.js @@ -17,6 +17,7 @@ export function handleOutputFiles(outputFiles, projectId, data) { if (!outputFile) return null outputFile.editorId = outputFile.editorId || EDITOR_SESSION_ID + outputFile.clsiCacheShard = data.clsiCacheShard || 'cache' // build the URL for viewing the PDF in the preview UI const params = new URLSearchParams() diff --git a/services/web/frontend/js/features/pdf-preview/util/pdf-caching-transport.js b/services/web/frontend/js/features/pdf-preview/util/pdf-caching-transport.js index 4497b57398..f568c634a4 100644 --- a/services/web/frontend/js/features/pdf-preview/util/pdf-caching-transport.js +++ b/services/web/frontend/js/features/pdf-preview/util/pdf-caching-transport.js @@ -116,7 +116,9 @@ export function generatePdfCachingTransportFactory() { return ( u.pathname.endsWith( `build/${this.pdfFile.editorId}-${this.pdfFile.build}/output/output.pdf` - ) && u.searchParams.get('clsiserverid') === 'cache' + ) && + (u.searchParams.get('clsiserverid') === 'cache' || + u.searchParams.get('clsiserverid')?.startsWith('clsi-cache-')) ) } const canTryFromCache = err => { @@ -127,7 +129,7 @@ export function generatePdfCachingTransportFactory() { const getOutputPDFURLFromCache = () => { if (usesCache(this.url)) return this.url const u = new URL(this.url) - u.searchParams.set('clsiserverid', 'cache') + u.searchParams.set('clsiserverid', this.pdfFile.clsiCacheShard) u.pathname = u.pathname.replace( /build\/[a-f0-9-]+\//, `build/${this.pdfFile.editorId}-${this.pdfFile.build}/` diff --git a/services/web/types/compile.ts b/services/web/types/compile.ts index 541d03149c..3038893529 100644 --- a/services/web/types/compile.ts +++ b/services/web/types/compile.ts @@ -23,6 +23,7 @@ export type CompileResponseData = { outputFiles: CompileOutputFile[] compileGroup?: string clsiServerId?: string + clsiCacheShard?: string pdfDownloadDomain?: string pdfCachingMinChunkSize: number validationProblems: any