let CompileManager const Crypto = require('crypto') const Settings = require('@overleaf/settings') const RedisWrapper = require('../../infrastructure/RedisWrapper') const rclient = RedisWrapper.client('clsi_recently_compiled') const ProjectGetter = require('../Project/ProjectGetter') const ProjectRootDocManager = require('../Project/ProjectRootDocManager') const UserGetter = require('../User/UserGetter') const ClsiManager = require('./ClsiManager') const Metrics = require('@overleaf/metrics') const { RateLimiter } = require('../../infrastructure/RateLimiter') const UserAnalyticsIdCache = require('../Analytics/UserAnalyticsIdCache') const SplitTestHandler = require('../SplitTests/SplitTestHandler') const { callbackify, callbackifyMultiResult, } = require('@overleaf/promise-utils') function instrumentWithTimer(fn, key) { return async (...args) => { const timer = new Metrics.Timer(key) try { return await fn(...args) } finally { timer.done() } } } function generateBuildId() { return `${Date.now().toString(16)}-${Crypto.randomBytes(8).toString('hex')}` } async function compile(projectId, userId, options = {}) { const recentlyCompiled = await CompileManager._checkIfRecentlyCompiled( projectId, userId ) if (recentlyCompiled) { return { status: 'too-recently-compiled', outputFiles: [] } } try { const canCompile = await CompileManager._checkIfAutoCompileLimitHasBeenHit( options.isAutoCompile, 'everyone' ) if (!canCompile) { return { status: 'autocompile-backoff', outputFiles: [] } } } catch (error) { return { status: 'autocompile-backoff', outputFiles: [] } } await ProjectRootDocManager.promises.ensureRootDocumentIsSet(projectId) const limits = await CompileManager.promises.getProjectCompileLimits(projectId) for (const key in limits) { const value = limits[key] options[key] = value } try { const canCompile = await CompileManager._checkCompileGroupAutoCompileLimit( options.isAutoCompile, limits.compileGroup ) if (!canCompile) { return { status: 'autocompile-backoff', outputFiles: [] } } } catch (error) { return { message: 'autocompile-backoff', outputFiles: [] } } // Generate the buildId ahead of fetching the project content from redis/mongo so that the buildId's timestamp is before any lastUpdated date. options.buildId = generateBuildId() // only pass userId down to clsi if this is a per-user compile const compileAsUser = Settings.disablePerUserCompiles ? undefined : userId const { status, outputFiles, clsiServerId, validationProblems, stats, timings, outputUrlPrefix, buildId, clsiCacheShard, } = await ClsiManager.promises.sendRequest(projectId, compileAsUser, options) return { status, outputFiles, clsiServerId, limits, validationProblems, stats, timings, outputUrlPrefix, buildId, clsiCacheShard, } } const instrumentedCompile = instrumentWithTimer(compile, 'editor.compile') async function getProjectCompileLimits(projectId) { const project = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1, }) const owner = await UserGetter.promises.getUser(project.owner_ref, { _id: 1, alphaProgram: 1, analyticsId: 1, betaProgram: 1, features: 1, }) const ownerFeatures = (owner && owner.features) || {} // put alpha users into their own compile group if (owner && owner.alphaProgram) { ownerFeatures.compileGroup = 'alpha' } if (ownerFeatures.compileTimeout === 20) { const overrideCompileTimeout = await SplitTestHandler.promises.getAssignmentForUser( project.owner_ref, '10s-timeout-enforcement' ) if (overrideCompileTimeout.variant === 'enabled') { ownerFeatures.compileTimeout = 10 } } const analyticsId = await UserAnalyticsIdCache.get(owner._id) const compileGroup = ownerFeatures.compileGroup || Settings.defaultFeatures.compileGroup const limits = { timeout: ownerFeatures.compileTimeout || Settings.defaultFeatures.compileTimeout, compileGroup, compileBackendClass: compileGroup === 'standard' ? 'n2d' : 'c2d', ownerAnalyticsId: analyticsId, } return limits } async function wordCount(projectId, userId, file, clsiserverid) { const limits = await CompileManager.promises.getProjectCompileLimits(projectId) return await ClsiManager.promises.wordCount( projectId, userId, file, limits, clsiserverid ) } async function stopCompile(projectId, userId) { const limits = await CompileManager.promises.getProjectCompileLimits(projectId) return await ClsiManager.promises.stopCompile(projectId, userId, limits) } async function deleteAuxFiles(projectId, userId, clsiserverid) { const limits = await CompileManager.promises.getProjectCompileLimits(projectId) return await ClsiManager.promises.deleteAuxFiles( projectId, userId, limits, clsiserverid ) } module.exports = CompileManager = { promises: { compile: instrumentedCompile, deleteAuxFiles, getProjectCompileLimits, stopCompile, wordCount, }, compile: callbackifyMultiResult(instrumentedCompile, [ 'status', 'outputFiles', 'clsiServerId', 'limits', 'validationProblems', 'stats', 'timings', 'outputUrlPrefix', 'buildId', 'clsiCacheShard', ]), stopCompile: callbackify(stopCompile), deleteAuxFiles: callbackify(deleteAuxFiles), getProjectCompileLimits: callbackify(getProjectCompileLimits), COMPILE_DELAY: 1, // seconds async _checkIfRecentlyCompiled(projectId, userId) { const key = `compile:${projectId}:${userId}` const ok = await rclient.set(key, true, 'EX', this.COMPILE_DELAY, 'NX') return ok !== 'OK' }, async _checkCompileGroupAutoCompileLimit(isAutoCompile, compileGroup) { if (!isAutoCompile) { return true } if (compileGroup === 'standard') { // apply extra limits to the standard compile group return await CompileManager._checkIfAutoCompileLimitHasBeenHit( isAutoCompile, compileGroup ) } else { Metrics.inc(`auto-compile-${compileGroup}`) return true } }, // always allow priority group users to compile async _checkIfAutoCompileLimitHasBeenHit(isAutoCompile, compileGroup) { if (!isAutoCompile) { return true } Metrics.inc(`auto-compile-${compileGroup}`) const rateLimiter = getAutoCompileRateLimiter(compileGroup) try { await rateLimiter.consume('global', 1, { method: 'global' }) return true } catch (e) { // Don't differentiate between errors and rate limits. Silently trigger // the rate limit if there's an error consuming the points. Metrics.inc(`auto-compile-${compileGroup}-limited`) return false } }, wordCount: callbackify(wordCount), } const autoCompileRateLimiters = new Map() function getAutoCompileRateLimiter(compileGroup) { let rateLimiter = autoCompileRateLimiters.get(compileGroup) if (rateLimiter == null) { rateLimiter = new RateLimiter(`auto-compile:${compileGroup}`, { points: Settings.rateLimit.autoCompile[compileGroup] || 25, duration: 20, }) autoCompileRateLimiters.set(compileGroup, rateLimiter) } return rateLimiter }