From 33a8a90134bbb08f736302642a08cd58c46ff7f2 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween <5454374+emcsween@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:42:11 +0100 Subject: [PATCH] Merge pull request #13510 from overleaf/em-promisify-clsi-manager Clean up of ClsiManager and ClsiCookieManager GitOrigin-RevId: e5047b253613e87fd6cb4f12855b821028fcaf8e --- .../src/Features/Compile/ClsiCookieManager.js | 73 +- .../app/src/Features/Compile/ClsiManager.js | 1853 ++++++++--------- services/web/app/src/router.js | 2 +- .../src/Compile/ClsiCookieManagerTests.js | 113 +- .../test/unit/src/Compile/ClsiManagerTests.js | 1770 +++++++--------- 5 files changed, 1723 insertions(+), 2088 deletions(-) diff --git a/services/web/app/src/Features/Compile/ClsiCookieManager.js b/services/web/app/src/Features/Compile/ClsiCookieManager.js index a5f0bc3fd8..701fa15acf 100644 --- a/services/web/app/src/Features/Compile/ClsiCookieManager.js +++ b/services/web/app/src/Features/Compile/ClsiCookieManager.js @@ -1,32 +1,19 @@ -/* eslint-disable - n/handle-callback-err, - max-len, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let rclientSecondary const { URL, URLSearchParams } = require('url') const OError = require('@overleaf/o-error') const Settings = require('@overleaf/settings') const request = require('request').defaults({ timeout: 30 * 1000 }) const RedisWrapper = require('../../infrastructure/RedisWrapper') -const rclient = RedisWrapper.client('clsi_cookie') -if (Settings.redis.clsi_cookie_secondary != null) { - rclientSecondary = RedisWrapper.client('clsi_cookie_secondary') -} const Cookie = require('cookie') const logger = require('@overleaf/logger') const Metrics = require('@overleaf/metrics') -const clsiCookiesEnabled = - (Settings.clsiCookie != null ? Settings.clsiCookie.key : undefined) != null && - Settings.clsiCookie.key.length !== 0 +const clsiCookiesEnabled = (Settings.clsiCookie?.key ?? '') !== '' + +const rclient = RedisWrapper.client('clsi_cookie') +let rclientSecondary +if (Settings.redis.clsi_cookie_secondary != null) { + rclientSecondary = RedisWrapper.client('clsi_cookie_secondary') +} module.exports = function (backendGroup) { return { @@ -45,15 +32,12 @@ module.exports = function (backendGroup) { compileBackendClass, callback ) { - if (callback == null) { - callback = function () {} - } - return rclient.get(this.buildKey(projectId, userId), (err, serverId) => { - if (err != null) { + rclient.get(this.buildKey(projectId, userId), (err, serverId) => { + if (err) { return callback(err) } if (serverId == null || serverId === '') { - return this._populateServerIdViaRequest( + this._populateServerIdViaRequest( projectId, userId, compileGroup, @@ -61,7 +45,7 @@ module.exports = function (backendGroup) { callback ) } else { - return callback(null, serverId) + callback(null, serverId) } }) }, @@ -73,16 +57,13 @@ module.exports = function (backendGroup) { compileBackendClass, callback ) { - if (callback == null) { - callback = function () {} - } const u = new URL(`${Settings.apis.clsi.url}/project/${projectId}/status`) u.search = new URLSearchParams({ compileGroup, compileBackendClass, }).toString() request.post(u.href, (err, res, body) => { - if (err != null) { + if (err) { OError.tag(err, 'error getting initial server id for project', { project_id: projectId, }) @@ -96,25 +77,21 @@ module.exports = function (backendGroup) { res, null, function (err, serverId) { - if (err != null) { + if (err) { logger.warn( { err, projectId }, 'error setting server id via populate request' ) } - return callback(err, serverId) + callback(err, serverId) } ) }) }, _parseServerIdFromResponse(response) { - const cookies = Cookie.parse( - (response.headers['set-cookie'] != null - ? response.headers['set-cookie'][0] - : undefined) || '' - ) - return cookies != null ? cookies[Settings.clsiCookie.key] : undefined + const cookies = Cookie.parse(response.headers['set-cookie']?.[0] || '') + return cookies?.[Settings.clsiCookie.key] }, checkIsLoadSheddingEvent(clsiserverid, compileGroup, compileBackendClass) { @@ -155,9 +132,6 @@ module.exports = function (backendGroup) { previous, callback ) { - if (callback == null) { - callback = function () {} - } if (!clsiCookiesEnabled) { return callback() } @@ -195,9 +169,6 @@ module.exports = function (backendGroup) { }, _setServerIdInRedis(rclient, projectId, userId, serverId, callback) { - if (callback == null) { - callback = function () {} - } rclient.setex( this.buildKey(projectId, userId), this._getTTLInSeconds(serverId), @@ -207,13 +178,10 @@ module.exports = function (backendGroup) { }, clearServerId(projectId, userId, callback) { - if (callback == null) { - callback = function () {} - } if (!clsiCookiesEnabled) { return callback() } - return rclient.del(this.buildKey(projectId, userId), callback) + rclient.del(this.buildKey(projectId, userId), callback) }, getCookieJar( @@ -223,13 +191,10 @@ module.exports = function (backendGroup) { compileBackendClass, callback ) { - if (callback == null) { - callback = function () {} - } if (!clsiCookiesEnabled) { return callback(null, request.jar(), undefined) } - return this._getServerId( + this._getServerId( projectId, userId, compileGroup, @@ -246,7 +211,7 @@ module.exports = function (backendGroup) { ) const jar = request.jar() jar.setCookie(serverCookie, Settings.apis.clsi.url) - return callback(null, jar, serverId) + callback(null, jar, serverId) } ) }, diff --git a/services/web/app/src/Features/Compile/ClsiManager.js b/services/web/app/src/Features/Compile/ClsiManager.js index 5c19b9a4c4..fbd579be78 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.js +++ b/services/web/app/src/Features/Compile/ClsiManager.js @@ -8,12 +8,10 @@ const { URL, URLSearchParams } = require('url') const OError = require('@overleaf/o-error') const ClsiCookieManager = require('./ClsiCookieManager')( - Settings.apis.clsi != null ? Settings.apis.clsi.backendGroupName : undefined + Settings.apis.clsi?.backendGroupName ) const NewBackendCloudClsiCookieManager = require('./ClsiCookieManager')( - Settings.apis.clsi_new != null - ? Settings.apis.clsi_new.backendGroupName - : undefined + Settings.apis.clsi_new?.backendGroupName ) const ClsiStateManager = require('./ClsiStateManager') const _ = require('underscore') @@ -40,305 +38,110 @@ function collectMetricsOnBlgFiles(outputFiles) { Metrics.count('blg_output_file', nested, 1, { path: 'nested' }) } -const ClsiManager = { - sendRequest(projectId, userId, options, callback) { - if (options == null) { - options = {} +function sendRequest(projectId, userId, options, callback) { + if (options == null) { + options = {} + } + sendRequestOnce(projectId, userId, options, (err, status, ...result) => { + if (err != null) { + return callback(err) } - ClsiManager.sendRequestOnce( - projectId, - userId, - options, - (err, status, ...result) => { - if (err != null) { - return callback(err) - } - if (status === 'conflict') { - // Try again, with a full compile - return ClsiManager.sendRequestOnce( - projectId, - userId, - { ...options, syncType: 'full' }, - callback - ) - } else if (status === 'unavailable') { - return ClsiManager.sendRequestOnce( - projectId, - userId, - { ...options, syncType: 'full', forceNewClsiServer: true }, - callback - ) - } - callback(null, status, ...result) - } - ) - }, - - sendRequestOnce(projectId, userId, options, callback) { - if (options == null) { - options = {} - } - ClsiManager._buildRequest(projectId, options, (err, req) => { - if (err != null) { - if (err.message === 'no main file specified') { - return callback(null, 'validation-problems', null, null, { - mainFile: err.message, - }) - } else { - return callback( - OError.tag(err, 'Could not build request to CLSI', { - projectId, - options, - }) - ) - } - } - ClsiManager._sendBuiltRequest( + if (status === 'conflict') { + // Try again, with a full compile + return sendRequestOnce( projectId, userId, - req, - options, - (err, status, ...result) => { - if (err != null) { - return callback( - OError.tag(err, 'CLSI compile failed', { projectId, userId }) - ) - } - callback(null, status, ...result) - } + { ...options, syncType: 'full' }, + callback ) - }) - }, - - // for public API requests where there is no project id - sendExternalRequest(submissionId, clsiRequest, options, callback) { - if (options == null) { - options = {} - } - ClsiManager._sendBuiltRequest( - submissionId, - null, - clsiRequest, - options, - (err, status, ...result) => { - if (err != null) { - return callback( - OError.tag(err, 'CLSI compile failed', { - submissionId, - options, - }) - ) - } - callback(null, status, ...result) - } - ) - }, - - stopCompile(projectId, userId, options, callback) { - if (options == null) { - options = {} - } - const { compileBackendClass, compileGroup } = options - const compilerUrl = this._getCompilerUrl( - compileBackendClass, - compileGroup, - projectId, - userId, - 'compile/stop' - ) - const opts = { - url: compilerUrl, - method: 'POST', - } - ClsiManager._makeRequest( - projectId, - userId, - compileGroup, - compileBackendClass, - opts, - callback - ) - }, - - deleteAuxFiles(projectId, userId, options, clsiserverid, callback) { - if (options == null) { - options = {} - } - const { compileBackendClass, compileGroup } = options - const compilerUrl = this._getCompilerUrl( - compileBackendClass, - compileGroup, - projectId, - userId - ) - const opts = { - url: compilerUrl, - method: 'DELETE', - } - ClsiManager._makeRequestWithClsiServerId( - projectId, - userId, - compileGroup, - compileBackendClass, - opts, - clsiserverid, - clsiErr => { - // always clear the project state from the docupdater, even if there - // was a problem with the request to the clsi - DocumentUpdaterHandler.clearProjectState(projectId, docUpdaterErr => { - ClsiCookieManager.clearServerId(projectId, userId, redisError => { - if (clsiErr) { - return callback( - OError.tag(clsiErr, 'Failed to delete aux files', { projectId }) - ) - } - if (docUpdaterErr) { - return callback( - OError.tag( - docUpdaterErr, - 'Failed to clear project state in doc updater', - { projectId } - ) - ) - } - if (redisError) { - // redis errors need wrapping as the instance may be shared - return callback( - OError( - 'Failed to clear clsi persistence', - { projectId }, - redisError - ) - ) - } - callback() - }) - }) - } - ) - }, - - _sendBuiltRequest(projectId, userId, req, options, callback) { - if (options == null) { - options = {} - } - if (options.forceNewClsiServer) { - // Clear clsi cookie, then try again - return ClsiCookieManager.clearServerId(projectId, userId, err => { - if (err) { - return callback(err) - } - options.forceNewClsiServer = false // backend has now been reset - return ClsiManager._sendBuiltRequest( - projectId, - userId, - req, - options, - callback - ) - }) - } - ClsiFormatChecker.checkRecoursesForProblems( - req.compile != null ? req.compile.resources : undefined, - (err, validationProblems) => { - if (err != null) { - return callback( - OError.tag( - err, - 'could not check resources for potential problems before sending to clsi' - ) - ) - } - if (validationProblems != null) { - logger.debug( - { projectId, validationProblems }, - 'problems with users latex before compile was attempted' - ) - return callback( - null, - 'validation-problems', - null, - null, - validationProblems - ) - } - ClsiManager._postToClsi( - projectId, - userId, - req, - options.compileBackendClass, - options.compileGroup, - (err, response, clsiServerId) => { - if (err != null) { - return callback( - OError.tag(err, 'error sending request to clsi', { - projectId, - userId, - }) - ) - } - const outputFiles = ClsiManager._parseOutputFiles( - projectId, - response && response.compile && response.compile.outputFiles - ) - collectMetricsOnBlgFiles(outputFiles) - const compile = (response && response.compile) || {} - const status = compile.status - const stats = compile.stats - const timings = compile.timings - const outputUrlPrefix = compile.outputUrlPrefix - const validationProblems = undefined - callback( - null, - status, - outputFiles, - clsiServerId, - validationProblems, - stats, - timings, - outputUrlPrefix - ) - } - ) - } - ) - }, - - _makeRequestWithClsiServerId( - projectId, - userId, - compileGroup, - compileBackendClass, - opts, - clsiserverid, - callback - ) { - if (clsiserverid) { - // ignore cookies and newBackend, go straight to the clsi node - opts.qs = Object.assign( - { compileGroup, compileBackendClass, clsiserverid }, - opts.qs - ) - request(opts, (err, response, body) => { - if (err) { - return callback( - OError.tag(err, 'error making request to CLSI', { projectId }) - ) - } - callback(null, response, body) - }) - } else { - ClsiManager._makeRequest( + } else if (status === 'unavailable') { + return sendRequestOnce( projectId, userId, - compileGroup, - compileBackendClass, - opts, + { ...options, syncType: 'full', forceNewClsiServer: true }, callback ) } - }, + callback(null, status, ...result) + }) +} +function sendRequestOnce(projectId, userId, options, callback) { + if (options == null) { + options = {} + } + _buildRequest(projectId, options, (err, req) => { + if (err != null) { + if (err.message === 'no main file specified') { + return callback(null, 'validation-problems', null, null, { + mainFile: err.message, + }) + } else { + return callback( + OError.tag(err, 'Could not build request to CLSI', { + projectId, + options, + }) + ) + } + } + _sendBuiltRequest( + projectId, + userId, + req, + options, + (err, status, ...result) => { + if (err != null) { + return callback( + OError.tag(err, 'CLSI compile failed', { projectId, userId }) + ) + } + callback(null, status, ...result) + } + ) + }) +} + +// for public API requests where there is no project id +function sendExternalRequest(submissionId, clsiRequest, options, callback) { + if (options == null) { + options = {} + } + _sendBuiltRequest( + submissionId, + null, + clsiRequest, + options, + (err, status, ...result) => { + if (err != null) { + return callback( + OError.tag(err, 'CLSI compile failed', { + submissionId, + options, + }) + ) + } + callback(null, status, ...result) + } + ) +} + +function stopCompile(projectId, userId, options, callback) { + if (options == null) { + options = {} + } + const { compileBackendClass, compileGroup } = options + const compilerUrl = _getCompilerUrl( + compileBackendClass, + compileGroup, + projectId, + userId, + 'compile/stop' + ) + const opts = { + url: compilerUrl, + method: 'POST', + } _makeRequest( projectId, userId, @@ -346,272 +149,843 @@ const ClsiManager = { compileBackendClass, opts, callback - ) { - async.series( - { - currentBackend(cb) { - const startTime = new Date() - ClsiCookieManager.getCookieJar( - projectId, - userId, - compileGroup, - compileBackendClass, - (err, jar, clsiServerId) => { - if (err != null) { - return callback( - OError.tag(err, 'error getting cookie jar for CLSI request', { - projectId, - }) - ) - } - opts.jar = jar - const timer = new Metrics.Timer('compile.currentBackend') - request(opts, (err, response, body) => { - if (err != null) { - return callback( - OError.tag(err, 'error making request to CLSI', { - projectId, - }) - ) - } - timer.done() - Metrics.inc( - `compile.currentBackend.response.${response.statusCode}` - ) - ClsiCookieManager.setServerId( - projectId, - userId, - compileGroup, - compileBackendClass, - response, - clsiServerId, - (err, newClsiServerId) => { - if (err != null) { - callback( - OError.tag(err, 'error setting server id', { - projectId, - }) - ) - } else { - // return as soon as the standard compile has returned - callback( - null, - response, - body, - newClsiServerId || clsiServerId - ) - } - cb(err, { - response, - body, - finishTime: new Date() - startTime, - }) - } - ) - }) - } - ) - }, - newBackend(cb) { - const startTime = new Date() - ClsiManager._makeNewBackendRequest( - projectId, - userId, - compileGroup, - compileBackendClass, - opts, - (err, response, body) => { - if (err != null) { - logger.warn({ err }, 'Error making request to new CLSI backend') - } - if (response != null) { - Metrics.inc( - `compile.newBackend.response.${response.statusCode}` - ) - } - cb(err, { - response, - body, - finishTime: new Date() - startTime, - }) - } - ) - }, - }, - (err, results) => { - if (err != null) { - // This was handled higher up - return - } - if (results.newBackend != null && results.newBackend.response != null) { - const currentStatusCode = results.currentBackend.response.statusCode - const newStatusCode = results.newBackend.response.statusCode - const statusCodeSame = newStatusCode === currentStatusCode - const currentCompileTime = results.currentBackend.finishTime - const newBackendCompileTime = results.newBackend.finishTime || 0 - const timeDifference = newBackendCompileTime - currentCompileTime - logger.debug( - { - statusCodeSame, - timeDifference, - currentCompileTime, - newBackendCompileTime, - projectId, - }, - 'both clsi requests returned' - ) - } - } - ) - }, + ) +} - _makeNewBackendRequest( +function deleteAuxFiles(projectId, userId, options, clsiserverid, callback) { + if (options == null) { + options = {} + } + const { compileBackendClass, compileGroup } = options + const compilerUrl = _getCompilerUrl( + compileBackendClass, + compileGroup, + projectId, + userId + ) + const opts = { + url: compilerUrl, + method: 'DELETE', + } + _makeRequestWithClsiServerId( projectId, userId, compileGroup, compileBackendClass, - baseOpts, - callback - ) { - if (Settings.apis.clsi_new == null || Settings.apis.clsi_new.url == null) { - return callback() + opts, + clsiserverid, + clsiErr => { + // always clear the project state from the docupdater, even if there + // was a problem with the request to the clsi + DocumentUpdaterHandler.clearProjectState(projectId, docUpdaterErr => { + ClsiCookieManager.clearServerId(projectId, userId, redisError => { + if (clsiErr) { + return callback( + OError.tag(clsiErr, 'Failed to delete aux files', { projectId }) + ) + } + if (docUpdaterErr) { + return callback( + OError.tag( + docUpdaterErr, + 'Failed to clear project state in doc updater', + { projectId } + ) + ) + } + if (redisError) { + // redis errors need wrapping as the instance may be shared + return callback( + OError( + 'Failed to clear clsi persistence', + { projectId }, + redisError + ) + ) + } + callback() + }) + }) } - const opts = { - ...baseOpts, - url: baseOpts.url.replace( - Settings.apis.clsi.url, - Settings.apis.clsi_new.url - ), - } - NewBackendCloudClsiCookieManager.getCookieJar( - projectId, - userId, - compileGroup, - compileBackendClass, - (err, jar, clsiServerId) => { - if (err != null) { - return callback( - OError.tag(err, 'error getting cookie jar for CLSI request', { - projectId, - }) + ) +} + +function _sendBuiltRequest(projectId, userId, req, options, callback) { + if (options == null) { + options = {} + } + if (options.forceNewClsiServer) { + // Clear clsi cookie, then try again + return ClsiCookieManager.clearServerId(projectId, userId, err => { + if (err) { + return callback(err) + } + options.forceNewClsiServer = false // backend has now been reset + _sendBuiltRequest(projectId, userId, req, options, callback) + }) + } + ClsiFormatChecker.checkRecoursesForProblems( + req.compile != null ? req.compile.resources : undefined, + (err, validationProblems) => { + if (err != null) { + return callback( + OError.tag( + err, + 'could not check resources for potential problems before sending to clsi' ) - } - opts.jar = jar - const timer = new Metrics.Timer('compile.newBackend') - request(opts, (err, response, body) => { - timer.done() + ) + } + if (validationProblems != null) { + logger.debug( + { projectId, validationProblems }, + 'problems with users latex before compile was attempted' + ) + return callback( + null, + 'validation-problems', + null, + null, + validationProblems + ) + } + _postToClsi( + projectId, + userId, + req, + options.compileBackendClass, + options.compileGroup, + (err, response, clsiServerId) => { if (err != null) { return callback( - OError.tag(err, 'error making request to new CLSI', { + OError.tag(err, 'error sending request to clsi', { projectId, - opts, + userId, }) ) } - NewBackendCloudClsiCookieManager.setServerId( + const outputFiles = _parseOutputFiles( projectId, - userId, - compileGroup, - compileBackendClass, - response, - clsiServerId, - err => { - if (err != null) { - return callback( - OError.tag(err, 'error setting server id on new backend', { - projectId, - }) - ) - } - callback(null, response, body) - } + response && response.compile && response.compile.outputFiles ) - }) + collectMetricsOnBlgFiles(outputFiles) + const compile = (response && response.compile) || {} + const status = compile.status + const stats = compile.stats + const timings = compile.timings + const outputUrlPrefix = compile.outputUrlPrefix + const validationProblems = undefined + callback( + null, + status, + outputFiles, + clsiServerId, + validationProblems, + stats, + timings, + outputUrlPrefix + ) + } + ) + } + ) +} + +function _makeRequestWithClsiServerId( + projectId, + userId, + compileGroup, + compileBackendClass, + opts, + clsiserverid, + callback +) { + if (clsiserverid) { + // ignore cookies and newBackend, go straight to the clsi node + opts.qs = Object.assign( + { compileGroup, compileBackendClass, clsiserverid }, + opts.qs + ) + request(opts, (err, response, body) => { + if (err) { + return callback( + OError.tag(err, 'error making request to CLSI', { projectId }) + ) } - ) - }, - - _getCompilerUrl( - compileBackendClass, - compileGroup, - projectId, - userId, - action - ) { - const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url) - if (userId != null) { - u.pathname += `/user/${userId}` - } - if (action != null) { - u.pathname += `/${action}` - } - u.search = new URLSearchParams({ - compileBackendClass, - compileGroup, - }).toString() - return u.href - }, - - _postToClsi( - projectId, - userId, - req, - compileBackendClass, - compileGroup, - callback - ) { - const compileUrl = this._getCompilerUrl( - compileBackendClass, - compileGroup, - projectId, - userId, - 'compile' - ) - const opts = { - url: compileUrl, - json: req, - method: 'POST', - } - ClsiManager._makeRequest( + callback(null, response, body) + }) + } else { + _makeRequest( projectId, userId, compileGroup, compileBackendClass, opts, - (err, response, body, clsiServerId) => { - if (err != null) { - return callback( - new OError( - 'failed to make request to CLSI', - { + callback + ) + } +} + +function _makeRequest( + projectId, + userId, + compileGroup, + compileBackendClass, + opts, + callback +) { + async.series( + { + currentBackend(cb) { + const startTime = new Date() + ClsiCookieManager.getCookieJar( + projectId, + userId, + compileGroup, + compileBackendClass, + (err, jar, clsiServerId) => { + if (err != null) { + return callback( + OError.tag(err, 'error getting cookie jar for CLSI request', { + projectId, + }) + ) + } + opts.jar = jar + const timer = new Metrics.Timer('compile.currentBackend') + request(opts, (err, response, body) => { + if (err != null) { + return callback( + OError.tag(err, 'error making request to CLSI', { + projectId, + }) + ) + } + timer.done() + Metrics.inc( + `compile.currentBackend.response.${response.statusCode}` + ) + ClsiCookieManager.setServerId( projectId, userId, - compileOptions: req.compile.options, - rootResourcePath: req.compile.rootResourcePath, - }, - err - ) + compileGroup, + compileBackendClass, + response, + clsiServerId, + (err, newClsiServerId) => { + if (err != null) { + callback( + OError.tag(err, 'error setting server id', { + projectId, + }) + ) + } else { + // return as soon as the standard compile has returned + callback( + null, + response, + body, + newClsiServerId || clsiServerId + ) + } + cb(err, { + response, + body, + finishTime: new Date() - startTime, + }) + } + ) + }) + } + ) + }, + newBackend(cb) { + const startTime = new Date() + _makeNewBackendRequest( + projectId, + userId, + compileGroup, + compileBackendClass, + opts, + (err, response, body) => { + if (err != null) { + logger.warn({ err }, 'Error making request to new CLSI backend') + } + if (response != null) { + Metrics.inc(`compile.newBackend.response.${response.statusCode}`) + } + cb(err, { + response, + body, + finishTime: new Date() - startTime, + }) + } + ) + }, + }, + (err, results) => { + if (err != null) { + // This was handled higher up + return + } + if (results.newBackend != null && results.newBackend.response != null) { + const currentStatusCode = results.currentBackend.response.statusCode + const newStatusCode = results.newBackend.response.statusCode + const statusCodeSame = newStatusCode === currentStatusCode + const currentCompileTime = results.currentBackend.finishTime + const newBackendCompileTime = results.newBackend.finishTime || 0 + const timeDifference = newBackendCompileTime - currentCompileTime + logger.debug( + { + statusCodeSame, + timeDifference, + currentCompileTime, + newBackendCompileTime, + projectId, + }, + 'both clsi requests returned' + ) + } + } + ) +} + +function _makeNewBackendRequest( + projectId, + userId, + compileGroup, + compileBackendClass, + baseOpts, + callback +) { + if (Settings.apis.clsi_new == null || Settings.apis.clsi_new.url == null) { + return callback() + } + const opts = { + ...baseOpts, + url: baseOpts.url.replace( + Settings.apis.clsi.url, + Settings.apis.clsi_new.url + ), + } + NewBackendCloudClsiCookieManager.getCookieJar( + projectId, + userId, + compileGroup, + compileBackendClass, + (err, jar, clsiServerId) => { + if (err != null) { + return callback( + OError.tag(err, 'error getting cookie jar for CLSI request', { + projectId, + }) + ) + } + opts.jar = jar + const timer = new Metrics.Timer('compile.newBackend') + request(opts, (err, response, body) => { + timer.done() + if (err != null) { + return callback( + OError.tag(err, 'error making request to new CLSI', { + projectId, + opts, + }) ) } + NewBackendCloudClsiCookieManager.setServerId( + projectId, + userId, + compileGroup, + compileBackendClass, + response, + clsiServerId, + err => { + if (err != null) { + return callback( + OError.tag(err, 'error setting server id on new backend', { + projectId, + }) + ) + } + callback(null, response, body) + } + ) + }) + } + ) +} + +function _getCompilerUrl( + compileBackendClass, + compileGroup, + projectId, + userId, + action +) { + const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url) + if (userId != null) { + u.pathname += `/user/${userId}` + } + if (action != null) { + u.pathname += `/${action}` + } + u.search = new URLSearchParams({ + compileBackendClass, + compileGroup, + }).toString() + return u.href +} + +function _postToClsi( + projectId, + userId, + req, + compileBackendClass, + compileGroup, + callback +) { + const compileUrl = _getCompilerUrl( + compileBackendClass, + compileGroup, + projectId, + userId, + 'compile' + ) + const opts = { + url: compileUrl, + json: req, + method: 'POST', + } + _makeRequest( + projectId, + userId, + compileGroup, + compileBackendClass, + opts, + (err, response, body, clsiServerId) => { + if (err != null) { + return callback( + new OError( + 'failed to make request to CLSI', + { + projectId, + userId, + compileOptions: req.compile.options, + rootResourcePath: req.compile.rootResourcePath, + }, + err + ) + ) + } + if (response.statusCode >= 200 && response.statusCode < 300) { + callback(null, body, clsiServerId) + } else if (response.statusCode === 413) { + callback(null, { compile: { status: 'project-too-large' } }) + } else if (response.statusCode === 409) { + callback(null, { compile: { status: 'conflict' } }) + } else if (response.statusCode === 423) { + callback(null, { compile: { status: 'compile-in-progress' } }) + } else if (response.statusCode === 503) { + callback(null, { compile: { status: 'unavailable' } }) + } else { + callback( + new OError(`CLSI returned non-success code: ${response.statusCode}`, { + projectId, + userId, + compileOptions: req.compile.options, + rootResourcePath: req.compile.rootResourcePath, + clsiResponse: body, + statusCode: response.statusCode, + }) + ) + } + } + ) +} + +function _parseOutputFiles(projectId, rawOutputFiles = []) { + const outputFiles = [] + for (const file of rawOutputFiles) { + const f = { + path: file.path, // the clsi is now sending this to web + url: new URL(file.url).pathname, // the location of the file on the clsi, excluding the host part + type: file.type, + build: file.build, + } + if (file.path === 'output.pdf') { + f.contentId = file.contentId + f.ranges = file.ranges || [] + f.size = file.size + f.startXRefTable = file.startXRefTable + f.createdAt = new Date() + } + outputFiles.push(f) + } + return outputFiles +} + +function _buildRequest(projectId, options, callback) { + if (options == null) { + options = {} + } + ProjectGetter.getProject( + projectId, + { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1 }, + (err, project) => { + if (err != null) { + return callback(OError.tag(err, 'failed to get project', { projectId })) + } + if (project == null) { + return callback( + new Errors.NotFoundError(`project does not exist: ${projectId}`) + ) + } + if (!VALID_COMPILERS.includes(project.compiler)) { + project.compiler = 'pdflatex' + } + + if (options.incrementalCompilesEnabled || options.syncType != null) { + // new way, either incremental or full + const timer = new Metrics.Timer('editor.compile-getdocs-redis') + getContentFromDocUpdaterIfMatch( + projectId, + project, + options, + (err, projectStateHash, docUpdaterDocs) => { + timer.done() + if (err != null) { + logger.error({ err, projectId }, 'error checking project state') + // note: we don't bail out when there's an error getting + // incremental files from the docupdater, we just fall back + // to a normal compile below + } + // see if we can send an incremental update to the CLSI + if ( + docUpdaterDocs != null && + options.syncType !== 'full' && + err == null + ) { + Metrics.inc('compile-from-redis') + _buildRequestFromDocupdater( + projectId, + options, + project, + projectStateHash, + docUpdaterDocs, + callback + ) + } else { + Metrics.inc('compile-from-mongo') + _buildRequestFromMongo( + projectId, + options, + project, + projectStateHash, + callback + ) + } + } + ) + } else { + // old way, always from mongo + const timer = new Metrics.Timer('editor.compile-getdocs-mongo') + _getContentFromMongo(projectId, (err, docs, files) => { + timer.done() + if (err != null) { + return callback( + OError.tag(err, 'failed to get contents from Mongo', { + projectId, + }) + ) + } + _finaliseRequest(projectId, options, project, docs, files, callback) + }) + } + } + ) +} + +function getContentFromDocUpdaterIfMatch( + projectId, + project, + options, + callback +) { + let projectStateHash + try { + projectStateHash = ClsiStateManager.computeHash(project, options) + } catch (err) { + return callback(err) + } + DocumentUpdaterHandler.getProjectDocsIfMatch( + projectId, + projectStateHash, + (err, docs) => { + if (err != null) { + return callback( + OError.tag(err, 'Failed to get project documents', { + projectId, + projectStateHash, + }) + ) + } + callback(null, projectStateHash, docs) + } + ) +} + +function getOutputFileStream( + projectId, + userId, + options, + clsiServerId, + buildId, + outputFilePath, + callback +) { + const url = `${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}` + const { compileBackendClass, compileGroup } = options + const readStream = request({ + url, + method: 'GET', + timeout: 60 * 1000, + qs: { compileBackendClass, compileGroup, clsiserverid: clsiServerId }, + }) + callback(null, readStream) +} + +function _buildRequestFromDocupdater( + projectId, + options, + project, + projectStateHash, + docUpdaterDocs, + callback +) { + let docPath + try { + docPath = ProjectEntityHandler.getAllDocPathsFromProject(project) + } catch (err) { + return callback( + OError.tag(err, 'Failed to get all doc paths from project', { + projectId, + }) + ) + } + const docs = {} + for (const doc of docUpdaterDocs || []) { + const path = docPath[doc._id] + docs[path] = doc + } + // send new docs but not files as those are already on the clsi + options = _.clone(options) + options.syncType = 'incremental' + options.syncState = projectStateHash + // create stub doc entries for any possible root docs, if not + // present in the docupdater. This allows finaliseRequest to + // identify the root doc. + const possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id] + for (const rootDocId of possibleRootDocIds) { + if (rootDocId != null && rootDocId in docPath) { + const path = docPath[rootDocId] + if (docs[path] == null) { + docs[path] = { _id: rootDocId, path } + } + } + } + _finaliseRequest(projectId, options, project, docs, [], callback) +} + +function _buildRequestFromMongo( + projectId, + options, + project, + projectStateHash, + callback +) { + _getContentFromMongo(projectId, (err, docs, files) => { + if (err != null) { + return callback( + OError.tag(err, 'failed to get project contents from Mongo', { + projectId, + }) + ) + } + options = { + ...options, + syncType: 'full', + syncState: projectStateHash, + } + _finaliseRequest(projectId, options, project, docs, files, callback) + }) +} + +function _getContentFromMongo(projectId, callback) { + DocumentUpdaterHandler.flushProjectToMongo(projectId, err => { + if (err != null) { + return callback( + OError.tag(err, 'failed to flush project to Mongo', { projectId }) + ) + } + ProjectEntityHandler.getAllDocs(projectId, (err, docs) => { + if (err != null) { + return callback( + OError.tag(err, 'failed to get project docs', { projectId }) + ) + } + ProjectEntityHandler.getAllFiles(projectId, (err, files) => { + if (err != null) { + return callback( + OError.tag(err, 'failed to get project files', { projectId }) + ) + } + if (files == null) { + files = {} + } + callback(null, docs || {}, files || {}) + }) + }) + }) +} + +function _finaliseRequest(projectId, options, project, docs, files, callback) { + const resources = [] + let flags + let rootResourcePath = null + let rootResourcePathOverride = null + let hasMainFile = false + let numberOfDocsInProject = 0 + + for (let path in docs) { + const doc = docs[path] + path = path.replace(/^\//, '') // Remove leading / + numberOfDocsInProject++ + if (doc.lines != null) { + // add doc to resources unless it is just a stub entry + resources.push({ + path, + content: doc.lines.join('\n'), + }) + } + if ( + project.rootDoc_id != null && + doc._id.toString() === project.rootDoc_id.toString() + ) { + rootResourcePath = path + } + if ( + options.rootDoc_id != null && + doc._id.toString() === options.rootDoc_id.toString() + ) { + rootResourcePathOverride = path + } + if (path === 'main.tex') { + hasMainFile = true + } + } + + if (rootResourcePathOverride != null) { + rootResourcePath = rootResourcePathOverride + } + if (rootResourcePath == null) { + if (hasMainFile) { + rootResourcePath = 'main.tex' + } else if (numberOfDocsInProject === 1) { + // only one file, must be the main document + for (const path in docs) { + // Remove leading / + rootResourcePath = path.replace(/^\//, '') + } + } else { + return callback(new OError('no main file specified', { projectId })) + } + } + + for (let path in files) { + const file = files[path] + path = path.replace(/^\//, '') // Remove leading / + resources.push({ + path, + url: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`, + modified: file.created?.getTime(), + }) + } + + if (options.fileLineErrors) { + flags = ['-file-line-error'] + } + + callback(null, { + compile: { + options: { + compiler: project.compiler, + timeout: options.timeout, + imageName: project.imageName, + draft: Boolean(options.draft), + stopOnFirstError: Boolean(options.stopOnFirstError), + check: options.check, + syncType: options.syncType, + syncState: options.syncState, + compileGroup: options.compileGroup, + enablePdfCaching: + (Settings.enablePdfCaching && options.enablePdfCaching) || false, + pdfCachingMinChunkSize: options.pdfCachingMinChunkSize, + flags, + metricsMethod: options.compileGroup, + }, + rootResourcePath, + resources, + }, + }) +} + +function wordCount(projectId, userId, file, options, clsiserverid, callback) { + const { compileBackendClass, compileGroup } = options + _buildRequest(projectId, options, (err, req) => { + if (err != null) { + return callback( + OError.tag(err, 'Failed to build CLSI request', { + projectId, + options, + }) + ) + } + const filename = file || req.compile.rootResourcePath + const wordCountUrl = _getCompilerUrl( + compileBackendClass, + compileGroup, + projectId, + userId, + 'wordcount' + ) + const opts = { + url: wordCountUrl, + qs: { + file: filename, + image: req.compile.options.imageName, + }, + json: true, + method: 'GET', + } + _makeRequestWithClsiServerId( + projectId, + userId, + compileGroup, + compileBackendClass, + opts, + clsiserverid, + (err, response, body) => { + if (err != null) { + return callback(OError.tag(err, 'CLSI request failed', { projectId })) + } if (response.statusCode >= 200 && response.statusCode < 300) { - callback(null, body, clsiServerId) - } else if (response.statusCode === 413) { - callback(null, { compile: { status: 'project-too-large' } }) - } else if (response.statusCode === 409) { - callback(null, { compile: { status: 'conflict' } }) - } else if (response.statusCode === 423) { - callback(null, { compile: { status: 'compile-in-progress' } }) - } else if (response.statusCode === 503) { - callback(null, { compile: { status: 'unavailable' } }) + callback(null, body) } else { callback( new OError( `CLSI returned non-success code: ${response.statusCode}`, { projectId, - userId, - compileOptions: req.compile.options, - rootResourcePath: req.compile.rootResourcePath, clsiResponse: body, statusCode: response.statusCode, } @@ -620,421 +994,14 @@ const ClsiManager = { } } ) - }, - - _parseOutputFiles(projectId, rawOutputFiles = []) { - const outputFiles = [] - for (const file of rawOutputFiles) { - const f = { - path: file.path, // the clsi is now sending this to web - url: new URL(file.url).pathname, // the location of the file on the clsi, excluding the host part - type: file.type, - build: file.build, - } - if (file.path === 'output.pdf') { - f.contentId = file.contentId - f.ranges = file.ranges || [] - f.size = file.size - f.startXRefTable = file.startXRefTable - f.createdAt = new Date() - } - outputFiles.push(f) - } - return outputFiles - }, - - _buildRequest(projectId, options, callback) { - if (options == null) { - options = {} - } - ProjectGetter.getProject( - projectId, - { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1 }, - (err, project) => { - if (err != null) { - return callback( - OError.tag(err, 'failed to get project', { projectId }) - ) - } - if (project == null) { - return callback( - new Errors.NotFoundError(`project does not exist: ${projectId}`) - ) - } - if (!VALID_COMPILERS.includes(project.compiler)) { - project.compiler = 'pdflatex' - } - - if (options.incrementalCompilesEnabled || options.syncType != null) { - // new way, either incremental or full - const timer = new Metrics.Timer('editor.compile-getdocs-redis') - ClsiManager.getContentFromDocUpdaterIfMatch( - projectId, - project, - options, - (err, projectStateHash, docUpdaterDocs) => { - timer.done() - if (err != null) { - logger.error({ err, projectId }, 'error checking project state') - // note: we don't bail out when there's an error getting - // incremental files from the docupdater, we just fall back - // to a normal compile below - } - // see if we can send an incremental update to the CLSI - if ( - docUpdaterDocs != null && - options.syncType !== 'full' && - err == null - ) { - Metrics.inc('compile-from-redis') - ClsiManager._buildRequestFromDocupdater( - projectId, - options, - project, - projectStateHash, - docUpdaterDocs, - callback - ) - } else { - Metrics.inc('compile-from-mongo') - ClsiManager._buildRequestFromMongo( - projectId, - options, - project, - projectStateHash, - callback - ) - } - } - ) - } else { - // old way, always from mongo - const timer = new Metrics.Timer('editor.compile-getdocs-mongo') - ClsiManager._getContentFromMongo(projectId, (err, docs, files) => { - timer.done() - if (err != null) { - return callback( - OError.tag(err, 'failed to get contents from Mongo', { - projectId, - }) - ) - } - ClsiManager._finaliseRequest( - projectId, - options, - project, - docs, - files, - callback - ) - }) - } - } - ) - }, - - getContentFromDocUpdaterIfMatch(projectId, project, options, callback) { - let projectStateHash - try { - projectStateHash = ClsiStateManager.computeHash(project, options) - } catch (err) { - return callback(err) - } - DocumentUpdaterHandler.getProjectDocsIfMatch( - projectId, - projectStateHash, - (err, docs) => { - if (err != null) { - return callback( - OError.tag(err, 'Failed to get project documents', { - projectId, - projectStateHash, - }) - ) - } - callback(null, projectStateHash, docs) - } - ) - }, - - getOutputFileStream( - projectId, - userId, - options, - clsiServerId, - buildId, - outputFilePath, - callback - ) { - const url = `${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}` - const { compileBackendClass, compileGroup } = options - const readStream = request({ - url, - method: 'GET', - timeout: 60 * 1000, - qs: { compileBackendClass, compileGroup, clsiserverid: clsiServerId }, - }) - callback(null, readStream) - }, - - _buildRequestFromDocupdater( - projectId, - options, - project, - projectStateHash, - docUpdaterDocs, - callback - ) { - let docPath - try { - docPath = ProjectEntityHandler.getAllDocPathsFromProject(project) - } catch (err) { - return callback( - OError.tag(err, 'Failed to get all doc paths from project', { - projectId, - }) - ) - } - const docs = {} - for (const doc of docUpdaterDocs || []) { - const path = docPath[doc._id] - docs[path] = doc - } - // send new docs but not files as those are already on the clsi - options = _.clone(options) - options.syncType = 'incremental' - options.syncState = projectStateHash - // create stub doc entries for any possible root docs, if not - // present in the docupdater. This allows finaliseRequest to - // identify the root doc. - const possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id] - for (const rootDocId of possibleRootDocIds) { - if (rootDocId != null && rootDocId in docPath) { - const path = docPath[rootDocId] - if (docs[path] == null) { - docs[path] = { _id: rootDocId, path } - } - } - } - ClsiManager._finaliseRequest( - projectId, - options, - project, - docs, - [], - callback - ) - }, - - _buildRequestFromMongo( - projectId, - options, - project, - projectStateHash, - callback - ) { - ClsiManager._getContentFromMongo(projectId, (err, docs, files) => { - if (err != null) { - return callback( - OError.tag(err, 'failed to get project contents from Mongo', { - projectId, - }) - ) - } - options = { - ...options, - syncType: 'full', - syncState: projectStateHash, - } - ClsiManager._finaliseRequest( - projectId, - options, - project, - docs, - files, - callback - ) - }) - }, - - _getContentFromMongo(projectId, callback) { - DocumentUpdaterHandler.flushProjectToMongo(projectId, err => { - if (err != null) { - return callback( - OError.tag(err, 'failed to flush project to Mongo', { projectId }) - ) - } - ProjectEntityHandler.getAllDocs(projectId, (err, docs) => { - if (err != null) { - return callback( - OError.tag(err, 'failed to get project docs', { projectId }) - ) - } - ProjectEntityHandler.getAllFiles(projectId, (err, files) => { - if (err != null) { - return callback( - OError.tag(err, 'failed to get project files', { projectId }) - ) - } - if (files == null) { - files = {} - } - callback(null, docs || {}, files || {}) - }) - }) - }) - }, - - _finaliseRequest(projectId, options, project, docs, files, callback) { - const resources = [] - let flags - let rootResourcePath = null - let rootResourcePathOverride = null - let hasMainFile = false - let numberOfDocsInProject = 0 - - for (let path in docs) { - const doc = docs[path] - path = path.replace(/^\//, '') // Remove leading / - numberOfDocsInProject++ - if (doc.lines != null) { - // add doc to resources unless it is just a stub entry - resources.push({ - path, - content: doc.lines.join('\n'), - }) - } - if ( - project.rootDoc_id != null && - doc._id.toString() === project.rootDoc_id.toString() - ) { - rootResourcePath = path - } - if ( - options.rootDoc_id != null && - doc._id.toString() === options.rootDoc_id.toString() - ) { - rootResourcePathOverride = path - } - if (path === 'main.tex') { - hasMainFile = true - } - } - - if (rootResourcePathOverride != null) { - rootResourcePath = rootResourcePathOverride - } - if (rootResourcePath == null) { - if (hasMainFile) { - rootResourcePath = 'main.tex' - } else if (numberOfDocsInProject === 1) { - // only one file, must be the main document - for (const path in docs) { - // Remove leading / - rootResourcePath = path.replace(/^\//, '') - } - } else { - return callback(new OError('no main file specified', { projectId })) - } - } - - for (let path in files) { - const file = files[path] - path = path.replace(/^\//, '') // Remove leading / - resources.push({ - path, - url: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`, - modified: file.created != null ? file.created.getTime() : undefined, - }) - } - - if (options.fileLineErrors) { - flags = ['-file-line-error'] - } - - callback(null, { - compile: { - options: { - compiler: project.compiler, - timeout: options.timeout, - imageName: project.imageName, - draft: Boolean(options.draft), - stopOnFirstError: Boolean(options.stopOnFirstError), - check: options.check, - syncType: options.syncType, - syncState: options.syncState, - compileGroup: options.compileGroup, - enablePdfCaching: - (Settings.enablePdfCaching && options.enablePdfCaching) || false, - pdfCachingMinChunkSize: options.pdfCachingMinChunkSize, - flags, - metricsMethod: options.compileGroup, - }, - rootResourcePath, - resources, - }, - }) - }, - - wordCount(projectId, userId, file, options, clsiserverid, callback) { - const { compileBackendClass, compileGroup } = options - ClsiManager._buildRequest(projectId, options, (err, req) => { - if (err != null) { - return callback( - OError.tag(err, 'Failed to build CLSI request', { - projectId, - options, - }) - ) - } - const filename = file || req.compile.rootResourcePath - const wordCountUrl = ClsiManager._getCompilerUrl( - compileBackendClass, - compileGroup, - projectId, - userId, - 'wordcount' - ) - const opts = { - url: wordCountUrl, - qs: { - file: filename, - image: req.compile.options.imageName, - }, - json: true, - method: 'GET', - } - ClsiManager._makeRequestWithClsiServerId( - projectId, - userId, - compileGroup, - compileBackendClass, - opts, - clsiserverid, - (err, response, body) => { - if (err != null) { - return callback( - OError.tag(err, 'CLSI request failed', { projectId }) - ) - } - if (response.statusCode >= 200 && response.statusCode < 300) { - callback(null, body) - } else { - callback( - new OError( - `CLSI returned non-success code: ${response.statusCode}`, - { - projectId, - clsiResponse: body, - statusCode: response.statusCode, - } - ) - ) - } - } - ) - }) - }, + }) } -module.exports = ClsiManager +module.exports = { + sendRequest, + sendExternalRequest, + stopCompile, + deleteAuxFiles, + getOutputFileStream, + wordCount, +} diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index 8d3cef2a1c..082cd9ae85 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -1272,7 +1272,7 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { const sendRes = _.once(function (statusCode, message) { res.status(statusCode) plainTextResponse(res, message) - ClsiCookieManager.clearServerId(projectId) + ClsiCookieManager.clearServerId(projectId, () => {}) }) // force every compile to a new server // set a timeout let handler = setTimeout(function () { diff --git a/services/web/test/unit/src/Compile/ClsiCookieManagerTests.js b/services/web/test/unit/src/Compile/ClsiCookieManagerTests.js index 76f5fcf486..27c7bd35e4 100644 --- a/services/web/test/unit/src/Compile/ClsiCookieManagerTests.js +++ b/services/web/test/unit/src/Compile/ClsiCookieManagerTests.js @@ -1,16 +1,3 @@ -/* eslint-disable - n/handle-callback-err, - max-len, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const sinon = require('sinon') const { assert, expect } = require('chai') const modulePath = '../../../../app/src/Features/Compile/ClsiCookieManager.js' @@ -19,7 +6,6 @@ const realRequst = require('request') describe('ClsiCookieManager', function () { beforeEach(function () { - const self = this this.redis = { auth() {}, get: sinon.stub(), @@ -57,25 +43,28 @@ describe('ClsiCookieManager', function () { '@overleaf/settings': this.settings, request: this.request, } - return (this.ClsiCookieManager = SandboxedModule.require(modulePath, { + this.ClsiCookieManager = SandboxedModule.require(modulePath, { requires: this.requires, - })()) + })() }) describe('getServerId', function () { it('should call get for the key', function (done) { this.redis.get.callsArgWith(1, null, 'clsi-7') - return this.ClsiCookieManager._getServerId( + this.ClsiCookieManager._getServerId( this.project_id, this.user_id, '', 'e2', (err, serverId) => { + if (err) { + return done(err) + } this.redis.get .calledWith(`clsiserver:${this.project_id}:${this.user_id}`) .should.equal(true) serverId.should.equal('clsi-7') - return done() + done() } ) }) @@ -85,15 +74,18 @@ describe('ClsiCookieManager', function () { .stub() .yields(null) this.redis.get.callsArgWith(1, null) - return this.ClsiCookieManager._getServerId( + this.ClsiCookieManager._getServerId( this.project_id, this.user_id, '', (err, serverId) => { + if (err) { + return done(err) + } this.ClsiCookieManager._populateServerIdViaRequest .calledWith(this.project_id, this.user_id) .should.equal(true) - return done() + done() } ) }) @@ -103,16 +95,19 @@ describe('ClsiCookieManager', function () { .stub() .yields(null) this.redis.get.callsArgWith(1, null, '') - return this.ClsiCookieManager._getServerId( + this.ClsiCookieManager._getServerId( this.project_id, this.user_id, '', 'e2', (err, serverId) => { + if (err) { + return done(err) + } this.ClsiCookieManager._populateServerIdViaRequest .calledWith(this.project_id, this.user_id) .should.equal(true) - return done() + done() } ) }) @@ -122,38 +117,42 @@ describe('ClsiCookieManager', function () { beforeEach(function () { this.response = 'some data' this.request.post.callsArgWith(1, null, this.response) - return (this.ClsiCookieManager.setServerId = sinon - .stub() - .yields(null, 'clsi-9')) + this.ClsiCookieManager.setServerId = sinon.stub().yields(null, 'clsi-9') }) it('should make a request to the clsi', function (done) { - return this.ClsiCookieManager._populateServerIdViaRequest( + this.ClsiCookieManager._populateServerIdViaRequest( this.project_id, this.user_id, 'standard', 'e2', (err, serverId) => { + if (err) { + return done(err) + } const args = this.ClsiCookieManager.setServerId.args[0] args[0].should.equal(this.project_id) args[1].should.equal(this.user_id) args[2].should.equal('standard') args[3].should.equal('e2') args[4].should.deep.equal(this.response) - return done() + done() } ) }) it('should return the server id', function (done) { - return this.ClsiCookieManager._populateServerIdViaRequest( + this.ClsiCookieManager._populateServerIdViaRequest( this.project_id, this.user_id, '', 'e2', (err, serverId) => { + if (err) { + return done(err) + } serverId.should.equal('clsi-9') - return done() + done() } ) }) @@ -168,7 +167,7 @@ describe('ClsiCookieManager', function () { }) it('should set the server id with a ttl', function (done) { - return this.ClsiCookieManager.setServerId( + this.ClsiCookieManager.setServerId( this.project_id, this.user_id, 'standard', @@ -176,6 +175,9 @@ describe('ClsiCookieManager', function () { this.response, null, err => { + if (err) { + return done(err) + } this.redis.setex .calledWith( `clsiserver:${this.project_id}:${this.user_id}`, @@ -183,7 +185,7 @@ describe('ClsiCookieManager', function () { 'clsi-8' ) .should.equal(true) - return done() + done() } ) }) @@ -200,6 +202,9 @@ describe('ClsiCookieManager', function () { this.response, null, err => { + if (err) { + return done(err) + } expect(this.redis.setex).to.have.been.calledWith( `clsiserver:${this.project_id}:${this.user_id}`, this.settings.clsiCookie.ttlInSecondsRegular, @@ -211,7 +216,7 @@ describe('ClsiCookieManager', function () { }) it('should return the server id', function (done) { - return this.ClsiCookieManager.setServerId( + this.ClsiCookieManager.setServerId( this.project_id, this.user_id, 'standard', @@ -219,8 +224,11 @@ describe('ClsiCookieManager', function () { this.response, null, (err, serverId) => { + if (err) { + return done(err) + } serverId.should.equal('clsi-8') - return done() + done() } ) }) @@ -233,7 +241,7 @@ describe('ClsiCookieManager', function () { }, requires: this.requires, })() - return this.ClsiCookieManager.setServerId( + this.ClsiCookieManager.setServerId( this.project_id, this.user_id, 'standard', @@ -241,8 +249,11 @@ describe('ClsiCookieManager', function () { this.response, null, (err, serverId) => { + if (err) { + return done(err) + } this.redis.setex.called.should.equal(false) - return done() + done() } ) }) @@ -251,7 +262,7 @@ describe('ClsiCookieManager', function () { this.ClsiCookieManager._parseServerIdFromResponse = sinon .stub() .returns(null) - return this.ClsiCookieManager.setServerId( + this.ClsiCookieManager.setServerId( this.project_id, this.user_id, 'standard', @@ -259,8 +270,11 @@ describe('ClsiCookieManager', function () { this.response, null, (err, serverId) => { + if (err) { + return done(err) + } this.redis.setex.called.should.equal(false) - return done() + done() } ) }) @@ -282,7 +296,7 @@ describe('ClsiCookieManager', function () { this.ClsiCookieManager._parseServerIdFromResponse = sinon .stub() .returns('clsi-8') - return this.ClsiCookieManager.setServerId( + this.ClsiCookieManager.setServerId( this.project_id, this.user_id, 'standard', @@ -290,6 +304,9 @@ describe('ClsiCookieManager', function () { this.response, null, (err, serverId) => { + if (err) { + return done(err) + } this.redis_secondary.setex .calledWith( `clsiserver:${this.project_id}:${this.user_id}`, @@ -297,7 +314,7 @@ describe('ClsiCookieManager', function () { 'clsi-8' ) .should.equal(true) - return done() + done() } ) }) @@ -305,25 +322,26 @@ describe('ClsiCookieManager', function () { describe('getCookieJar', function () { beforeEach(function () { - return (this.ClsiCookieManager._getServerId = sinon - .stub() - .yields(null, 'clsi-11')) + this.ClsiCookieManager._getServerId = sinon.stub().yields(null, 'clsi-11') }) it('should return a jar with the cookie set populated from redis', function (done) { - return this.ClsiCookieManager.getCookieJar( + this.ClsiCookieManager.getCookieJar( this.project_id, this.user_id, '', 'e2', (err, jar) => { + if (err) { + return done(err) + } jar._jar.store.idx['clsi.example.com']['/'][ this.settings.clsiCookie.key ].key.should.equal jar._jar.store.idx['clsi.example.com']['/'][ this.settings.clsiCookie.key ].value.should.equal('clsi-11') - return done() + done() } ) }) @@ -336,14 +354,17 @@ describe('ClsiCookieManager', function () { }, requires: this.requires, })() - return this.ClsiCookieManager.getCookieJar( + this.ClsiCookieManager.getCookieJar( this.project_id, this.user_id, '', 'e2', (err, jar) => { + if (err) { + return done(err) + } assert.deepEqual(jar, realRequst.jar()) - return done() + done() } ) }) diff --git a/services/web/test/unit/src/Compile/ClsiManagerTests.js b/services/web/test/unit/src/Compile/ClsiManagerTests.js index f805cae748..8ccee32b36 100644 --- a/services/web/test/unit/src/Compile/ClsiManagerTests.js +++ b/services/web/test/unit/src/Compile/ClsiManagerTests.js @@ -4,28 +4,71 @@ const modulePath = '../../../../app/src/Features/Compile/ClsiManager.js' const SandboxedModule = require('sandboxed-module') const tk = require('timekeeper') +const FILESTORE_URL = 'http://filestore.example.com' +const CLSI_URL = 'http://clsi.example.com' + describe('ClsiManager', function () { beforeEach(function () { this.jar = { cookie: 'stuff' } + this.user_id = 'user-id' + this.project = { + _id: 'project-id', + compiler: 'latex', + rootDoc_id: 'mock-doc-id-1', + imageName: 'mock-image-name', + } + this.docs = { + '/main.tex': { + name: 'main.tex', + _id: 'mock-doc-id-1', + lines: ['Hello', 'world'], + }, + '/chapters/chapter1.tex': { + name: 'chapter1.tex', + _id: 'mock-doc-id-2', + lines: ['Chapter 1'], + }, + } + this.files = { + '/images/image.png': { + name: 'image.png', + _id: 'mock-file-id-1', + created: new Date(), + }, + } + this.rawOutputFiles = {} + this.response = { statusCode: 200 } + this.responseBody = { + compile: { status: 'success' }, + } + + this.request = sinon.stub().yields(null, this.response, this.responseBody) this.ClsiCookieManager = { clearServerId: sinon.stub().yields(), getCookieJar: sinon.stub().yields(null, this.jar), setServerId: sinon.stub().yields(null), - _getServerId: sinon.stub(), } this.ClsiStateManager = { computeHash: sinon.stub().returns('01234567890abcdef'), } this.ClsiFormatChecker = { - checkRecoursesForProblems: sinon.stub().callsArgWith(1), + checkRecoursesForProblems: sinon.stub().yields(), } this.Project = {} - this.ProjectEntityHandler = {} - this.ProjectGetter = {} - this.DocumentUpdaterHandler = { - getProjectDocsIfMatch: sinon.stub().callsArgWith(2, null, null), + this.ProjectEntityHandler = { + getAllDocs: sinon.stub().yields(null, this.docs), + getAllFiles: sinon.stub().yields(null, this.files), + getAllDocPathsFromProject: sinon.stub(), + } + this.ProjectGetter = { + findById: sinon.stub().yields(null, this.project), + getProject: sinon.stub().yields(null, this.project), + } + this.DocumentUpdaterHandler = { + clearProjectState: sinon.stub().yields(), + flushProjectToMongo: sinon.stub().yields(), + getProjectDocsIfMatch: sinon.stub().yields(), } - this.request = sinon.stub() this.Metrics = { Timer: class Metrics { constructor() { @@ -35,16 +78,20 @@ describe('ClsiManager', function () { inc: sinon.stub(), count: sinon.stub(), } + this.SplitTestHandler = { + getAssignment: sinon.stub().yields(null, { variant: 'default' }), + } + this.ClsiManager = SandboxedModule.require(modulePath, { requires: { '@overleaf/settings': (this.settings = { apis: { filestore: { - url: 'filestore.example.com', + url: FILESTORE_URL, secret: 'secret', }, clsi: { - url: 'http://clsi.example.com', + url: CLSI_URL, defaultBackendClass: 'e2', }, clsi_priority: { @@ -65,15 +112,9 @@ describe('ClsiManager', function () { request: this.request, './ClsiFormatChecker': this.ClsiFormatChecker, '@overleaf/metrics': this.Metrics, - '../SplitTests/SplitTestHandler': { - getAssignment: (this.getAssignment = sinon.stub().yields(null, { - variant: 'default', - })), - }, + '../SplitTests/SplitTestHandler': this.SplitTestHandler, }, }) - this.project_id = 'project-id' - this.user_id = 'user-id' this.callback = sinon.stub() tk.freeze(Date.now()) }) @@ -84,821 +125,432 @@ describe('ClsiManager', function () { describe('sendRequest', function () { beforeEach(function () { - this.ClsiManager._buildRequest = sinon - .stub() - .callsArgWith(2, null, (this.request = 'mock-request')) - this.ClsiCookieManager._getServerId.callsArgWith(2, null, 'clsi3') + this.ClsiCookieManager.getCookieJar.yields(null, this.jar, 'clsi3') }) describe('with a successful compile', function () { - beforeEach(function () { - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'success'), - outputFiles: [ - { - url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - }, - { - url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ], - }, - }) - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - { compileBackendClass: 'e2', compileGroup: 'standard' }, - this.callback - ) - }) - - it('should build the request', function () { - this.ClsiManager._buildRequest - .calledWith(this.project_id) - .should.equal(true) - }) - - it('should send the request to the CLSI', function () { - this.ClsiManager._postToClsi - .calledWith( - this.project_id, - this.user_id, - this.request, - 'e2', - 'standard' - ) - .should.equal(true) - }) - - it('should call the callback with the status and output files', function () { - const outputFiles = [ - { - url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - ranges: [], - createdAt: new Date(), - // gets dropped by JSON.stringify - contentId: undefined, - size: undefined, - startXRefTable: undefined, - }, - { - url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - this.callback - .calledWith(null, this.status, outputFiles) - .should.equal(true) - }) - }) - - describe('with ranges on the pdf and stats/timings details', function () { - beforeEach(function () { - this.ClsiManager._postToClsi = sinon.stub().yields( - null, - { - compile: { - status: 'success', - stats: { fooStat: 1 }, - timings: { barTiming: 2 }, - outputFiles: [ - { - url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - contentId: '123-321', - ranges: [{ start: 1, end: 42, hash: 'foo' }], - startXRefTable: 42, - size: 42, - }, - { - url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ], - }, - }, - 'clsi-server-id-43' - ) - this.ClsiCookieManager._getServerId.yields(null, 'clsi-server-id-42') - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - { compileBackendClass: 'e2', compileGroup: 'standard' }, - this.callback - ) - }) - - it('should emit the caching details and stats/timings', function () { - const outputFiles = [ - { - url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - contentId: '123-321', - ranges: [{ start: 1, end: 42, hash: 'foo' }], - startXRefTable: 42, - size: 42, - createdAt: new Date(), - }, - { - url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - const validationError = undefined - expect(this.callback).to.have.been.calledWith( - null, - 'success', - outputFiles, - 'clsi-server-id-43', - validationError, - { fooStat: 1 }, - { barTiming: 2 } - ) - }) - }) - - describe('with a failed compile', function () { - beforeEach(function () { - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'failure'), - }, - }) - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - {}, - this.callback - ) - }) - - it('should call the callback with a failure status', function () { - this.callback.calledWith(null, this.status).should.equal(true) - }) - }) - - describe('with a sync conflict', function () { - beforeEach(function () { - this.ClsiManager.sendRequestOnce = sinon.stub() - this.ClsiManager.sendRequestOnce - .withArgs(this.project_id, this.user_id, { syncType: 'full' }) - .callsArgWith(3, null, (this.status = 'success')) - this.ClsiManager.sendRequestOnce - .withArgs(this.project_id, this.user_id, {}) - .callsArgWith(3, null, 'conflict') - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - {}, - this.callback - ) - }) - - it('should call the sendRequestOnce method twice', function () { - this.ClsiManager.sendRequestOnce.calledTwice.should.equal(true) - }) - - it('should call the sendRequestOnce method with syncType:full', function () { - this.ClsiManager.sendRequestOnce - .calledWith(this.project_id, this.user_id, { syncType: 'full' }) - .should.equal(true) - }) - - it('should call the sendRequestOnce method without syncType:full', function () { - this.ClsiManager.sendRequestOnce - .calledWith(this.project_id, this.user_id, {}) - .should.equal(true) - }) - - it('should call the callback with a success status', function () { - this.callback.calledWith(null, this.status).should.equal(true) - }) - }) - - describe('with an unavailable response', function () { - beforeEach(function () { - this.ClsiManager.sendRequestOnce = sinon.stub() - this.ClsiManager.sendRequestOnce - .withArgs(this.project_id, this.user_id, { - syncType: 'full', - forceNewClsiServer: true, - }) - .callsArgWith(3, null, (this.status = 'success')) - this.ClsiManager.sendRequestOnce - .withArgs(this.project_id, this.user_id, {}) - .callsArgWith(3, null, 'unavailable') - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - {}, - this.callback - ) - }) - - it('should call the sendRequestOnce method twice', function () { - this.ClsiManager.sendRequestOnce.calledTwice.should.equal(true) - }) - - it('should call the sendRequestOnce method with forceNewClsiServer:true', function () { - this.ClsiManager.sendRequestOnce - .calledWith(this.project_id, this.user_id, { - forceNewClsiServer: true, - syncType: 'full', - }) - .should.equal(true) - }) - - it('should call the sendRequestOnce method without forceNewClsiServer:true', function () { - this.ClsiManager.sendRequestOnce - .calledWith(this.project_id, this.user_id, {}) - .should.equal(true) - }) - - it('should call the callback with a success status', function () { - this.callback.calledWith(null, this.status).should.equal(true) - }) - }) - - describe('when the resources fail the precompile check', function () { - beforeEach(function () { - this.ClsiFormatChecker.checkRecoursesForProblems = sinon - .stub() - .callsArgWith(1, new Error('failed')) - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'failure'), - }, - }) - this.ClsiManager.sendRequest( - this.project_id, - this.user_id, - {}, - this.callback - ) - }) - - it('should call the callback only once', function () { - this.callback.calledOnce.should.equal(true) - }) - - it('should call the callback with an error', function () { - this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error)) - }) - }) - }) - - describe('sendExternalRequest', function () { - beforeEach(function () { - this.submission_id = 'submission-id' - this.clsi_request = 'mock-request' - this.ClsiCookieManager._getServerId.callsArgWith(2, null, 'clsi3') - }) - - describe('with a successful compile', function () { - beforeEach(function () { - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'success'), - outputFiles: [ - { - url: `${this.settings.apis.clsi.url}/project/${this.submission_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - }, - { - url: `${this.settings.apis.clsi.url}/project/${this.submission_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ], - }, - }) - this.ClsiManager.sendExternalRequest( - this.submission_id, - this.clsi_request, - { compileBackendClass: 'e2', compileGroup: 'standard' }, - this.callback - ) - }) - - it('should send the request to the CLSI', function () { - this.ClsiManager._postToClsi - .calledWith( - this.submission_id, - null, - this.clsi_request, - 'e2', - 'standard' - ) - .should.equal(true) - }) - - it('should call the callback with the status and output files', function () { - const outputFiles = [ - { - url: `/project/${this.submission_id}/build/1234/output/output.pdf`, - path: 'output.pdf', - type: 'pdf', - build: 1234, - ranges: [], - createdAt: new Date(), - // gets dropped by JSON.stringify - contentId: undefined, - size: undefined, - startXRefTable: undefined, - }, - { - url: `/project/${this.submission_id}/build/1234/output/output.log`, - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - this.callback.should.have.been.calledWith( - null, - this.status, - outputFiles - ) - }) - }) - - describe('with a failed compile', function () { - beforeEach(function () { - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'failure'), - }, - }) - this.ClsiManager.sendExternalRequest( - this.submission_id, - this.clsi_request, - {}, - this.callback - ) - }) - - it('should call the callback with a failure status', function () { - this.callback.calledWith(null, this.status).should.equal(true) - }) - }) - - describe('when the resources fail the precompile check', function () { - beforeEach(function () { - this.ClsiFormatChecker.checkRecoursesForProblems = sinon - .stub() - .callsArgWith(1, new Error('failed')) - this.ClsiManager._postToClsi = sinon.stub().yields(null, { - compile: { - status: (this.status = 'failure'), - }, - }) - this.ClsiManager.sendExternalRequest( - this.submission_id, - this.clsi_request, - {}, - this.callback - ) - }) - - it('should call the callback only once', function () { - this.callback.calledOnce.should.equal(true) - }) - - it('should call the callback with an error', function () { - this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error)) - }) - }) - }) - - describe('deleteAuxFiles', function () { - beforeEach(function () { - this.ClsiManager._makeRequestWithClsiServerId = sinon.stub().yields(null) - this.DocumentUpdaterHandler.clearProjectState = sinon.stub().callsArg(1) - }) - - describe('with the standard compileGroup', function () { - beforeEach(function () { - this.ClsiManager.deleteAuxFiles( - this.project_id, - this.user_id, - { compileBackendClass: 'e2', compileGroup: 'standard' }, - 'node-1', - this.callback - ) - }) - - it('should call the delete method in the standard CLSI', function () { - this.ClsiManager._makeRequestWithClsiServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - { - method: 'DELETE', - url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}?compileBackendClass=e2&compileGroup=standard`, - }, - 'node-1' - ) - .should.equal(true) - }) - - it('should clear the project state from the docupdater', function () { - this.DocumentUpdaterHandler.clearProjectState - .calledWith(this.project_id) - .should.equal(true) - }) - - it('should clear the clsi persistance', function () { - this.ClsiCookieManager.clearServerId - .calledWith(this.project_id, this.user_id) - .should.equal(true) - }) - - it('should call the callback', function () { - this.callback.called.should.equal(true) - }) - }) - }) - - describe('_buildRequest', function () { - beforeEach(function () { - this.project = { - _id: this.project_id, - compiler: (this.compiler = 'latex'), - rootDoc_id: 'mock-doc-id-1', - imageName: (this.image = 'mock-image-name'), - } - - this.docs = { - '/main.tex': (this.doc_1 = { - name: 'main.tex', - _id: 'mock-doc-id-1', - lines: ['Hello', 'world'], - }), - '/chapters/chapter1.tex': (this.doc_2 = { - name: 'chapter1.tex', - _id: 'mock-doc-id-2', - lines: ['Chapter 1'], - }), - } - - this.files = { - '/images/image.png': (this.file_1 = { - name: 'image.png', - _id: 'mock-file-id-1', - created: new Date(), - }), - } - - this.Project.findById = sinon.stub().callsArgWith(2, null, this.project) - this.ProjectEntityHandler.getAllDocs = sinon - .stub() - .callsArgWith(1, null, this.docs) - this.ProjectEntityHandler.getAllFiles = sinon - .stub() - .callsArgWith(1, null, this.files) - this.ProjectGetter.getProject = sinon - .stub() - .callsArgWith(2, null, this.project) - this.DocumentUpdaterHandler.flushProjectToMongo = sinon - .stub() - .callsArgWith(1, null) - }) - - describe('with a valid project', function () { beforeEach(function (done) { - this.ClsiManager._buildRequest( - this.project_id, - { timeout: 100, compileBackendClass: 'e2', compileGroup: 'standard' }, - (err, request) => { - if (err != null) { + this.outputFiles = [ + { + url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`, + path: 'output.pdf', + type: 'pdf', + build: 1234, + }, + { + url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`, + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + this.responseBody.compile.outputFiles = this.outputFiles.map( + outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url }) + ) + this.timeout = 100 + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + { + compileBackendClass: 'e2', + compileGroup: 'standard', + timeout: this.timeout, + }, + (err, status, outputFiles) => { + if (err) { return done(err) } - this.request = request + this.result = { status, outputFiles } done() } ) }) + it('should send the request to the CLSI', function () { + this.request.should.have.been.calledWith({ + url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`, + method: 'POST', + json: { + compile: { + options: sinon.match({ + compiler: this.project.compiler, + imageName: this.project.imageName, + timeout: this.timeout, + draft: false, + check: undefined, + syncType: undefined, // "full" + syncState: undefined, + compileGroup: 'standard', + enablePdfCaching: false, + pdfCachingMinChunkSize: undefined, + flags: undefined, + metricsMethod: 'standard', + stopOnFirstError: false, + }), + rootResourcePath: 'main.tex', + resources: _makeResources(this.project, this.docs, this.files), + }, + }, + jar: this.jar, + }) + }) + it('should get the project with the required fields', function () { - this.ProjectGetter.getProject - .calledWith(this.project_id, { + this.ProjectGetter.getProject.should.have.been.calledWith( + this.project._id, + { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1, - }) - .should.equal(true) + } + ) }) it('should flush the project to the database', function () { - this.DocumentUpdaterHandler.flushProjectToMongo - .calledWith(this.project_id) - .should.equal(true) + this.DocumentUpdaterHandler.flushProjectToMongo.should.have.been.calledWith( + this.project._id + ) }) it('should get all the docs', function () { - this.ProjectEntityHandler.getAllDocs - .calledWith(this.project_id) - .should.equal(true) + this.ProjectEntityHandler.getAllDocs.should.have.been.calledWith( + this.project._id + ) }) it('should get all the files', function () { - this.ProjectEntityHandler.getAllFiles - .calledWith(this.project_id) - .should.equal(true) + this.ProjectEntityHandler.getAllFiles.should.have.been.calledWith( + this.project._id + ) }) - it('should build up the CLSI request', function () { - expect(this.request).to.deep.equal({ - compile: { - options: { - compiler: this.compiler, - timeout: 100, - imageName: this.image, - draft: false, - check: undefined, - syncType: undefined, // "full" - syncState: undefined, - compileGroup: 'standard', - enablePdfCaching: false, - pdfCachingMinChunkSize: undefined, - flags: undefined, - metricsMethod: 'standard', - stopOnFirstError: false, - }, // "01234567890abcdef" - rootResourcePath: 'main.tex', - resources: [ - { - path: 'main.tex', - content: this.doc_1.lines.join('\n'), - }, - { - path: 'chapters/chapter1.tex', - content: this.doc_2.lines.join('\n'), - }, - { - path: 'images/image.png', - url: `${this.settings.apis.filestore.url}/project/${this.project_id}/file/${this.file_1._id}`, - modified: this.file_1.created.getTime(), - }, - ], + it('should call the callback with the status and output files', function () { + expect(this.result.status).to.equal('success') + expect(this.result.outputFiles.map(f => f.path)).to.have.members( + this.outputFiles.map(f => f.path) + ) + }) + + it('should process a request with a cookie jar', function () { + expect(this.request).to.have.been.calledWith( + sinon.match(opts => opts.jar === this.jar && opts.qs == null) + ) + }) + + it('should persist the cookie from the response', function () { + expect(this.ClsiCookieManager.setServerId).to.have.been.calledWith( + this.project._id, + this.user_id, + 'standard', + 'e2', + this.response + ) + }) + }) + + describe('with ranges on the pdf and stats/timings details', function () { + beforeEach(function (done) { + this.ranges = [{ start: 1, end: 42, hash: 'foo' }] + this.startXRefTable = 123 + this.size = 456 + this.contentId = '123-321' + this.outputFiles = [ + { + url: `/project/${this.project._id}/user/${this.user_id}/build/1234/output/output.pdf`, + path: 'output.pdf', + type: 'pdf', + build: 1234, + contentId: this.contentId, + ranges: this.ranges, + startXRefTable: this.startXRefTable, + size: this.size, }, - }) + { + url: `/project/${this.project._id}/user/${this.user_id}/build/1234/output/output.log`, + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + this.stats = { fooStat: 1 } + this.timings = { barTiming: 2 } + this.serverId = 'clsi-server-id-42' + this.responseBody.compile.outputFiles = this.outputFiles.map( + outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url }) + ) + this.responseBody.compile.stats = this.stats + this.responseBody.compile.timings = this.timings + this.ClsiCookieManager.getCookieJar.yields( + null, + this.jar, + this.serverId + ) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + { compileBackendClass: 'e2', compileGroup: 'standard' }, + ( + err, + status, + outputFiles, + serverId, + validationError, + stats, + timings + ) => { + if (err) { + return done(err) + } + this.result = { + status, + outputFiles, + serverId, + validationError, + stats, + timings, + } + done() + } + ) + }) + + it('should emit the caching details and stats/timings', function () { + expect(this.result.status).to.equal('success') + expect(this.result.serverId).to.equal(this.serverId) + expect(this.result.validationError).to.be.undefined + expect(this.result.stats).to.equal(this.stats) + expect(this.result.timings).to.equal(this.timings) + const outputPdf = this.result.outputFiles.find( + f => f.path === 'output.pdf' + ) + expect(outputPdf.ranges).to.equal(this.ranges) + expect(outputPdf.startXRefTable).to.equal(this.startXRefTable) + expect(outputPdf.contentId).to.equal(this.contentId) + expect(outputPdf.size).to.equal(this.size) }) }) describe('with the incremental compile option', function () { beforeEach(function (done) { - this.project_state_hash = '01234567890abcdef' - this.ClsiStateManager.computeHash = sinon - .stub() - .returns(this.project_state_hash) - this.DocumentUpdaterHandler.getProjectDocsIfMatch = sinon - .stub() - .callsArgWith(2, null, [ - { _id: this.doc_1._id, lines: this.doc_1.lines, v: 123 }, - ]) - this.ProjectEntityHandler.getAllDocPathsFromProject = sinon - .stub() - .returns({ 'mock-doc-id-1': 'main.tex' }) - this.ClsiManager._buildRequest( - this.project_id, + const doc = this.docs['/main.tex'] + this.DocumentUpdaterHandler.getProjectDocsIfMatch.yields(null, [ + { _id: doc._id, lines: doc.lines, v: 123 }, + ]) + this.ProjectEntityHandler.getAllDocPathsFromProject.returns({ + 'mock-doc-id-1': 'main.tex', + }) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, { timeout: 100, incrementalCompilesEnabled: true, + compileBackendClass: 'e2', compileGroup: 'priority', enablePdfCaching: true, pdfCachingMinChunkSize: 1337, }, - (err, request) => { - if (err != null) { + (err, status, outputFiles) => { + if (err) { return done(err) } - this.request = request + this.result = { status, outputFiles } done() } ) }) it('should get the project with the required fields', function () { - this.ProjectGetter.getProject - .calledWith(this.project_id, { + this.ProjectGetter.getProject.should.have.been.calledWith( + this.project._id, + { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1, - }) - .should.equal(true) + } + ) }) it('should not explicitly flush the project to the database', function () { - this.DocumentUpdaterHandler.flushProjectToMongo - .calledWith(this.project_id) - .should.equal(false) + this.DocumentUpdaterHandler.flushProjectToMongo.should.not.have.been.calledWith( + this.project._id + ) }) it('should get only the live docs from the docupdater with a background flush in docupdater', function () { - this.DocumentUpdaterHandler.getProjectDocsIfMatch - .calledWith(this.project_id) - .should.equal(true) + this.DocumentUpdaterHandler.getProjectDocsIfMatch.should.have.been.calledWith( + this.project._id + ) }) it('should not get any of the files', function () { - this.ProjectEntityHandler.getAllFiles.called.should.equal(false) + this.ProjectEntityHandler.getAllFiles.should.not.have.been.called }) it('should build up the CLSI request', function () { - expect(this.request).to.deep.equal({ - compile: { - options: { - compiler: this.compiler, - timeout: 100, - imageName: this.image, - draft: false, - check: undefined, - syncType: 'incremental', - syncState: '01234567890abcdef', - compileGroup: 'priority', - enablePdfCaching: true, - pdfCachingMinChunkSize: 1337, - flags: undefined, - metricsMethod: 'priority', - stopOnFirstError: false, - }, - rootResourcePath: 'main.tex', - resources: [ - { - path: 'main.tex', - content: this.doc_1.lines.join('\n'), + this.request.should.have.been.calledWith({ + url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=priority`, + method: 'POST', + json: { + compile: { + options: { + compiler: this.project.compiler, + timeout: 100, + imageName: this.project.imageName, + draft: false, + check: undefined, + syncType: 'incremental', + syncState: '01234567890abcdef', + compileGroup: 'priority', + enablePdfCaching: true, + pdfCachingMinChunkSize: 1337, + flags: undefined, + metricsMethod: 'priority', + stopOnFirstError: false, }, - ], + rootResourcePath: 'main.tex', + resources: [ + { + path: 'main.tex', + content: this.docs['/main.tex'].lines.join('\n'), + }, + ], + }, }, + jar: this.jar, }) }) + }) - describe('when the root doc is set and not in the docupdater', function () { - beforeEach(function (done) { - this.project_state_hash = '01234567890abcdef' - this.ClsiStateManager.computeHash = sinon - .stub() - .returns(this.project_state_hash) - this.DocumentUpdaterHandler.getProjectDocsIfMatch = sinon - .stub() - .callsArgWith(2, null, [ - { _id: this.doc_1._id, lines: this.doc_1.lines, v: 123 }, - ]) - this.ProjectEntityHandler.getAllDocPathsFromProject = sinon - .stub() - .returns({ - 'mock-doc-id-1': 'main.tex', - 'mock-doc-id-2': '/chapters/chapter1.tex', - }) - this.ClsiManager._buildRequest( - this.project_id, - { - timeout: 100, - incrementalCompilesEnabled: true, - rootDoc_id: 'mock-doc-id-2', - }, - (err, request) => { - if (err != null) { - return done(err) - } - this.request = request - done() - } - ) + describe('when the root doc is set and not in the docupdater', function () { + beforeEach(function (done) { + const doc = this.docs['/main.tex'] + this.DocumentUpdaterHandler.getProjectDocsIfMatch.yields(null, [ + { _id: doc._id, lines: doc.lines, v: 123 }, + ]) + this.ProjectEntityHandler.getAllDocPathsFromProject.returns({ + 'mock-doc-id-1': 'main.tex', + 'mock-doc-id-2': '/chapters/chapter1.tex', }) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + { + timeout: 100, + incrementalCompilesEnabled: true, + rootDoc_id: 'mock-doc-id-2', + }, + done + ) + }) - it('should still change the root path', function () { - this.request.compile.rootResourcePath.should.equal( - 'chapters/chapter1.tex' + it('should still change the root path', function () { + this.request.should.have.been.calledWith( + sinon.match( + opts => + opts.json.compile.rootResourcePath === 'chapters/chapter1.tex' ) - }) + ) }) }) describe('when root doc override is valid', function () { beforeEach(function (done) { - this.ClsiManager._buildRequest( - this.project_id, + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, { rootDoc_id: 'mock-doc-id-2' }, - (err, request) => { - if (err != null) { - return done(err) - } - this.request = request - done() - } + done ) }) it('should change root path', function () { - this.request.compile.rootResourcePath.should.equal( - 'chapters/chapter1.tex' + this.request.should.have.been.calledWith( + sinon.match( + opts => + opts.json.compile.rootResourcePath === 'chapters/chapter1.tex' + ) ) }) }) describe('when root doc override is invalid', function () { beforeEach(function (done) { - this.ClsiManager._buildRequest( - this.project_id, + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, { rootDoc_id: 'invalid-id' }, - (err, request) => { - if (err != null) { - return done(err) - } - this.request = request - done() - } + done ) }) it('should fallback to default root doc', function () { - this.request.compile.rootResourcePath.should.equal('main.tex') + this.request.should.have.been.calledWith( + sinon.match(opts => opts.json.compile.rootResourcePath === 'main.tex') + ) }) }) describe('when the project has an invalid compiler', function () { beforeEach(function (done) { this.project.compiler = 'context' - this.ClsiManager._buildRequest(this.project, null, (err, request) => { - if (err != null) { - return done(err) - } - this.request = request - done() - }) + this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done) }) it('should set the compiler to pdflatex', function () { - this.request.compile.options.compiler.should.equal('pdflatex') + expect(this.request).to.have.been.calledWith( + sinon.match(opts => opts.json.compile.options.compiler === 'pdflatex') + ) }) }) describe('when there is no valid root document', function () { beforeEach(function (done) { this.project.rootDoc_id = 'not-valid' - this.ClsiManager._buildRequest(this.project, null, (error, request) => { - this.error = error - this.request = request - done() - }) + this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done) }) it('should set to main.tex', function () { - this.request.compile.rootResourcePath.should.equal('main.tex') + expect(this.request).to.have.been.calledWith( + sinon.match(opts => opts.json.compile.rootResourcePath === 'main.tex') + ) }) }) describe('when there is no valid root document and no main.tex document', function () { - beforeEach(function () { + beforeEach(function (done) { this.project.rootDoc_id = 'not-valid' this.docs = { - '/other.tex': (this.doc_1 = { + '/other.tex': { name: 'other.tex', _id: 'mock-doc-id-1', lines: ['Hello', 'world'], - }), - '/chapters/chapter1.tex': (this.doc_2 = { + }, + '/chapters/chapter1.tex': { name: 'chapter1.tex', _id: 'mock-doc-id-2', lines: ['Chapter 1'], - }), + }, } - this.ProjectEntityHandler.getAllDocs = sinon - .stub() - .callsArgWith(1, null, this.docs) - this.ClsiManager._buildRequest(this.project, null, this.callback) + this.ProjectEntityHandler.getAllDocs.yields(null, this.docs) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + {}, + (err, status) => { + if (err) { + return done(err) + } + this.result = { status } + done() + } + ) }) - it('should report an error', function () { - this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error)) + it('should report a validation problem', function () { + expect(this.result.status).to.equal('validation-problems') }) }) @@ -906,411 +558,441 @@ describe('ClsiManager', function () { beforeEach(function (done) { this.project.rootDoc_id = 'not-valid' this.docs = { - '/other.tex': (this.doc_1 = { + '/other.tex': { name: 'other.tex', _id: 'mock-doc-id-1', lines: ['Hello', 'world'], - }), + }, } - this.ProjectEntityHandler.getAllDocs = sinon - .stub() - .callsArgWith(1, null, this.docs) - this.ClsiManager._buildRequest(this.project, null, (error, request) => { - this.error = error - this.request = request - done() - }) + this.ProjectEntityHandler.getAllDocs.yields(null, this.docs) + this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done) }) it('should set io to the only file', function () { - this.request.compile.rootResourcePath.should.equal('other.tex') + expect(this.request).to.have.been.calledWith( + sinon.match( + opts => opts.json.compile.rootResourcePath === 'other.tex' + ) + ) }) }) describe('with the draft option', function () { - it('should add the draft option into the request', function (done) { - this.ClsiManager._buildRequest( - this.project_id, + beforeEach(function (done) { + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, { timeout: 100, draft: true }, - (err, request) => { - if (err != null) { + done + ) + }) + + it('should add the draft option into the request', function () { + expect(this.request).to.have.been.calledWith( + sinon.match(opts => opts.json.compile.options.draft === true) + ) + }) + }) + + describe('with a failed compile', function () { + beforeEach(function (done) { + this.responseBody.compile.status = 'failure' + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + {}, + (err, status) => { + if (err) { return done(err) } - request.compile.options.draft.should.equal(true) + this.result = { status } done() } ) }) - }) - }) - describe('_postToClsi', function () { - beforeEach(function () { - this.req = { mock: 'req', compile: {} } - }) - - describe('successfully', function () { - beforeEach(function () { - this.ClsiManager._makeRequest = sinon - .stub() - .yields(null, { statusCode: 204 }, (this.body = { mock: 'foo' })) - this.ClsiManager._postToClsi( - this.project_id, - this.user_id, - this.req, - 'e2', - 'standard', - this.callback - ) + it('should call the callback with a failure status', function () { + expect(this.result.status).to.equal('failure') }) + }) - it('should send the request to the CLSI', function () { - const url = `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard` - this.ClsiManager._makeRequest - .calledWith(this.project_id, this.user_id, 'standard', 'e2', { - method: 'POST', - url, - json: this.req, + describe('with a sync conflict', function () { + beforeEach(function (done) { + this.request + .withArgs( + sinon.match(opts => opts.json.compile.options.syncType !== 'full') + ) + .yields(null, this.response, { + compile: { status: 'conflict' }, }) - .should.equal(true) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + {}, + (err, status) => { + if (err) { + return done(err) + } + this.result = { status } + done() + } + ) }) - it('should call the callback with the body and no error', function () { - this.callback.calledWith(null, this.body).should.equal(true) + it('should send two requests to CLSI', function () { + this.request.should.have.been.calledTwice + }) + + it('should call the CLSI first without syncType:full', function () { + const compileOptions = + this.request.getCall(0).args[0].json.compile.options + expect(compileOptions.syncType).to.be.undefined + }) + + it('should call the CLSI a second time with syncType:full', function () { + const compileOptions = + this.request.getCall(1).args[0].json.compile.options + expect(compileOptions.syncType).to.equal('full') + }) + + it('should call the callback with a success status', function () { + this.result.status.should.equal('success') }) }) - describe('when the CLSI returns an error', function () { - beforeEach(function () { - this.ClsiManager._makeRequest = sinon - .stub() - .yields(null, { statusCode: 500 }, (this.body = { mock: 'foo' })) - this.ClsiManager._postToClsi( - this.project_id, + describe('with an unavailable response', function () { + beforeEach(function (done) { + this.request.onCall(0).yields(null, this.response, { + compile: { status: 'unavailable' }, + }) + this.ClsiManager.sendRequest( + this.project._id, this.user_id, - this.req, - 'e2', - 'standard', + {}, + (err, status) => { + if (err) { + return done(err) + } + this.result = { status } + done() + } + ) + }) + + it('should send two requests to CLSI', function () { + this.request.should.have.been.calledTwice + }) + + it('should call the CLSI first without syncType:full', function () { + const compileOptions = + this.request.getCall(0).args[0].json.compile.options + expect(compileOptions.syncType).to.be.undefined + }) + + it('should call the CLSI a second time with syncType:full', function () { + const compileOptions = + this.request.getCall(1).args[0].json.compile.options + expect(compileOptions.syncType).to.equal('full') + }) + + it('should clear the CLSI server id cookie', function () { + expect(this.ClsiCookieManager.clearServerId).to.have.been.calledWith( + this.project._id, + this.user_id + ) + }) + + it('should call the callback with a success status', function () { + expect(this.result.status).to.equal('success') + }) + }) + + describe('when the resources fail the precompile check', function () { + beforeEach(function () { + this.ClsiFormatChecker.checkRecoursesForProblems.yields( + new Error('failed') + ) + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + {}, this.callback ) }) + it('should call the callback only once', function () { + this.callback.calledOnce.should.equal(true) + }) + it('should call the callback with an error', function () { this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error)) }) }) + + describe('when a new backend is configured', function () { + beforeEach(function (done) { + this.settings.apis.clsi_new = { url: 'https://compiles.somewhere.test' } + this.ClsiManager.sendRequest( + this.project._id, + this.user_id, + { compileBackendClass: 'e2', compileGroup: 'standard' }, + err => { + if (err) { + return done(err) + } + // wait for the background task to finish + setTimeout(done, 0) + } + ) + }) + + it('makes a request to the new backend', function () { + expect(this.request).to.have.been.calledTwice + expect(this.request).to.have.been.calledWith( + sinon.match({ + url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`, + }) + ) + expect(this.request).to.have.been.calledWith( + sinon.match({ + url: `${this.settings.apis.clsi_new.url}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`, + }) + ) + }) + }) + }) + + describe('sendExternalRequest', function () { + beforeEach(function () { + this.submissionId = 'submission-id' + this.clsiRequest = 'mock-request' + this.ClsiCookieManager.getCookieJar.yields(null, this.jar, 'clsi3') + }) + + describe('with a successful compile', function () { + beforeEach(function (done) { + this.outputFiles = [ + { + url: `/project/${this.submissionId}/build/1234/output/output.pdf`, + path: 'output.pdf', + type: 'pdf', + build: 1234, + }, + { + url: `/project/${this.submissionId}/build/1234/output/output.log`, + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + this.responseBody.compile.outputFiles = this.outputFiles.map( + outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url }) + ) + this.ClsiManager.sendExternalRequest( + this.submissionId, + this.clsiRequest, + { compileBackendClass: 'e2', compileGroup: 'standard' }, + (err, status, outputFiles) => { + if (err) { + return done(err) + } + this.result = { status, outputFiles } + done() + } + ) + }) + + it('should send the request to the CLSI', function () { + this.request.should.have.been.calledWith({ + url: `${CLSI_URL}/project/${this.submissionId}/compile?compileBackendClass=e2&compileGroup=standard`, + method: 'POST', + json: this.clsiRequest, + jar: this.jar, + }) + }) + + it('should call the callback with the status and output files', function () { + expect(this.result.status).to.equal('success') + expect(this.result.outputFiles.map(f => f.path)).to.have.members( + this.outputFiles.map(f => f.path) + ) + }) + }) + + describe('with a failed compile', function () { + beforeEach(function (done) { + this.responseBody.compile.status = 'failure' + this.ClsiManager.sendExternalRequest( + this.submissionId, + this.clsiRequest, + {}, + (err, status) => { + if (err) { + return done(err) + } + this.result = { status } + done() + } + ) + }) + + it('should call the callback with a failure status', function () { + expect(this.result.status).to.equal('failure') + }) + }) + + describe('when the resources fail the precompile check', function () { + beforeEach(function (done) { + this.ClsiFormatChecker.checkRecoursesForProblems.yields( + new Error('failed') + ) + this.responseBody.compile.status = 'failure' + this.ClsiManager.sendExternalRequest( + this.submissionId, + this.clsiRequest, + {}, + err => { + this.err = err + done() + } + ) + }) + + it('should call the callback with an error', function () { + expect(this.err).to.be.instanceof(Error) + }) + }) + }) + + describe('deleteAuxFiles', function () { + describe('with the standard compileGroup', function () { + beforeEach(function (done) { + this.ClsiManager.deleteAuxFiles( + this.project._id, + this.user_id, + { compileBackendClass: 'e2', compileGroup: 'standard' }, + 'node-1', + done + ) + }) + + it('should call the delete method in the standard CLSI', function () { + this.request.should.have.been.calledWith({ + url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}?compileBackendClass=e2&compileGroup=standard`, + method: 'DELETE', + qs: { + compileGroup: 'standard', + compileBackendClass: 'e2', + clsiserverid: 'node-1', + }, + }) + }) + + it('should clear the project state from the docupdater', function () { + this.DocumentUpdaterHandler.clearProjectState + .calledWith(this.project._id) + .should.equal(true) + }) + + it('should clear the clsi persistance', function () { + this.ClsiCookieManager.clearServerId + .calledWith(this.project._id, this.user_id) + .should.equal(true) + }) + + it('should not add a cookie jar', function () { + expect(this.request).to.have.been.calledWith( + sinon.match(opts => opts.jar == null) + ) + }) + + it('should not persist a cookie on response', function () { + expect(this.ClsiCookieManager.setServerId).not.to.have.been.called + }) + }) }) describe('wordCount', function () { - beforeEach(function () { - this.ClsiManager._makeRequestWithClsiServerId = sinon - .stub() - .yields(null, { statusCode: 200 }, (this.body = { mock: 'foo' })) - this.ClsiManager._buildRequest = sinon.stub().yields( - null, - (this.req = { - compile: { rootResourcePath: 'rootfile.text', options: {} }, - }) - ) - }) - describe('with root file', function () { - beforeEach(function () { + beforeEach(function (done) { this.ClsiManager.wordCount( - this.project_id, + this.project._id, this.user_id, false, { compileBackendClass: 'e2', compileGroup: 'standard' }, 'node-1', - this.callback + done ) }) it('should call wordCount with root file', function () { - this.ClsiManager._makeRequestWithClsiServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - { - method: 'GET', - url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`, - qs: { - file: 'rootfile.text', - image: undefined, - }, - json: true, - }, - 'node-1' - ) - .should.equal(true) + expect(this.request).to.have.been.calledWith({ + url: `http://clsi.example.com/project/${this.project._id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`, + method: 'GET', + qs: { + compileGroup: 'standard', + compileBackendClass: 'e2', + clsiserverid: 'node-1', + file: 'main.tex', + image: 'mock-image-name', + }, + json: true, + }) }) - it('should call the callback', function () { - this.callback.called.should.equal(true) + it('should not persist a cookie on response', function () { + expect(this.ClsiCookieManager.setServerId).not.to.have.been.called }) }) describe('with param file', function () { - beforeEach(function () { + beforeEach(function (done) { this.ClsiManager.wordCount( - this.project_id, + this.project._id, this.user_id, - 'main.tex', + 'other.tex', { compileBackendClass: 'e2', compileGroup: 'standard' }, 'node-2', - this.callback + done ) }) it('should call wordCount with param file', function () { - this.ClsiManager._makeRequestWithClsiServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - { - method: 'GET', - url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`, - qs: { file: 'main.tex', image: undefined }, - json: true, - }, - 'node-2' - ) - .should.equal(true) - }) - }) - - describe('with image', function () { - beforeEach(function () { - this.req.compile.options.imageName = this.image = - 'example.com/mock/image' - this.ClsiManager.wordCount( - this.project_id, - this.user_id, - 'main.tex', - { compileBackendClass: 'e2', compileGroup: 'standard' }, - 'node-3', - this.callback - ) + expect(this.request).to.have.been.calledWith({ + url: `http://clsi.example.com/project/${this.project._id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`, + method: 'GET', + qs: { + compileGroup: 'standard', + compileBackendClass: 'e2', + clsiserverid: 'node-2', + file: 'other.tex', + image: 'mock-image-name', + }, + json: true, + }) }) - it('should call wordCount with file and image', function () { - this.ClsiManager._makeRequestWithClsiServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - { - method: 'GET', - url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`, - qs: { file: 'main.tex', image: this.image }, - json: true, - }, - 'node-3' - ) - .should.equal(true) + it('should not persist a cookie on response', function () { + expect(this.ClsiCookieManager.setServerId).not.to.have.been.called }) }) }) - - describe('_makeRequest', function () { - beforeEach(function () { - this.response = { there: 'something' } - this.request.callsArgWith(1, null, this.response) - this.opts = { - method: 'SOMETHIGN', - url: 'http://a place on the web', - } - }) - - it('should process a request with a cookie jar', function (done) { - this.ClsiManager._makeRequest( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - () => { - const args = this.request.args[0] - args[0].method.should.equal(this.opts.method) - args[0].url.should.equal(this.opts.url) - args[0].jar.should.equal(this.jar) - done() - } - ) - }) - - it('should set the cookie again on response as it might have changed', function (done) { - this.ClsiManager._makeRequest( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - () => { - this.ClsiCookieManager.setServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.response - ) - .should.equal(true) - done() - } - ) - }) - }) - - describe('_makeRequestWithClsiServerId', function () { - beforeEach(function () { - this.response = { statusCode: 200 } - this.request.yields(null, this.response) - this.opts = { - method: 'GET', - url: 'http://clsi', - } - }) - - describe('with a regular request', function () { - it('should process a request with a cookie jar', function (done) { - this.ClsiManager._makeRequestWithClsiServerId( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - undefined, - err => { - if (err) return done(err) - const args = this.request.args[0] - args[0].method.should.equal(this.opts.method) - args[0].url.should.equal(this.opts.url) - args[0].jar.should.equal(this.jar) - expect(args[0].qs).to.not.exist - done() - } - ) - }) - - it('should persist the cookie from the response', function (done) { - this.ClsiManager._makeRequestWithClsiServerId( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - undefined, - err => { - if (err) return done(err) - this.ClsiCookieManager.setServerId - .calledWith( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.response - ) - .should.equal(true) - done() - } - ) - }) - }) - - describe('with a persistent request', function () { - it('should not add a cookie jar', function (done) { - this.ClsiManager._makeRequestWithClsiServerId( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - 'node-1', - err => { - if (err) return done(err) - const requestOpts = this.request.args[0][0] - expect(requestOpts.method).to.equal(this.opts.method) - expect(requestOpts.url).to.equal(this.opts.url) - expect(requestOpts.jar).to.not.exist - expect(requestOpts.qs).to.deep.equal({ - clsiserverid: 'node-1', - compileGroup: 'standard', - compileBackendClass: 'e2', - }) - done() - } - ) - }) - - it('should not persist a cookie on response', function (done) { - this.ClsiManager._makeRequestWithClsiServerId( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - 'node-1', - err => { - if (err) return done(err) - expect(this.ClsiCookieManager.setServerId.called).to.equal(false) - done() - } - ) - }) - }) - }) - - describe('_makeGoogleCloudRequest', function () { - beforeEach(function () { - this.settings.apis.clsi_new = { url: 'https://compiles.somewhere.test' } - this.response = { there: 'something' } - this.request.callsArgWith(1, null, this.response) - this.opts = { - url: this.ClsiManager._getCompilerUrl( - 'e2', - 'standard', - this.project_id - ), - } - }) - - it('should change the domain on the url', function (done) { - this.ClsiManager._makeNewBackendRequest( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - () => { - const args = this.request.args[0] - args[0].url.should.equal( - `https://compiles.somewhere.test/project/${this.project_id}?compileBackendClass=e2&compileGroup=standard` - ) - done() - } - ) - }) - - it('should not make a request if there is not clsi_new url', function (done) { - this.settings.apis.clsi_new = undefined - this.ClsiManager._makeNewBackendRequest( - this.project_id, - this.user_id, - 'standard', - 'e2', - this.opts, - err => { - expect(err).to.equal(undefined) - this.request.callCount.should.equal(0) - done() - } - ) - }) - }) }) + +function _makeResources(project, docs, files) { + const resources = [] + for (const [path, doc] of Object.entries(docs)) { + resources.push({ + path: path.replace(/^\//, ''), + content: doc.lines.join('\n'), + }) + } + for (const [path, file] of Object.entries(files)) { + resources.push({ + path: path.replace(/^\//, ''), + url: `${FILESTORE_URL}/project/${project._id}/file/${file._id}`, + modified: file.created.getTime(), + }) + } + return resources +}