diff --git a/services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee b/services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee new file mode 100644 index 0000000000..b285d0e6b7 --- /dev/null +++ b/services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee @@ -0,0 +1,68 @@ +_ = require("lodash") +async = require("async") + +module.exports = ClsiFormatChecker = + + checkRecoursesForProblems: (resources, callback)-> + jobs = + conflictedPaths: (cb)-> + ClsiFormatChecker._checkForConflictingPaths resources, cb + + sizeCheck: (cb)-> + ClsiFormatChecker._checkDocsAreUnderSizeLimit resources, cb + + async.series jobs, (err, problems)-> + if err? + return callback(err) + + problems = _.omitBy(problems, _.isEmpty) + + if _.isEmpty(problems) + return callback() + else + callback(null, problems) + + + _checkForConflictingPaths: (resources, callback)-> + paths = _.map(resources, 'path') + + conflicts = _.filter paths, (path)-> + matchingPaths = _.filter paths, (checkPath)-> + return checkPath.indexOf(path+"/") != -1 + + return matchingPaths.length > 0 + + conflictObjects = _.map conflicts, (conflict)-> + path:conflict + + callback null, conflictObjects + + _checkDocsAreUnderSizeLimit: (resources, callback)-> + + FIVEMB = 1000 * 1000 * 5 + + totalSize = 0 + + sizedResources = _.map resources, (resource)-> + result = {path:resource.path} + if resource.content? + result.size = resource.content.replace(/\n/g).length + result.kbSize = Math.ceil(result.size / 1000) + else + result.size = 0 + totalSize += result.size + return result + + tooLarge = totalSize > FIVEMB + if !tooLarge + return callback() + else + sizedResources = _.sortBy(sizedResources, "size").reverse().slice(0, 10) + return callback(null, {resources:sizedResources, totalSize:totalSize}) + + + + + + + diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index 9af1602d2c..602a446ab4 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -7,25 +7,34 @@ ProjectEntityHandler = require("../Project/ProjectEntityHandler") logger = require "logger-sharelatex" Url = require("url") ClsiCookieManager = require("./ClsiCookieManager") - +_ = require("underscore") +async = require("async") +ClsiFormatChecker = require("./ClsiFormatChecker") module.exports = ClsiManager = - sendRequest: (project_id, options = {}, callback = (error, success) ->) -> + sendRequest: (project_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) -> ClsiManager._buildRequest project_id, options, (error, req) -> return callback(error) if error? logger.log project_id: project_id, "sending compile to CLSI" - ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) -> - if error? - logger.err err:error, project_id:project_id, "error sending request to clsi" - return callback(error) - logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI" - ClsiCookieManager._getServerId project_id, (err, clsiServerId)-> - if err? - logger.err err:err, project_id:project_id, "error getting server id" - return callback(err) - outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles) - callback(null, response?.compile?.status, outputFiles, clsiServerId) + ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)-> + if err? + logger.err err, project_id, "could not check resources for potential problems before sending to clsi" + return callback(err) + if validationProblems? + logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted" + return callback(null, "validation-problems", null, null, validationProblems) + ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) -> + if error? + logger.err err:error, project_id:project_id, "error sending request to clsi" + return callback(error) + logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI" + ClsiCookieManager._getServerId project_id, (err, clsiServerId)-> + if err? + logger.err err:err, project_id:project_id, "error getting server id" + return callback(err) + outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles) + callback(null, response?.compile?.status, outputFiles, clsiServerId) deleteAuxFiles: (project_id, options, callback = (error) ->) -> compilerUrl = @_getCompilerUrl(options?.compileGroup) diff --git a/services/web/app/coffee/Features/Compile/CompileController.coffee b/services/web/app/coffee/Features/Compile/CompileController.coffee index b50c6ffd1a..3926fb262d 100755 --- a/services/web/app/coffee/Features/Compile/CompileController.coffee +++ b/services/web/app/coffee/Features/Compile/CompileController.coffee @@ -30,7 +30,7 @@ module.exports = CompileController = if req.body?.draft options.draft = req.body.draft logger.log {options, project_id}, "got compile request" - CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits) -> + CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits, validationProblems) -> return next(error) if error? res.contentType("application/json") res.status(200).send JSON.stringify { @@ -38,6 +38,7 @@ module.exports = CompileController = outputFiles: outputFiles compileGroup: limits?.compileGroup clsiServerId:clsiServerId + validationProblems:validationProblems } downloadPdf: (req, res, next = (error) ->)-> @@ -138,10 +139,7 @@ module.exports = CompileController = # expand any url parameter passed in as {url:..., qs:...} if typeof url is "object" {url, qs} = url - if limits.compileGroup == "priority" - compilerUrl = Settings.apis.clsi_priority.url - else - compilerUrl = Settings.apis.clsi.url + compilerUrl = Settings.apis.clsi.url url = "#{compilerUrl}#{url}" logger.log url: url, "proxying to CLSI" oneMinute = 60 * 1000 diff --git a/services/web/app/coffee/Features/Compile/CompileManager.coffee b/services/web/app/coffee/Features/Compile/CompileManager.coffee index 0d8d480b7b..744d16364f 100755 --- a/services/web/app/coffee/Features/Compile/CompileManager.coffee +++ b/services/web/app/coffee/Features/Compile/CompileManager.coffee @@ -37,10 +37,10 @@ module.exports = CompileManager = return callback(error) if error? for key, value of limits options[key] = value - ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId) -> + ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId, validationProblems) -> return callback(error) if error? logger.log files: outputFiles, "output files" - callback(null, status, outputFiles, clsiServerId, limits) + callback(null, status, outputFiles, clsiServerId, limits, validationProblems) deleteAuxFiles: (project_id, callback = (error) ->) -> CompileManager.getProjectCompileLimits project_id, (error, limits) -> diff --git a/services/web/app/views/project/editor/pdf.jade b/services/web/app/views/project/editor/pdf.jade index df97ff889e..f77753213a 100644 --- a/services/web/app/views/project/editor/pdf.jade +++ b/services/web/app/views/project/editor/pdf.jade @@ -154,6 +154,27 @@ div.full-size.pdf(ng-controller="PdfController") ng-if="settings.pdfViewer == 'native'" ) + .pdf-validation-problems(ng-switch-when="validation-problems") + + .alert.alert-danger(ng-show="pdf.validation.duplicatePaths") + strong #{translate("latex_error")} + span #{translate("duplicate_paths_found")} + + + .alert.alert-danger(ng-show="pdf.validation.sizeCheck") + strong #{translate("project_too_large")} + div #{translate("project_too_large_please_reduce")} + div + li(ng-repeat="entry in pdf.validation.sizeCheck.resources") {{ '/'+entry['path'] }} - {{entry['kbSize']}}kb + + .alert.alert-danger(ng-show="pdf.validation.conflictedPaths") + div + strong #{translate("conflicting_paths_found")} + div #{translate("following_paths_conflict")} + div + li(ng-repeat="entry in pdf.validation.conflictedPaths") {{ '/'+entry['path'] }} + + .pdf-errors(ng-switch-when="errors") .alert.alert-danger(ng-show="pdf.error") diff --git a/services/web/package.json b/services/web/package.json index ff82759f65..71070cfe0e 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -30,6 +30,7 @@ "http-proxy": "^1.8.1", "jade": "~1.3.1", "ldapjs": "^1.0.0", + "lodash": "^4.13.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.3.1", "lynx": "0.1.1", "marked": "^0.3.5", @@ -57,7 +58,6 @@ "temp": "^0.8.3", "translations-sharelatex": "git+https://github.com/sharelatex/translations-sharelatex.git#master", "underscore": "1.6.0", - "underscore.string": "^3.0.2", "v8-profiler": "^5.2.3", "xml2js": "0.2.0" }, diff --git a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee index 05d99124c7..ea80977fe7 100644 --- a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee +++ b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee @@ -42,11 +42,11 @@ define [ $scope.pdf.error = false $scope.pdf.timedout = false $scope.pdf.failure = false - $scope.pdf.projectTooLarge = false $scope.pdf.url = null $scope.pdf.clsiMaintenance = false $scope.pdf.tooRecentlyCompiled = false $scope.pdf.renderingError = false + $scope.pdf.projectTooLarge = false # make a cache to look up files by name fileByPath = {} @@ -72,6 +72,9 @@ define [ else if response.status == "too-recently-compiled" $scope.pdf.view = 'errors' $scope.pdf.tooRecentlyCompiled = true + else if response.status == "validation-problems" + $scope.pdf.view = "validation-problems" + $scope.pdf.validation = response.validationProblems else if response.status == "success" $scope.pdf.view = 'pdf' $scope.shouldShowLogs = false diff --git a/services/web/public/stylesheets/app/editor/pdf.less b/services/web/public/stylesheets/app/editor/pdf.less index 353984de51..ed104a5c91 100644 --- a/services/web/public/stylesheets/app/editor/pdf.less +++ b/services/web/public/stylesheets/app/editor/pdf.less @@ -3,7 +3,7 @@ top: 58px; } -.pdf-logs, .pdf-errors, .pdf-uncompiled { +.pdf-logs, .pdf-errors, .pdf-uncompiled, .pdf-validation-problems{ padding: @line-height-computed / 2; } diff --git a/services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee b/services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee new file mode 100644 index 0000000000..d2f1c5ef3a --- /dev/null +++ b/services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee @@ -0,0 +1,151 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../../app/js/Features/Compile/ClsiFormatChecker.js" +SandboxedModule = require('sandboxed-module') + +describe "ClsiFormatChecker", -> + beforeEach -> + @ClsiFormatChecker = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings ={} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } + @project_id = "project-id" + + + describe "checkRecoursesForProblems", -> + + beforeEach -> + @resources = [{ + path: "main.tex" + content: "stuff" + }, { + path: "chapters/chapter1" + content: "other stuff" + }, { + path: "stuff/image/image.png" + url: "http:somewhere.com/project/#{@project_id}/file/1234124321312" + modified: "more stuff" + }] + + it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)-> + + @ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null) + @ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1) + @ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=> + @ClsiFormatChecker._checkForConflictingPaths.called.should.equal true + @ClsiFormatChecker._checkDocsAreUnderSizeLimit.called.should.equal true + done() + + it "should remove undefined errors", (done)-> + @ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, []) + @ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {}) + @ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=> + expect(problems).to.not.exist + expect(problems).to.not.exist + done() + + it "should keep populated arrays", (done)-> + @ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [{path:"somewhere/main.tex"}]) + @ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {}) + @ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=> + problems.conflictedPaths[0].path.should.equal "somewhere/main.tex" + expect(problems.sizeCheck).to.not.exist + done() + + it "should keep populated object", (done)-> + @ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, []) + @ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {resources:[{"a.tex"},{"b.tex"}], totalSize:1000000}) + @ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=> + problems.sizeCheck.resources.length.should.equal 2 + problems.sizeCheck.totalSize.should.equal 1000000 + expect(problems.conflictedPaths).to.not.exist + done() + + describe "_checkForConflictingPaths", -> + + beforeEach -> + + @resources.push({ + path: "chapters/chapter1.tex" + content: "other stuff" + }) + + @resources.push({ + path: "chapters.tex" + content: "other stuff" + }) + + it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)-> + @resources.push({ + path: "stuff/image" + url: "http://somwhere.com" + }) + + @ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)-> + conflictPathErrors.length.should.equal 1 + conflictPathErrors[0].path.should.equal "stuff/image" + done() + + it "should flag up when a root level file has folder with same subpath as file elsewhere", (done)-> + @resources.push({ + path: "stuff" + content: "other stuff" + }) + + @ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)-> + conflictPathErrors.length.should.equal 1 + conflictPathErrors[0].path.should.equal "stuff" + done() + + it "should not flag up when the file is a substring of a path", (done)-> + @resources.push({ + path: "stuf" + content: "other stuff" + }) + + @ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)-> + conflictPathErrors.length.should.equal 0 + done() + + + describe "_checkDocsAreUnderSizeLimit", -> + + it "should error when there is more than 5mb of data", (done)-> + + @resources.push({ + path: "massive.tex" + content: require("crypto").randomBytes(1000 * 1000 * 5).toString("hex") + }) + + while @resources.length < 20 + @resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"}) + + @ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)-> + sizeError.totalSize.should.equal 10000016 + sizeError.resources.length.should.equal 10 + sizeError.resources[0].path.should.equal "massive.tex" + sizeError.resources[0].size.should.equal 1000 * 1000 * 10 + done() + + + it "should return nothing when project is correct size", (done)-> + + @resources.push({ + path: "massive.tex" + content: require("crypto").randomBytes(1000 * 1000 * 1).toString("hex") + }) + + while @resources.length < 20 + @resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"}) + + @ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)-> + expect(sizeError).to.not.exist + done() + + + + + + + diff --git a/services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee b/services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee index 9889bcb87b..211ef9b1f9 100644 --- a/services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee @@ -12,6 +12,8 @@ describe "ClsiManager", -> getCookieJar: sinon.stub().callsArgWith(1, null, @jar) setServerId: sinon.stub().callsArgWith(2) _getServerId:sinon.stub() + @ClsiFormatChecker = + checkRecoursesForProblems:sinon.stub().callsArgWith(1) @ClsiManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = apis: @@ -27,6 +29,7 @@ describe "ClsiManager", -> "./ClsiCookieManager": @ClsiCookieManager "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } "request": @request = sinon.stub() + "./ClsiFormatChecker": @ClsiFormatChecker @project_id = "project-id" @callback = sinon.stub() @@ -323,17 +326,3 @@ describe "ClsiManager", -> - - - - - - - - - - - - - - diff --git a/services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee b/services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee index 6bae5803c4..c724798a3e 100644 --- a/services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee @@ -206,16 +206,6 @@ describe "CompileController", -> @CompileManager.getProjectCompileLimits = sinon.stub().callsArgWith(1, null, {compileGroup: "priority"}) @CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next) - it "should proxy to the priority url if the user has the feature", ()-> - @request - .calledWith( - jar:@jar - method: @req.method - url: "#{@settings.apis.clsi_priority.url}#{@url}", - timeout: 60 * 1000 - ) - .should.equal true - describe "user with standard priority via query string", -> beforeEach -> @req.query = {compileGroup: 'standard'} @@ -239,20 +229,6 @@ describe "CompileController", -> it "should bind an error handle to the request proxy", -> @proxy.on.calledWith("error").should.equal true - describe "user with priority compile via query string", -> - beforeEach -> - @req.query = {compileGroup: 'priority'} - @CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next) - - it "should proxy to the priority url if the user has the feature", ()-> - @request - .calledWith( - jar:@jar - method: @req.method - url: "#{@settings.apis.clsi_priority.url}#{@url}", - timeout: 60 * 1000 - ) - .should.equal true describe "user with non-existent priority via query string", -> beforeEach -> @@ -316,25 +292,7 @@ describe "CompileController", -> it "should bind an error handle to the request proxy", -> @proxy.on.calledWith("error").should.equal true - describe "user with priority compile", -> - beforeEach -> - @CompileManager.getProjectCompileLimits = sinon.stub().callsArgWith(1, null, {compileGroup: "priority"}) - @CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next) - it "should proxy to the priority url if the user has the feature", ()-> - @request - .calledWith( - jar:@jar - method: @req.method - url: "#{@settings.apis.clsi_priority.url}#{@url}", - timeout: 60 * 1000 - headers: { - 'Range': '123-456' - 'If-Range': 'abcdef' - 'If-Modified-Since': 'Mon, 15 Dec 2014 15:23:56 GMT' - } - ) - .should.equal true describe "user with build parameter via query string", -> beforeEach ->