From 1462e17f0cf952acb6bb0fff881b6879ec0a3646 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:25:13 +0100 Subject: [PATCH 1/4] put the build id in the output file urls the url attribute will now give the preferred location for accessing the output file, without the url having to be constructed by the web client --- services/clsi/app/coffee/CompileController.coffee | 5 ++++- services/clsi/test/unit/coffee/CompileControllerTests.coffee | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index 5cc9a5e2f4..478d91bfa7 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -33,7 +33,10 @@ module.exports = CompileController = status: status error: error?.message or error outputFiles: outputFiles.map (file) -> - url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}" + url: + "#{Settings.apis.clsi.url}/project/#{request.project_id}" + + (if file.build? then "/build/#{file.build}" else "") + + "/output/#{file.path}" path: file.path type: file.type build: file.build diff --git a/services/clsi/test/unit/coffee/CompileControllerTests.coffee b/services/clsi/test/unit/coffee/CompileControllerTests.coffee index d8d6e8620f..7693799e4c 100644 --- a/services/clsi/test/unit/coffee/CompileControllerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileControllerTests.coffee @@ -75,7 +75,7 @@ describe "CompileController", -> status: "success" error: null outputFiles: @output_files.map (file) => - url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}" + url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/build/#{file.build}/output/#{file.path}" path: file.path type: file.type build: file.build From df641549c4e8828955e75dcb77855a1ce4cf9956 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:29:26 +0100 Subject: [PATCH 2/4] add per-user routes and methods --- services/clsi/app.coffee | 12 ++++- .../clsi/app/coffee/CompileController.coffee | 11 ++-- .../clsi/app/coffee/CompileManager.coffee | 50 ++++++++++++------- .../unit/coffee/CompileControllerTests.coffee | 12 ++--- .../unit/coffee/CompileManagerTests.coffee | 25 +++++----- 5 files changed, 68 insertions(+), 42 deletions(-) diff --git a/services/clsi/app.coffee b/services/clsi/app.coffee index 9f54f4a016..cfe8ebccdc 100644 --- a/services/clsi/app.coffee +++ b/services/clsi/app.coffee @@ -42,8 +42,8 @@ app.param 'project_id', (req, res, next, project_id) -> else next new Error("invalid project id") -app.param 'user_id', (req, res, next, project_id) -> - if project_id?.match /^[a-zA-Z0-9_-]+$/ +app.param 'user_id', (req, res, next, user_id) -> + if user_id?.match /^[0-9a-f]{24}$/ next() else next new Error("invalid user id") @@ -63,6 +63,14 @@ app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf app.get "/project/:project_id/wordcount", CompileController.wordcount app.get "/project/:project_id/status", CompileController.status +# Per-user containers +app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile +# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache + +app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode +app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf +app.get "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount + ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks" # create a static server which does not allow access to any symlinks diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index 478d91bfa7..7e1f07890a 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -11,6 +11,7 @@ module.exports = CompileController = RequestParser.parse req.body, (error, request) -> return next(error) if error? request.project_id = req.params.project_id + request.user_id = req.params.user_id if req.params.user_id? ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> return next(error) if error? CompileManager.doCompile request, (error, outputFiles = []) -> @@ -35,6 +36,7 @@ module.exports = CompileController = outputFiles: outputFiles.map (file) -> url: "#{Settings.apis.clsi.url}/project/#{request.project_id}" + + (if request.user_id? then "/user/#{request.user_id}" else "") + (if file.build? then "/build/#{file.build}" else "") + "/output/#{file.path}" path: file.path @@ -52,8 +54,9 @@ module.exports = CompileController = line = parseInt(req.query.line, 10) column = parseInt(req.query.column, 10) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromCode project_id, file, line, column, (error, pdfPositions) -> + CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) -> return next(error) if error? res.send JSON.stringify { pdf: pdfPositions @@ -64,8 +67,9 @@ module.exports = CompileController = h = parseFloat(req.query.h) v = parseFloat(req.query.v) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromPdf project_id, page, h, v, (error, codePositions) -> + CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) -> return next(error) if error? res.send JSON.stringify { code: codePositions @@ -74,10 +78,11 @@ module.exports = CompileController = wordcount: (req, res, next = (error) ->) -> file = req.query.file || "main.tex" project_id = req.params.project_id + user_id = req.params.user_id image = req.query.image logger.log {image, file, project_id}, "word count request" - CompileManager.wordcount project_id, file, image, (error, result) -> + CompileManager.wordcount project_id, user_id, file, image, (error, result) -> return next(error) if error? res.send JSON.stringify { texcount: result diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index 41377abaf8..cd332a4f9f 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -15,17 +15,23 @@ commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" CommandRunner = require(commandRunner) +getCompileName = (project_id, user_id) -> + if user_id? then "#{project_id}-#{user_id}" else project_id + +getCompileDir = (project_id, user_id) -> + Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) + module.exports = CompileManager = doCompile: (request, callback = (error, outputFiles) ->) -> - compileDir = Path.join(Settings.path.compilesDir, request.project_id) + compileDir = getCompileDir(request.project_id, request.user_id) timer = new Metrics.Timer("write-to-disk") - logger.log project_id: request.project_id, "starting doCompile" + logger.log project_id: request.project_id, user_id: request.user_id, "starting compile" ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) -> if error? - logger.err err:error, project_id: request.project_id, "error writing resources to disk" + logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" return callback(error) - logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk" + logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" timer.done() injectDraftModeIfRequired = (callback) -> @@ -42,7 +48,8 @@ module.exports = CompileManager = tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test Metrics.inc("compiles") Metrics.inc("compiles-with-image.#{tag}") - LatexRunner.runLatex request.project_id, { + compileName = getCompileName(request.project_id, request.user_id) + LatexRunner.runLatex compileName, { directory: compileDir mainFile: request.rootResourcePath compiler: request.compiler @@ -58,7 +65,7 @@ module.exports = CompileManager = loadavg = os.loadavg?() Metrics.gauge("load-avg", loadavg[0]) if loadavg? ts = timer.done() - logger.log {project_id: request.project_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" + logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" if stats?["latex-runs"] > 0 Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]) if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 @@ -106,24 +113,28 @@ module.exports = CompileManager = else callback(null, true) # directory exists - syncFromCode: (project_id, file_name, line, column, callback = (error, pdfPositions) ->) -> + syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) -> # If LaTeX was run in a virtual environment, the file path that synctex expects # might not match the file path on the host. The .synctex.gz file however, will be accessed # wherever it is on the host. - base_dir = Settings.path.synctexBaseDir(project_id) + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) file_path = base_dir + "/" + file_name - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" + logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" callback null, CompileManager._parseSynctexFromCodeOutput(stdout) - syncFromPdf: (project_id, page, h, v, callback = (error, filePositions) ->) -> - base_dir = Settings.path.synctexBaseDir(project_id) - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) -> + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" + logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) _runSynctex: (args, callback = (error, stdout) ->) -> @@ -162,19 +173,20 @@ module.exports = CompileManager = } return results - wordcount: (project_id, file_name, image, callback = (error, pdfPositions) ->) -> - logger.log project_id:project_id, file_name:file_name, image:image, "running wordcount" + wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> + logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" file_path = "$COMPILE_DIR/" + file_name command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"] - directory = Path.join(Settings.path.compilesDir, project_id) + directory = getCompileDir(project_id, user_id) timeout = 10 * 1000 + compileName = getCompileName(project_id, user_id) - CommandRunner.run project_id, command, directory, image, timeout, (error) -> + CommandRunner.run compileName, command, directory, image, timeout, (error) -> return callback(error) if error? try stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") catch err - logger.err err:err, command:command, directory:directory, project_id:project_id, "error reading word count output" + logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output" return callback(err) callback null, CompileManager._parseWordcountFromOutput(stdout) diff --git a/services/clsi/test/unit/coffee/CompileControllerTests.coffee b/services/clsi/test/unit/coffee/CompileControllerTests.coffee index 7693799e4c..1fc6a99bd5 100644 --- a/services/clsi/test/unit/coffee/CompileControllerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileControllerTests.coffee @@ -146,12 +146,12 @@ describe "CompileController", -> column: @column.toString() @res.send = sinon.stub() - @CompileManager.syncFromCode = sinon.stub().callsArgWith(4, null, @pdfPositions = ["mock-positions"]) + @CompileManager.syncFromCode = sinon.stub().callsArgWith(5, null, @pdfPositions = ["mock-positions"]) @CompileController.syncFromCode @req, @res, @next it "should find the corresponding location in the PDF", -> @CompileManager.syncFromCode - .calledWith(@project_id, @file, @line, @column) + .calledWith(@project_id, undefined, @file, @line, @column) .should.equal true it "should return the positions", -> @@ -175,12 +175,12 @@ describe "CompileController", -> v: @v.toString() @res.send = sinon.stub() - @CompileManager.syncFromPdf = sinon.stub().callsArgWith(4, null, @codePositions = ["mock-positions"]) + @CompileManager.syncFromPdf = sinon.stub().callsArgWith(5, null, @codePositions = ["mock-positions"]) @CompileController.syncFromPdf @req, @res, @next it "should find the corresponding location in the code", -> @CompileManager.syncFromPdf - .calledWith(@project_id, @page, @h, @v) + .calledWith(@project_id, undefined, @page, @h, @v) .should.equal true it "should return the positions", -> @@ -201,12 +201,12 @@ describe "CompileController", -> image: @image = "example.com/image" @res.send = sinon.stub() - @CompileManager.wordcount = sinon.stub().callsArgWith(3, null, @texcount = ["mock-texcount"]) + @CompileManager.wordcount = sinon.stub().callsArgWith(4, null, @texcount = ["mock-texcount"]) @CompileController.wordcount @req, @res, @next it "should return the word count of a file", -> @CompileManager.wordcount - .calledWith(@project_id, @file, @image) + .calledWith(@project_id, undefined, @file, @image) .should.equal true it "should return the texcount info", -> diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index 55f5cc5200..0ba62b95c1 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -43,11 +43,12 @@ describe "CompileManager", -> resources: @resources = "mock-resources" rootResourcePath: @rootResourcePath = "main.tex" project_id: @project_id = "project-id-123" + user_id: @user_id = "1234" compiler: @compiler = "pdflatex" timeout: @timeout = 42000 imageName: @image = "example.com/image" @Settings.compileDir = "compiles" - @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}" + @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3) @LatexRunner.runLatex = sinon.stub().callsArg(2) @OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files) @@ -65,7 +66,7 @@ describe "CompileManager", -> it "should run LaTeX", -> @LatexRunner.runLatex - .calledWith(@project_id, { + .calledWith("#{@project_id}-#{@user_id}", { directory: @compileDir mainFile: @rootResourcePath compiler: @compiler @@ -150,17 +151,17 @@ describe "CompileManager", -> @column = 3 @file_name = "main.tex" @child_process.execFile = sinon.stub() - @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}" + @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" describe "syncFromCode", -> beforeEach -> @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "") - @CompileManager.syncFromCode @project_id, @file_name, @line, @column, @callback + @CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" - file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" + file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}" @child_process.execFile .calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000) .should.equal true @@ -178,12 +179,12 @@ describe "CompileManager", -> describe "syncFromPdf", -> beforeEach -> - @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") - @CompileManager.syncFromPdf @project_id, @page, @h, @v, @callback + @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") + @CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" @child_process.execFile .calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000) .should.equal true @@ -209,15 +210,15 @@ describe "CompileManager", -> @Settings.path.compilesDir = "/local/compile/directory" @image = "example.com/image" - @CompileManager.wordcount @project_id, @file_name, @image, @callback + @CompileManager.wordcount @project_id, @user_id, @file_name, @image, @callback it "should run the texcount command", -> - @directory = "#{@Settings.path.compilesDir}/#{@project_id}" + @directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @file_path = "$COMPILE_DIR/#{@file_name}" @command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run - .calledWith(@project_id, @command, @directory, @image, @timeout) + .calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout) .should.equal true it "should call the callback with the parsed output", -> From 5367bc22e579091d9df96abe86747429f35cf1ee Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:31:44 +0100 Subject: [PATCH 3/4] add per-user routes for clearing cache and extend expiry methods this adds separate functionality for clearing the cache (assets and database) and the project compile directory for a specific user --- services/clsi/app.coffee | 2 +- .../clsi/app/coffee/CompileController.coffee | 2 +- .../clsi/app/coffee/CompileManager.coffee | 27 ++++++++++++++-- .../coffee/ProjectPersistenceManager.coffee | 32 ++++++++++++------- .../unit/coffee/CompileManagerTests.coffee | 10 +++--- .../ProjectPersistenceManagerTests.coffee | 12 ++++--- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/services/clsi/app.coffee b/services/clsi/app.coffee index cfe8ebccdc..6a8d73fe1c 100644 --- a/services/clsi/app.coffee +++ b/services/clsi/app.coffee @@ -65,7 +65,7 @@ app.get "/project/:project_id/status", CompileController.status # Per-user containers app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile -# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache +app.delete "/project/:project_id/user/:user_id", CompileController.clearCache app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index 7e1f07890a..d4dddd4d82 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -45,7 +45,7 @@ module.exports = CompileController = } clearCache: (req, res, next = (error) ->) -> - ProjectPersistenceManager.clearProject req.params.project_id, (error) -> + ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> return next(error) if error? res.sendStatus(204) # No content diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index cd332a4f9f..bb93dbdb9b 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -9,7 +9,9 @@ Metrics = require "./Metrics" child_process = require "child_process" DraftModeManager = require "./DraftModeManager" fs = require("fs") +fse = require "fs-extra" os = require("os") +async = require "async" commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" @@ -76,12 +78,12 @@ module.exports = CompileManager = OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> callback null, newOutputFiles - clearProject: (project_id, _callback = (error) ->) -> + clearProject: (project_id, user_id, _callback = (error) ->) -> callback = (error) -> _callback(error) _callback = () -> - compileDir = Path.join(Settings.path.compilesDir, project_id) + compileDir = getCompileDir(project_id, user_id) CompileManager._checkDirectory compileDir, (err, exists) -> return callback(err) if err? @@ -100,6 +102,27 @@ module.exports = CompileManager = else return callback(new Error("rm -r #{compileDir} failed: #{stderr}")) + _findAllDirs: (callback = (error, allDirs) ->) -> + root = Settings.path.compilesDir + fs.readdir root, (err, files) -> + return callback(err) if err? + allDirs = (Path.join(root, file) for file in files) + callback(null, allDirs) + + clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) -> + now = Date.now() + # action for each directory + expireIfNeeded = (checkDir, cb) -> + fs.stat checkDir, (err, stats) -> + return cb() if err? # ignore errors checking directory + age = now - stats.mtime + hasExpired = (age > max_cache_age_ms) + if hasExpired then fse.remove(checkDir, cb) else cb() + # iterate over all project directories + CompileManager._findAllDirs (error, allDirs) -> + return callback() if error? + async.eachSeries allDirs, expireIfNeeded, callback + _checkDirectory: (compileDir, callback = (error, exists) ->) -> fs.lstat compileDir, (err, stats) -> if err?.code is 'ENOENT' diff --git a/services/clsi/app/coffee/ProjectPersistenceManager.coffee b/services/clsi/app/coffee/ProjectPersistenceManager.coffee index f70f43cab7..200d977fc6 100644 --- a/services/clsi/app/coffee/ProjectPersistenceManager.coffee +++ b/services/clsi/app/coffee/ProjectPersistenceManager.coffee @@ -27,21 +27,30 @@ module.exports = ProjectPersistenceManager = jobs = for project_id in (project_ids or []) do (project_id) -> (callback) -> - ProjectPersistenceManager.clearProject project_id, (err) -> + ProjectPersistenceManager.clearProjectFromCache project_id, (err) -> if err? logger.error err: err, project_id: project_id, "error clearing project" callback() - async.series jobs, callback - - clearProject: (project_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project" - CompileManager.clearProject project_id, (error) -> - return callback(error) if error? - UrlCache.clearProject project_id, (error) -> + async.series jobs, (error) -> return callback(error) if error? - ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> - return callback(error) if error? - callback() + CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) -> + callback() # ignore any errors from deleting directories + + clearProject: (project_id, user_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project for user" + CompileManager.clearProject project_id, user_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager.clearProjectFromCache project_id, (error) -> + return callback(error) if error? + callback() + + clearProjectFromCache: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project from cache" + UrlCache.clearProject project_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> + return callback(error) if error? + callback() _clearProjectFromDatabase: (project_id, callback = (error) ->) -> db.Project.destroy(where: {project_id: project_id}) @@ -54,5 +63,4 @@ module.exports = ProjectPersistenceManager = callback null, projects.map((project) -> project.project_id) ).error callback - logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout" diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index 0ba62b95c1..611ed11b01 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -105,12 +105,12 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.emit "close", 0 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback", -> @@ -124,13 +124,13 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.stderr.emit "data", @error = "oops" @proc.emit "close", 1 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback with an error from the stderr", -> @@ -138,7 +138,7 @@ describe "CompileManager", -> .calledWith(new Error()) .should.equal true - @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}" + @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}" describe "syncing", -> beforeEach -> diff --git a/services/clsi/test/unit/coffee/ProjectPersistenceManagerTests.coffee b/services/clsi/test/unit/coffee/ProjectPersistenceManagerTests.coffee index f8c78ef783..69bfd4fa5e 100644 --- a/services/clsi/test/unit/coffee/ProjectPersistenceManagerTests.coffee +++ b/services/clsi/test/unit/coffee/ProjectPersistenceManagerTests.coffee @@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", -> "./db": @db = {} @callback = sinon.stub() @project_id = "project-id-123" + @user_id = "1234" describe "clearExpiredProjects", -> beforeEach -> @@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", -> "project-id-2" ] @ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids) - @ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1) + @ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1) + @CompileManager.clearExpiredProjects = sinon.stub().callsArg(1) @ProjectPersistenceManager.clearExpiredProjects @callback it "should clear each expired project", -> for project_id in @project_ids - @ProjectPersistenceManager.clearProject + @ProjectPersistenceManager.clearProjectFromCache .calledWith(project_id) .should.equal true @@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", -> beforeEach -> @ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1) @UrlCache.clearProject = sinon.stub().callsArg(1) - @CompileManager.clearProject = sinon.stub().callsArg(1) - @ProjectPersistenceManager.clearProject @project_id, @callback + @CompileManager.clearProject = sinon.stub().callsArg(2) + @ProjectPersistenceManager.clearProject @project_id, @user_id, @callback it "should clear the project from the database", -> @ProjectPersistenceManager._clearProjectFromDatabase @@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", -> it "should clear the project compile folder", -> @CompileManager.clearProject - .calledWith(@project_id) + .calledWith(@project_id, @user_id) .should.equal true it "should call the callback", -> From d3395ef763bcc659bcec38528d5aad61a572b474 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 31 May 2016 16:12:55 +0100 Subject: [PATCH 4/4] log user_id when clearing project --- services/clsi/app/coffee/ProjectPersistenceManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/clsi/app/coffee/ProjectPersistenceManager.coffee b/services/clsi/app/coffee/ProjectPersistenceManager.coffee index 200d977fc6..403043fac9 100644 --- a/services/clsi/app/coffee/ProjectPersistenceManager.coffee +++ b/services/clsi/app/coffee/ProjectPersistenceManager.coffee @@ -37,7 +37,7 @@ module.exports = ProjectPersistenceManager = callback() # ignore any errors from deleting directories clearProject: (project_id, user_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project for user" + logger.log project_id: project_id, user_id:user_id, "clearing project for user" CompileManager.clearProject project_id, user_id, (error) -> return callback(error) if error? ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->