From 07e53bbb2fef5eb7e92a717e50490426583ca683 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 31 Mar 2015 10:35:33 +0100 Subject: [PATCH 1/8] handle error when listing bucket --- services/filestore/app/coffee/S3PersistorManager.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index ef1824491e..d5cb06074e 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -104,6 +104,9 @@ module.exports = secret: settings.filestore.s3.secret bucket: bucketName s3Client.list prefix:key, (err, data)-> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" + return callback(err) keys = _.map data.Contents, (entry)-> return entry.Key s3Client.deleteMultiple keys, callback From b3432f016cdeb7fe0a1010385d8bd799e1610913 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 9 Apr 2015 17:14:47 +0100 Subject: [PATCH 2/8] allow proxying of template images though filestore --- services/filestore/app.coffee | 1 + services/filestore/app/coffee/FileConverter.coffee | 2 +- services/filestore/app/coffee/KeyBuilder.coffee | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 293f5da8e8..f42cf4f1b4 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -74,6 +74,7 @@ app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileContro app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile app.get "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile +app.get "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile app.get "/heapdump", (req, res)-> diff --git a/services/filestore/app/coffee/FileConverter.coffee b/services/filestore/app/coffee/FileConverter.coffee index 0d6eb0d9f3..da10711f99 100644 --- a/services/filestore/app/coffee/FileConverter.coffee +++ b/services/filestore/app/coffee/FileConverter.coffee @@ -40,7 +40,7 @@ module.exports = args = "nice convert -flatten -background white -density 300 -define pdf:fit-page=#{width} #{sourcePath} -resize #{width} #{destPath}" exec args, childProcessOpts, (err, stdout, stderr)-> if err? - logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to preview" + logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to thumbnail" else logger.log sourcePath:sourcePath, destPath:destPath, "finished thumbnailing file" callback(err, destPath) diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 71f6c05785..03a65a8337 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -24,8 +24,8 @@ module.exports = next() templateFileKey: (req, res, next)-> - {template_id, format, version} = req.params - req.key = "#{template_id}/v/#{version}/#{format}" + {template_id, format, version, sub_type} = req.params + req.key = "#{template_id}/v/#{version}/#{format}/#{sub_type}" req.bucket = settings.filestore.stores.template_files req.version = version opts = req.query From 7c2e4b00ed1de139c24fd6b9c49e02ad15e67d6e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 13 Apr 2015 12:01:49 +0100 Subject: [PATCH 3/8] no file subtype can be an issue, added a check on it in the keybuilder --- services/filestore/app/coffee/KeyBuilder.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 03a65a8337..9b2a101e6b 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -25,7 +25,9 @@ module.exports = templateFileKey: (req, res, next)-> {template_id, format, version, sub_type} = req.params - req.key = "#{template_id}/v/#{version}/#{format}/#{sub_type}" + req.key = "#{template_id}/v/#{version}/#{format}" + if sub_type? + req.key = "#{req.key}/#{sub_type}" req.bucket = settings.filestore.stores.template_files req.version = version opts = req.query From 33e654accf91dba9481ecffc00faa1cd7e3cd05e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 22 Apr 2015 18:27:34 +0100 Subject: [PATCH 4/8] get the logging locally to work with bunyan --- services/filestore/Gruntfile.coffee | 9 +++++++-- services/filestore/package.json | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/filestore/Gruntfile.coffee b/services/filestore/Gruntfile.coffee index c91476c201..ebf94e2fc9 100644 --- a/services/filestore/Gruntfile.coffee +++ b/services/filestore/Gruntfile.coffee @@ -52,7 +52,10 @@ module.exports = (grunt) -> options: ext:"*.coffee" - + execute: + app: + src: "app.js" + concurrent: dev: tasks: ['nodemon', 'watch'] @@ -79,13 +82,15 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-concurrent' grunt.loadNpmTasks 'grunt-mocha-test' grunt.loadNpmTasks 'grunt-forever' + grunt.loadNpmTasks 'grunt-bunyan' + grunt.loadNpmTasks 'grunt-execute' grunt.registerTask "test:unit", ["coffee", "mochaTest:unit"] grunt.registerTask "test:acceptence", ["coffee", "mochaTest:acceptence"] grunt.registerTask "test:acceptance", ["test:acceptence"] grunt.registerTask "ci", "test:unit" - grunt.registerTask 'default', ['coffee', 'concurrent'] + grunt.registerTask 'default', ['coffee', 'bunyan','execute'] grunt.registerTask "compile", "coffee" grunt.registerTask "install", "compile" diff --git a/services/filestore/package.json b/services/filestore/package.json index c2ba0e4952..dd84289f30 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -8,8 +8,11 @@ }, "dependencies": { "async": "~0.2.10", + "bunyan": "^1.3.5", "coffee-script": "~1.7.1", "express": "~3.4.8", + "grunt-bunyan": "^0.5.0", + "grunt-execute": "^0.2.2", "grunt-mocha-test": "~0.8.2", "heapdump": "^0.3.2", "knox": "~0.9.1", From d5f98c9800945fd2adefb0b8d2564edd26abd331 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 30 Apr 2015 15:04:55 +0100 Subject: [PATCH 5/8] make startup message consistent --- services/filestore/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index f42cf4f1b4..aaacf704b2 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -126,7 +126,7 @@ beginShutdown = () -> logger.log "server will stop accepting connections" server.listen port, host, -> - logger.log("filestore listening on #{host}:#{port}") + logger.info "Filestore starting up, listening on #{host}:#{port}" process.on 'SIGTERM', () -> logger.log("filestore got SIGTERM, shutting down gracefully") From 7e9f8b885b90f396904657f738d9fe5b0c1b56bc Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 8 May 2015 15:15:48 +0100 Subject: [PATCH 6/8] add support for public files --- services/filestore/app.coffee | 8 ++++++++ services/filestore/app/coffee/KeyBuilder.coffee | 9 +++++++++ services/filestore/config/settings.defaults.coffee | 1 + 3 files changed, 18 insertions(+) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index aaacf704b2..11c323667d 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -77,6 +77,14 @@ app.get "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey app.get "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile + +app.get "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile +app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile + +app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.copyFile +app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile + + app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> res.send filename diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 9b2a101e6b..0b5e41525f 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -23,6 +23,15 @@ module.exports = req.bucket = settings.filestore.stores.user_files next() + publicFileKey: (req, res, next)-> + {project_id, public_file_id} = req.params + if not settings.filestore.stores.public_files? + res.status(501).send("public files not available") + else + req.key = "#{project_id}/#{public_file_id}" + req.bucket = settings.filestore.stores.public_files + next() + templateFileKey: (req, res, next)-> {template_id, format, version, sub_type} = req.params req.key = "#{template_id}/v/#{version}/#{format}" diff --git a/services/filestore/config/settings.defaults.coffee b/services/filestore/config/settings.defaults.coffee index dccefea525..f84928598e 100644 --- a/services/filestore/config/settings.defaults.coffee +++ b/services/filestore/config/settings.defaults.coffee @@ -21,6 +21,7 @@ module.exports = # Must contain full path, e.g. "/var/lib/sharelatex/data". # This path must exist, not be tmpfs and be writable to by the user sharelatex is run as. user_files: Path.resolve(__dirname + "/../user_files") + public_files: Path.resolve(__dirname + "/../public_files") template_files: Path.resolve(__dirname + "/../template_files") # if you are using S3, then fill in your S3 details below # s3: From 8ebbacee5ac806f93d8a4ff50be13808221321dd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 8 May 2015 15:19:45 +0100 Subject: [PATCH 7/8] move pipe after error handler setup --- services/filestore/app/coffee/LocalFileWriter.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/filestore/app/coffee/LocalFileWriter.coffee b/services/filestore/app/coffee/LocalFileWriter.coffee index 12d505b05b..2ecb3885df 100644 --- a/services/filestore/app/coffee/LocalFileWriter.coffee +++ b/services/filestore/app/coffee/LocalFileWriter.coffee @@ -14,7 +14,6 @@ module.exports = fsPath = @_getPath(key) logger.log fsPath:fsPath, "writing file locally" writeStream = fs.createWriteStream(fsPath) - stream.pipe writeStream writeStream.on "finish", -> timer.done() logger.log fsPath:fsPath, "finished writing file locally" @@ -25,6 +24,7 @@ module.exports = stream.on "error", (err)-> logger.log err:err, fsPath:fsPath, "problem writing file locally, with read stream" callback err + stream.pipe writeStream deleteFile: (fsPath, callback)-> logger.log fsPath:fsPath, "removing local temp file" From 9b844220929f338124914f18f3c02aa5e118f1ff Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Jun 2015 13:41:14 +0100 Subject: [PATCH 8/8] added event loop monitor --- services/filestore/app.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 11c323667d..9c744e75ab 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -13,6 +13,7 @@ streamBuffers = require("stream-buffers") Metrics = require "metrics-sharelatex" Metrics.initialize("filestore") Metrics.open_sockets.monitor(logger) +Metrics.event_loop?.monitor(logger) app.configure -> app.use express.bodyParser()