diff --git a/services/filestore/Gruntfile.coffee b/services/filestore/Gruntfile.coffee index c91476c201..ebf94e2fc9 100644 --- a/services/filestore/Gruntfile.coffee +++ b/services/filestore/Gruntfile.coffee @@ -52,7 +52,10 @@ module.exports = (grunt) -> options: ext:"*.coffee" - + execute: + app: + src: "app.js" + concurrent: dev: tasks: ['nodemon', 'watch'] @@ -79,13 +82,15 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-concurrent' grunt.loadNpmTasks 'grunt-mocha-test' grunt.loadNpmTasks 'grunt-forever' + grunt.loadNpmTasks 'grunt-bunyan' + grunt.loadNpmTasks 'grunt-execute' grunt.registerTask "test:unit", ["coffee", "mochaTest:unit"] grunt.registerTask "test:acceptence", ["coffee", "mochaTest:acceptence"] grunt.registerTask "test:acceptance", ["test:acceptence"] grunt.registerTask "ci", "test:unit" - grunt.registerTask 'default', ['coffee', 'concurrent'] + grunt.registerTask 'default', ['coffee', 'bunyan','execute'] grunt.registerTask "compile", "coffee" grunt.registerTask "install", "compile" diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 293f5da8e8..9c744e75ab 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -13,6 +13,7 @@ streamBuffers = require("stream-buffers") Metrics = require "metrics-sharelatex" Metrics.initialize("filestore") Metrics.open_sockets.monitor(logger) +Metrics.event_loop?.monitor(logger) app.configure -> app.use express.bodyParser() @@ -74,8 +75,17 @@ app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileContro app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile app.get "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile +app.get "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile + +app.get "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile +app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile + +app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.copyFile +app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile + + app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> res.send filename @@ -125,7 +135,7 @@ beginShutdown = () -> logger.log "server will stop accepting connections" server.listen port, host, -> - logger.log("filestore listening on #{host}:#{port}") + logger.info "Filestore starting up, listening on #{host}:#{port}" process.on 'SIGTERM', () -> logger.log("filestore got SIGTERM, shutting down gracefully") diff --git a/services/filestore/app/coffee/FileConverter.coffee b/services/filestore/app/coffee/FileConverter.coffee index c142d3a5e0..aaf56388dd 100644 --- a/services/filestore/app/coffee/FileConverter.coffee +++ b/services/filestore/app/coffee/FileConverter.coffee @@ -39,7 +39,7 @@ module.exports = args = "nice convert -flatten -background white -density 300 -define pdf:fit-page=#{width} #{sourcePath} -resize #{width} #{destPath}" safe_exec args, childProcessOpts, (err, stdout, stderr)-> if err? - logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to preview" + logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to thumbnail" else logger.log sourcePath:sourcePath, destPath:destPath, "finished thumbnailing file" callback(err, destPath) diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 71f6c05785..0b5e41525f 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -23,9 +23,20 @@ module.exports = req.bucket = settings.filestore.stores.user_files next() + publicFileKey: (req, res, next)-> + {project_id, public_file_id} = req.params + if not settings.filestore.stores.public_files? + res.status(501).send("public files not available") + else + req.key = "#{project_id}/#{public_file_id}" + req.bucket = settings.filestore.stores.public_files + next() + templateFileKey: (req, res, next)-> - {template_id, format, version} = req.params + {template_id, format, version, sub_type} = req.params req.key = "#{template_id}/v/#{version}/#{format}" + if sub_type? + req.key = "#{req.key}/#{sub_type}" req.bucket = settings.filestore.stores.template_files req.version = version opts = req.query diff --git a/services/filestore/app/coffee/LocalFileWriter.coffee b/services/filestore/app/coffee/LocalFileWriter.coffee index 12d505b05b..2ecb3885df 100644 --- a/services/filestore/app/coffee/LocalFileWriter.coffee +++ b/services/filestore/app/coffee/LocalFileWriter.coffee @@ -14,7 +14,6 @@ module.exports = fsPath = @_getPath(key) logger.log fsPath:fsPath, "writing file locally" writeStream = fs.createWriteStream(fsPath) - stream.pipe writeStream writeStream.on "finish", -> timer.done() logger.log fsPath:fsPath, "finished writing file locally" @@ -25,6 +24,7 @@ module.exports = stream.on "error", (err)-> logger.log err:err, fsPath:fsPath, "problem writing file locally, with read stream" callback err + stream.pipe writeStream deleteFile: (fsPath, callback)-> logger.log fsPath:fsPath, "removing local temp file" diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index ef1824491e..d5cb06074e 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -104,6 +104,9 @@ module.exports = secret: settings.filestore.s3.secret bucket: bucketName s3Client.list prefix:key, (err, data)-> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" + return callback(err) keys = _.map data.Contents, (entry)-> return entry.Key s3Client.deleteMultiple keys, callback diff --git a/services/filestore/config/settings.defaults.coffee b/services/filestore/config/settings.defaults.coffee index dccefea525..f84928598e 100644 --- a/services/filestore/config/settings.defaults.coffee +++ b/services/filestore/config/settings.defaults.coffee @@ -21,6 +21,7 @@ module.exports = # Must contain full path, e.g. "/var/lib/sharelatex/data". # This path must exist, not be tmpfs and be writable to by the user sharelatex is run as. user_files: Path.resolve(__dirname + "/../user_files") + public_files: Path.resolve(__dirname + "/../public_files") template_files: Path.resolve(__dirname + "/../template_files") # if you are using S3, then fill in your S3 details below # s3: diff --git a/services/filestore/package.json b/services/filestore/package.json index c2ba0e4952..dd84289f30 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -8,8 +8,11 @@ }, "dependencies": { "async": "~0.2.10", + "bunyan": "^1.3.5", "coffee-script": "~1.7.1", "express": "~3.4.8", + "grunt-bunyan": "^0.5.0", + "grunt-execute": "^0.2.2", "grunt-mocha-test": "~0.8.2", "heapdump": "^0.3.2", "knox": "~0.9.1",