diff --git a/services/filestore/Gruntfile.coffee b/services/filestore/Gruntfile.coffee index 91ae43bed1..c91476c201 100644 --- a/services/filestore/Gruntfile.coffee +++ b/services/filestore/Gruntfile.coffee @@ -19,7 +19,7 @@ module.exports = (grunt) -> app_server: expand: true, flatten: false, - src: ['app.coffee'], + src: ['app.coffee', 'cluster.coffee'], dest: './', ext: '.js' @@ -41,7 +41,7 @@ module.exports = (grunt) -> watch: server_coffee: - files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee"] + files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee", "cluster.coffee"] tasks: ["clean", 'coffee', 'mochaTest'] clean: ["app/js", "test/unit/js", "app.js"] diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index bf7a6c2346..293f5da8e8 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -37,28 +37,36 @@ app.use (req, res, next) -> requestDomain.add req requestDomain.add res requestDomain.on "error", (err)-> - # request a shutdown to prevent memory leaks - appIsOk = false - setTimeout(-> + try + appIsOk = false + # request a shutdown to prevent memory leaks + beginShutdown() if !res.headerSent - res.send(500) - , 3000) - logger = require('logger-sharelatex') - req = - body:req.body - headers:req.headers - url:req.url - key: req.key - statusCode: req.statusCode - err = - message: err.message - stack: err.stack - name: err.name - type: err.type - arguments: err.arguments - logger.err err:err, req:req, res:res, "uncaught exception thrown on request" + res.send(500, "uncaught exception") + logger = require('logger-sharelatex') + req = + body:req.body + headers:req.headers + url:req.url + key: req.key + statusCode: req.statusCode + err = + message: err.message + stack: err.stack + name: err.name + type: err.type + arguments: err.arguments + logger.err err:err, req:req, res:res, "uncaught exception thrown on request" + catch exception + logger.err err: exception, "exception in request domain handler" requestDomain.run next +app.use (req, res, next) -> + if not appIsOk + # when shutting down, close any HTTP keep-alive connections + res.set 'Connection', 'close' + next() + app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile @@ -95,19 +103,30 @@ app.get "/health_check", (req, res)-> else res.send(503) - - app.get '*', (req, res)-> res.send 404 -serverDomain = domain.create() -serverDomain.run -> - server = require('http').createServer(app) - port = settings.internal.filestore.port or 3009 - host = settings.internal.filestore.host or "localhost" - server.listen port, host, -> - logger.log("filestore store listening on #{host}:#{port}") +server = require('http').createServer(app) +port = settings.internal.filestore.port or 3009 +host = settings.internal.filestore.host or "localhost" -serverDomain.on "error", (err)-> - logger.log err:err, "top level uncaught exception" +beginShutdown = () -> + if appIsOk + appIsOk = false + # hard-terminate this process if graceful shutdown fails + killTimer = setTimeout () -> + process.exit 1 + , 120*1000 + killTimer.unref?() # prevent timer from keeping process alive + server.close () -> + logger.log "closed all connections" + Metrics.close() + process.disconnect?() + logger.log "server will stop accepting connections" +server.listen port, host, -> + logger.log("filestore listening on #{host}:#{port}") + +process.on 'SIGTERM', () -> + logger.log("filestore got SIGTERM, shutting down gracefully") + beginShutdown() diff --git a/services/filestore/cluster.coffee b/services/filestore/cluster.coffee new file mode 100644 index 0000000000..a0ca60a619 --- /dev/null +++ b/services/filestore/cluster.coffee @@ -0,0 +1,9 @@ +recluster = require "recluster" # https://github.com/doxout/recluster +path = require "path" + +cluster = recluster path.join(__dirname, 'app.js'), { + workers: 2, + backoff: 0, + readyWhen: "listening" +} +cluster.run() diff --git a/services/filestore/package.json b/services/filestore/package.json index 919adc488f..bb3cae64e9 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -20,6 +20,7 @@ "node-transloadit": "0.0.4", "node-uuid": "~1.4.1", "pngcrush": "0.0.3", + "recluster": "^0.3.7", "request": "2.14.0", "response": "0.14.0", "rimraf": "2.2.8",