From 74fc095913935cd5b84c895d264f4edc7331ddc8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Dec 2014 16:46:31 +0000 Subject: [PATCH 1/9] close HTTP keep-alive connections when shutting down --- services/filestore/app.coffee | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index bf7a6c2346..1f92d6e943 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -62,6 +62,12 @@ app.use (req, res, next) -> app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile +app.use (req, res, next) -> + if not appIsOk + # when shutting down, close any HTTP keep-alive connections + res.set 'Connection', 'close' + next() + app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.copyFile app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile From 48f62990b4d479053d4c45153a8a2de237bb0595 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Dec 2014 16:48:11 +0000 Subject: [PATCH 2/9] server can exit immediately, does not need a domain exit with an exception as no requests can be served --- services/filestore/app.coffee | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 1f92d6e943..c17b277bf5 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -106,14 +106,18 @@ app.get "/health_check", (req, res)-> app.get '*', (req, res)-> res.send 404 -serverDomain = domain.create() -serverDomain.run -> - server = require('http').createServer(app) - port = settings.internal.filestore.port or 3009 - host = settings.internal.filestore.host or "localhost" - server.listen port, host, -> - logger.log("filestore store listening on #{host}:#{port}") +server = require('http').createServer(app) +port = settings.internal.filestore.port or 3009 +host = settings.internal.filestore.host or "localhost" -serverDomain.on "error", (err)-> - logger.log err:err, "top level uncaught exception" +beginShutdown = () -> + appIsOk = false + server.close() + logger.log "server will stop accepting connections" +server.on "close", () -> + logger.log "closed all connections" + process.exit 1 + +server.listen port, host, -> + logger.log("filestore store listening on #{host}:#{port}") From d3fa6b4a01855cf897cd6b24ed470e8a0f47cab5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Dec 2014 16:49:34 +0000 Subject: [PATCH 3/9] catch exceptions within the body of the domain error handler --- services/filestore/app.coffee | 40 ++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index c17b277bf5..e96b0414e3 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -37,26 +37,28 @@ app.use (req, res, next) -> requestDomain.add req requestDomain.add res requestDomain.on "error", (err)-> - # request a shutdown to prevent memory leaks - appIsOk = false - setTimeout(-> + try + appIsOk = false + # request a shutdown to prevent memory leaks + beginShutdown() if !res.headerSent - res.send(500) - , 3000) - logger = require('logger-sharelatex') - req = - body:req.body - headers:req.headers - url:req.url - key: req.key - statusCode: req.statusCode - err = - message: err.message - stack: err.stack - name: err.name - type: err.type - arguments: err.arguments - logger.err err:err, req:req, res:res, "uncaught exception thrown on request" + res.send(500, "uncaught exception") + logger = require('logger-sharelatex') + req = + body:req.body + headers:req.headers + url:req.url + key: req.key + statusCode: req.statusCode + err = + message: err.message + stack: err.stack + name: err.name + type: err.type + arguments: err.arguments + logger.err err:err, req:req, res:res, "uncaught exception thrown on request" + catch exception + logger.err err: exception, "exception in request domain handler" requestDomain.run next app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile From d5478a8ebb9649b5208ab44ac37d085026281bde Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Dec 2014 16:55:34 +0000 Subject: [PATCH 4/9] make sure shutdown only happens once --- services/filestore/app.coffee | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index e96b0414e3..f2b4c405be 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -113,9 +113,10 @@ port = settings.internal.filestore.port or 3009 host = settings.internal.filestore.host or "localhost" beginShutdown = () -> - appIsOk = false - server.close() - logger.log "server will stop accepting connections" + if appIsOk + appIsOk = false + server.close() + logger.log "server will stop accepting connections" server.on "close", () -> logger.log "closed all connections" From e8e2338c0cafa212782329cffdf48c089063d411 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 5 Jan 2015 16:56:19 +0000 Subject: [PATCH 5/9] move connection:close middleware ahead of other routes --- services/filestore/app.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index f2b4c405be..9a1bfc7ef9 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -61,15 +61,15 @@ app.use (req, res, next) -> logger.err err: exception, "exception in request domain handler" requestDomain.run next -app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile -app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile - app.use (req, res, next) -> if not appIsOk # when shutting down, close any HTTP keep-alive connections res.set 'Connection', 'close' next() +app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile +app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile + app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.copyFile app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile From 02101868e1d9f46bede47b345a9efff3c386dc16 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Jan 2015 10:26:30 +0000 Subject: [PATCH 6/9] put all shutdown logic in beginShutdown, force program to exit after 2 mins --- services/filestore/app.coffee | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 9a1bfc7ef9..2638787b83 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -103,8 +103,6 @@ app.get "/health_check", (req, res)-> else res.send(503) - - app.get '*', (req, res)-> res.send 404 @@ -115,12 +113,16 @@ host = settings.internal.filestore.host or "localhost" beginShutdown = () -> if appIsOk appIsOk = false - server.close() + # hard-terminate this process if graceful shutdown fails + killTimer = setTimeout () -> + process.exit 1 + , 120*1000 + killTimer.unref?() # prevent timer from keeping process alive + server.close () -> + logger.log "closed all connections" + Metrics.close() + process.disconnect?() logger.log "server will stop accepting connections" -server.on "close", () -> - logger.log "closed all connections" - process.exit 1 - server.listen port, host, -> logger.log("filestore store listening on #{host}:#{port}") From 191d0a49866946911cbcaa38544047241d2a476b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Jan 2015 10:28:47 +0000 Subject: [PATCH 7/9] handle SIGTERM in filestore --- services/filestore/app.coffee | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 2638787b83..d76d53a993 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -126,3 +126,7 @@ beginShutdown = () -> server.listen port, host, -> logger.log("filestore store listening on #{host}:#{port}") + +process.on 'SIGTERM', () -> + logger.log("filestore got SIGTERM, shutting down gracefully") + beginShutdown() From 79292b46022e28ca7721e59bfb79635d3a5cfd95 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Jan 2015 10:29:05 +0000 Subject: [PATCH 8/9] fix log message --- services/filestore/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index d76d53a993..293f5da8e8 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -125,7 +125,7 @@ beginShutdown = () -> logger.log "server will stop accepting connections" server.listen port, host, -> - logger.log("filestore store listening on #{host}:#{port}") + logger.log("filestore listening on #{host}:#{port}") process.on 'SIGTERM', () -> logger.log("filestore got SIGTERM, shutting down gracefully") From e03ee962167f60fa156f9cfc0941ae2c68f49167 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Jan 2015 10:33:55 +0000 Subject: [PATCH 9/9] added simple clustering master --- services/filestore/Gruntfile.coffee | 4 ++-- services/filestore/cluster.coffee | 9 +++++++++ services/filestore/package.json | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 services/filestore/cluster.coffee diff --git a/services/filestore/Gruntfile.coffee b/services/filestore/Gruntfile.coffee index aa40cf7460..a506496578 100644 --- a/services/filestore/Gruntfile.coffee +++ b/services/filestore/Gruntfile.coffee @@ -14,7 +14,7 @@ module.exports = (grunt) -> app_server: expand: true, flatten: false, - src: ['app.coffee'], + src: ['app.coffee', 'cluster.coffee'], dest: './', ext: '.js' @@ -36,7 +36,7 @@ module.exports = (grunt) -> watch: server_coffee: - files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee"] + files: ['app/*.coffee','app/**/*.coffee', 'test/unit/coffee/**/*.coffee', 'test/unit/coffee/*.coffee', "app.coffee", "cluster.coffee"] tasks: ["clean", 'coffee', 'mochaTest'] clean: ["app/js", "test/unit/js", "app.js"] diff --git a/services/filestore/cluster.coffee b/services/filestore/cluster.coffee new file mode 100644 index 0000000000..a0ca60a619 --- /dev/null +++ b/services/filestore/cluster.coffee @@ -0,0 +1,9 @@ +recluster = require "recluster" # https://github.com/doxout/recluster +path = require "path" + +cluster = recluster path.join(__dirname, 'app.js'), { + workers: 2, + backoff: 0, + readyWhen: "listening" +} +cluster.run() diff --git a/services/filestore/package.json b/services/filestore/package.json index be103715f7..3586041555 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -20,6 +20,7 @@ "node-transloadit": "0.0.4", "node-uuid": "~1.4.1", "pngcrush": "0.0.3", + "recluster": "^0.3.7", "request": "2.14.0", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#master", "stream-buffers": "~0.2.5",