From 5b54d36b375e82488e63eadee35512fe683c4fd5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 10:41:35 +0100 Subject: [PATCH 1/4] fail readiness check when shutting down --- services/real-time/app.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 7b53730ce6..56db154f83 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -56,7 +56,10 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - res.send "real-time-sharelatex is alive" + if shutDownInProgress + res.send 503 # Service unavailable + else + res.send "real-time-sharelatex is alive" app.get "/debug/events", (req, res, next) -> Settings.debugEvents = parseInt(req.query?.count,10) || 20 From 53431953fcd55ba064216d0fb98f93564ebec32a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:56:48 +0100 Subject: [PATCH 2/4] make shutDownInProgress available via settings --- services/real-time/app.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 56db154f83..5a78588591 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -56,7 +56,7 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - if shutDownInProgress + if Settings.shutDownInProgress res.send 503 # Service unavailable else res.send "real-time-sharelatex is alive" @@ -120,17 +120,17 @@ forceDrain = -> DrainManager.startDrain(io, 4) , Settings.forceDrainMsDelay -shutDownInProgress = false +Settings.shutDownInProgress = false if Settings.forceDrainMsDelay? Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if shutDownInProgress + if Settings.shutDownInProgress logger.log signal: signal, "shutdown already in progress, ignoring signal" return else - shutDownInProgress = true + Settings.shutDownInProgress = true logger.log signal: signal, "received interrupt, cleaning up" shutdownCleanly(signal) forceDrain() From 0708f717fd48e1745eac8420543fd8501ab9f6a5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:59:15 +0100 Subject: [PATCH 3/4] reject connections when shutdown in progress send a message to the client to reconnect immediately --- services/real-time/app/coffee/Router.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ad74ebffa1..838d8de6f5 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -39,6 +39,11 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + if settings.shutDownInProgress + client.emit("connectionRejected", {message: "retry"}) + client.disconnect() + return + if client? and error?.message?.match(/could not look up session by key/) logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key From 20d442120f1727626c50d7d56c62ce79f6fd8fc7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 17:36:53 +0100 Subject: [PATCH 4/4] notify docupdate if the flush is from a shutdown --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 40ef992d8f..9968b78753 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -40,7 +40,8 @@ module.exports = DocumentUpdaterManager = logger.log project_id:project_id, "deleting project from document updater" timer = new metrics.Timer("delete.mongo.project") # flush the project in the background when all users have left - url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" + url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" + + (if settings.shutDownInProgress then "&shutdown=true" else "") request.del url, (err, res, body)-> timer.done() if err?