diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index e7243c5291..7656cf4526 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.15.1 as app +FROM node:10.16.3 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:6.15.1 +FROM node:10.16.3 COPY --from=app /app /app diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 708e5f467c..c2f9e7fa9c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -119,22 +119,32 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 +drainAndShutdown = (signal) -> + if Settings.shutDownInProgress + logger.log signal: signal, "shutdown already in progress, ignoring signal" + return + else + Settings.shutDownInProgress = true + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + shutdownCleanly(signal) + + Settings.shutDownInProgress = false if Settings.shutdownDrainTimeWindow? shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if Settings.shutDownInProgress - logger.log signal: signal, "shutdown already in progress, ignoring signal" - return - else - Settings.shutDownInProgress = true - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) - + drainAndShutdown(signal) + # global exception handler + if Settings.errors?.catchUncaughtErrors + process.removeAllListeners('uncaughtException') + process.on 'uncaughtException', (error) -> + logger.error err: error, 'uncaught exception' + if Settings.errors?.shutdownOnUncaughtError + drainAndShutdown('SIGABRT') if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index ad142ccc47..2590a96726 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -36,4 +36,4 @@ module.exports = DrainManager = if haveDrainedNClients break if drainedCount < N - logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file + logger.log "All clients have been told to reconnectGracefully" diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 92a17729b7..fcdb0c93dc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -42,6 +42,12 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + client?.on "error", (err) -> + logger.err { clientErr: err }, "socket.io client error" + if client.connected + client.emit("reconnectGracefully") + client.disconnect() + if settings.shutDownInProgress client.emit("connectionRejected", {message: "retry"}) client.disconnect() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7fba6ea0d6..75b39ab2ee 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -62,6 +62,10 @@ settings = sentry: dsn: process.env.SENTRY_DSN + + errors: + catchUncaughtErrors: true + shutdownOnUncaughtError: true # console.log settings.redis module.exports = settings