diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 67e25a704c..40d2a151fc 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -116,17 +116,10 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 -forceDrain = -> - logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" - setTimeout ()-> - logger.log "starting drain" - DrainManager.startDrain(io, 4) - , Settings.forceDrainMsDelay - shutDownInProgress = false -if Settings.forceDrainMsDelay? - Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) - logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" +if Settings.shutdownDrainTimeWindow? + Settings.forceDrainMsDelay = parseInt(Settings.shutdownDrainTimeWindow, 10) + logger.log shutdownDrainTimeWindow: Settings.shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> if shutDownInProgress @@ -134,9 +127,9 @@ if Settings.forceDrainMsDelay? return else shutDownInProgress = true - logger.log signal: signal, "received interrupt, cleaning up" + logger.log signal: signal, "received interrupt, starting drain over #{Settings.shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) shutdownCleanly(signal) - forceDrain() diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index da6b331e23..2219291267 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -1,6 +1,11 @@ logger = require "logger-sharelatex" -module.exports = +module.exports = DrainManager = + + startDrainTimeWindow: (io, minsToDrain)-> + drainPerMin = io.sockets.clients().length / minsToDrain + DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate + startDrain: (io, rate) -> # Clear out any old interval clearInterval @interval diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index a3128e84d1..6fe0f4bd1e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -48,7 +48,7 @@ settings = max_doc_length: 2 * 1024 * 1024 # 2mb - forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false + shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee index b3cdaf1ca0..88009f02cd 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -12,6 +12,23 @@ describe "DrainManager", -> sockets: clients: sinon.stub() + describe "startDrainTimeWindow", -> + beforeEach -> + @clients = [] + for i in [0..5399] + @clients[i] = { + id: i + emit: sinon.stub() + } + @io.sockets.clients.returns @clients + @DrainManager.startDrain = sinon.stub() + + it "should set a drain rate fast enough", (done)-> + @DrainManager.startDrainTimeWindow(@io, 9) + @DrainManager.startDrain.calledWith(@io, 10).should.equal true + done() + + describe "reconnectNClients", -> beforeEach -> @clients = []