From c2a0d0d988ffba1e11bc46405458fc321db613e1 Mon Sep 17 00:00:00 2001 From: Antoine Clausse Date: Fri, 22 Nov 2024 09:52:57 +0100 Subject: [PATCH] [filestore] Replace `delayShutdownMs` by `gracefulShutdownDelayInMs` (#22077) * Add a default 30s timeout for `delayShutdownMs` `settings.delayShutdownMs` doesn't seem to be defined anywhere Logs typically often this pair of entries: ``` INFO 2024-11-21T15:51:42.115Z [resource.labels.containerName: filestore] received interrupt, cleaning up INFO 2024-11-21T15:51:42.120Z [resource.labels.containerName: filestore] shutdown timed out, exiting ``` This indicates that there is no delay between the interrupt signal and the shutdown. The 100ms delay also doesn't happen. We believe that the `server.close` callback is called after `server.closeAllConnections()`, and the `server closed` log is usually lost because of the process exiting immediately. See: https://cloudlogging.app.goo.gl/WJQ6mc3gWwotVQya7 * Add timeout before exiting * Replace `delayShutdownMs` by `gracefulShutdownDelayInMs`. Looks like the error came from a wrong merge in https://github.com/overleaf/internal/pull/18756 We don't want a default 30s timeout for the shutdown, per https://github.com/overleaf/internal/pull/16888 GitOrigin-RevId: c1bdc8986f78a6e18e8b8b1fe60b33aa6ffef909 --- services/filestore/app.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index 9c28b47c16..bc2d270132 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -166,7 +166,10 @@ function handleShutdownSignal(signal) { // stop accepting new connections, the callback is called when existing connections have finished server.close(() => { logger.info({ signal }, 'server closed') - process.exit() + // exit after a short delay so logs can be flushed + setTimeout(() => { + process.exit() + }, 100) }) // close idle http keep-alive connections server.closeIdleConnections() @@ -178,7 +181,7 @@ function handleShutdownSignal(signal) { setTimeout(() => { process.exit() }, 100) - }, settings.delayShutdownMs) + }, settings.gracefulShutdownDelayInMs) } process.on('SIGTERM', handleShutdownSignal)