From 199ced5eef6e8dbd210db8f5bd1d755558aaf226 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 29 Aug 2022 13:29:20 +0100 Subject: [PATCH] Merge pull request #9231 from overleaf/jpa-clsi-lifespan-fix [clsi] cycle VMs before their 24h limit with some jitter GitOrigin-RevId: c08bea6765dc8b03db9f0792a8bb20084f595f59 --- services/clsi/app.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/services/clsi/app.js b/services/clsi/app.js index 386c9efad5..fb8d335999 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -240,15 +240,19 @@ app.get('/status', (req, res, next) => res.send('CLSI is alive\n')) Settings.processTooOld = false if (Settings.processLifespanLimitMs) { - Settings.processLifespanLimitMs += + // Pre-emp instances have a maximum lifespan of 24h after which they will be + // shutdown, with a 30s grace period. + // Spread cycling of VMs by up-to 2.4h _before_ their limit to avoid large + // numbers of VMs that are temporarily unavailable (while they reboot). + Settings.processLifespanLimitMs -= Settings.processLifespanLimitMs * (Math.random() / 10) - logger.debug( - 'Lifespan limited to ', - Date.now() + Settings.processLifespanLimitMs + logger.info( + { target: new Date(Date.now() + Settings.processLifespanLimitMs) }, + 'Lifespan limited' ) setTimeout(() => { - logger.debug('shutting down, process is too old') + logger.info({}, 'shutting down, process is too old') Settings.processTooOld = true }, Settings.processLifespanLimitMs) }