diff --git a/services/real-time/app.js b/services/real-time/app.js index e65561cf43..bde00b2e17 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -22,6 +22,7 @@ const CookieParser = require('cookie-parser') const DrainManager = require('./app/js/DrainManager') const HealthCheckManager = require('./app/js/HealthCheckManager') +const DeploymentManager = require('./app/js/DeploymentManager') // NOTE: debug is invoked for every blob that is put on the wire const socketIoLogger = { @@ -36,6 +37,9 @@ const socketIoLogger = { log() {} } +// monitor status file to take dark deployments out of the load-balancer +DeploymentManager.initialise() + // Set up socket.io server const app = express() @@ -79,13 +83,20 @@ io.configure(function () { }) // a 200 response on '/' is required for load balancer health checks -app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) +// these operate separately from kubernetes readiness checks +app.get('/', function (req, res) { + if (Settings.shutDownInProgress || DeploymentManager.deploymentIsClosed()) { + res.sendStatus(503) // Service unavailable + } else { + res.send('real-time is open') + } +}) app.get('/status', function (req, res) { if (Settings.shutDownInProgress) { res.sendStatus(503) // Service unavailable } else { - res.send('real-time-sharelatex is alive') + res.send('real-time is alive') } }) diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js new file mode 100644 index 0000000000..ddb98fd4ce --- /dev/null +++ b/services/real-time/app/js/DeploymentManager.js @@ -0,0 +1,59 @@ +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const fs = require('fs') + +// Monitor a status file (e.g. /etc/real_time_status) periodically and close the +// service if the file contents don't contain the matching deployment colour. + +const FILE_CHECK_INTERVAL = 5000 +const statusFile = settings.deploymentFile +const deploymentColour = settings.deploymentColour + +var serviceCloseTime + +function updateDeploymentStatus(fileContent) { + const closed = fileContent && !fileContent.includes(deploymentColour) + if (closed && !settings.serviceIsClosed) { + settings.serviceIsClosed = true + serviceCloseTime = Date.now() + 60 * 1000 // delay closing by 1 minute + logger.warn({ fileContent }, 'closing service') + } else if (!closed && settings.serviceIsClosed) { + settings.serviceIsClosed = false + logger.warn({ fileContent }, 'opening service') + } +} + +function pollStatusFile() { + fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => { + if (err) { + logger.error( + { file: statusFile, fsErr: err }, + 'error reading service status file' + ) + return + } + updateDeploymentStatus(fileContent) + }) +} + +function checkStatusFileSync() { + // crash on start up if file does not exist + const content = fs.readFileSync(statusFile, { encoding: 'utf8' }) + updateDeploymentStatus(content) +} + +module.exports = { + initialise() { + if (statusFile && deploymentColour) { + logger.log( + { statusFile, deploymentColour, interval: FILE_CHECK_INTERVAL }, + 'monitoring deployment status file' + ) + checkStatusFileSync() // perform an initial synchronous check at start up + setInterval(pollStatusFile, FILE_CHECK_INTERVAL) // continue checking periodically + } + }, + deploymentIsClosed() { + return settings.serviceIsClosed && Date.now() > serviceCloseTime + } +} diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 486b686083..c15ca57f6f 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -148,6 +148,12 @@ const settings = { statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'), + // The deployment colour for this app (if any). Used for blue green deploys. + deploymentColour: process.env.DEPLOYMENT_COLOUR, + // Load balancer health checks will return 200 only when this file contains + // the deployment colour for this app. + deploymentFile: process.env.DEPLOYMENT_FILE, + sentry: { dsn: process.env.SENTRY_DSN },