Merge pull request #161 from overleaf/jpa-msm-decaf

[misc] decaffeinate real-time

services/real-time/.dockerignore

@@ -5,5 +5,3 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
-app.js
-**/js/*

services/real-time/.eslintrc

@@ -0,0 +1,64 @@
+// this file was auto-generated, do not edit it directly.
+// instead run bin/update_build_scripts from
+// https://github.com/sharelatex/sharelatex-dev-environment
+{
+  "extends": [
+    "standard",
+    "prettier",
+    "prettier/standard"
+  ],
+  "parserOptions": {
+    "ecmaVersion": 2018
+  },
+  "plugins": [
+    "mocha",
+    "chai-expect",
+    "chai-friendly"
+  ],
+  "env": {
+    "node": true,
+    "mocha": true
+  },
+  "rules": {
+    // Swap the no-unused-expressions rule with a more chai-friendly one
+    "no-unused-expressions": 0,
+    "chai-friendly/no-unused-expressions": "error"
+  },
+  "overrides": [
+    {
+      // Test specific rules
+      "files": ["test/**/*.js"],
+      "globals": {
+        "expect": true
+      },
+      "rules": {
+        // mocha-specific rules
+        "mocha/handle-done-callback": "error",
+        "mocha/no-exclusive-tests": "error",
+        "mocha/no-global-tests": "error",
+        "mocha/no-identical-title": "error",
+        "mocha/no-nested-tests": "error",
+        "mocha/no-pending-tests": "error",
+        "mocha/no-skipped-tests": "error",
+        "mocha/no-mocha-arrows": "error",
+
+        // chai-specific rules
+        "chai-expect/missing-assertion": "error",
+        "chai-expect/terminating-properties": "error",
+
+        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
+        // we don't enforce this at the top-level - just in tests to manage `this` scope
+        // based on mocha's context mechanism
+        "mocha/prefer-arrow-callback": "error"
+      }
+    },
+    {
+      // Backend specific rules
+      "files": ["app/**/*.js", "app.js", "index.js"],
+      "rules": {
+        // don't allow console.log in backend code
+        "no-console": "error"
+      }
+    }
+  ]
+}

services/real-time/.gitignore

@@ -1,7 +1,2 @@
 node_modules
 forever
-app.js
-app/js
-test/unit/js
-test/acceptance/js
-**/*.map

services/real-time/.prettierrc

@@ -0,0 +1,7 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+{
+  "semi": false,
+  "singleQuote": true
+}

services/real-time/Dockerfile

@@ -1,7 +1,6 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.3.5
 
 FROM node:10.21.0 as base
 
@@ -12,12 +11,11 @@ FROM base as app
 #wildcard as some files may not be in all repos
 COPY package*.json npm-shrink*.json /app/
 
-RUN npm install --quiet
+RUN npm ci --quiet
 
 COPY . /app
 
 
-RUN npm run compile:all
 
 FROM base
 

services/real-time/Jenkinsfile

@@ -37,6 +37,13 @@ pipeline {
       }
     }
 
+    stage('Linting') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
+      }
+    }
+
     stage('Unit Tests') {
       steps {
         sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

services/real-time/Makefile

@@ -1,11 +1,12 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.3.5
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 PROJECT_NAME = real-time
+BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
+
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
 DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	BRANCH_NAME=$(BRANCH_NAME) \
@@ -13,34 +14,63 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker-compose ${DOCKER_COMPOSE_FLAGS}
 
+DOCKER_COMPOSE_TEST_ACCEPTANCE = \
+	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+
+DOCKER_COMPOSE_TEST_UNIT = \
+	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+
 clean:
 	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	rm -f app.js
-	rm -rf app/js
-	rm -rf test/unit/js
-	rm -rf test/acceptance/js
 
-test: test_unit test_acceptance
+format:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+
+format_fix:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+
+lint:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+
+test: format lint test_unit test_acceptance
 
 test_unit:
-	@[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
+ifneq (,$(wildcard test/unit))
+	$(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit
+	$(MAKE) test_unit_clean
+endif
 
-test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run
+test_clean: test_unit_clean
+test_unit_clean:
+ifneq (,$(wildcard test/unit))
+	$(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0
+endif
 
-test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug
+test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run
+	$(MAKE) test_acceptance_clean
+
+test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug
+	$(MAKE) test_acceptance_clean
 
 test_acceptance_run:
-	@[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
+ifneq (,$(wildcard test/acceptance))
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance
+endif
 
 test_acceptance_run_debug:
-	@[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
+ifneq (,$(wildcard test/acceptance))
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
+endif
 
-test_clean:
-	$(DOCKER_COMPOSE) down -v -t 0
+test_clean: test_acceptance_clean
+test_acceptance_clean:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+endif
 
 build:
 	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
@@ -54,8 +84,5 @@ publish:
 
 	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
-lint:
-
-format:
 
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/real-time/app.coffee

@@ -1,182 +0,0 @@
-Metrics = require("metrics-sharelatex")
-Settings = require "settings-sharelatex"
-Metrics.initialize(Settings.appName or "real-time")
-async = require("async")
-_ = require "underscore"
-
-logger = require "logger-sharelatex"
-logger.initialize("real-time")
-Metrics.event_loop.monitor(logger)
-
-express = require("express")
-session = require("express-session")
-redis = require("redis-sharelatex")
-if Settings.sentry?.dsn?
-	logger.initializeErrorReporting(Settings.sentry.dsn)
-
-sessionRedisClient = redis.createClient(Settings.redis.websessions)
-
-RedisStore = require('connect-redis')(session)
-SessionSockets = require('./app/js/SessionSockets')
-CookieParser = require("cookie-parser")
-
-DrainManager = require("./app/js/DrainManager")
-HealthCheckManager = require("./app/js/HealthCheckManager")
-
-# work around frame handler bug in socket.io v0.9.16
-require("./socket.io.patch.js") 
-# Set up socket.io server
-app = express()
-
-server = require('http').createServer(app)
-io = require('socket.io').listen(server)
-
-# Bind to sessions
-sessionStore = new RedisStore(client: sessionRedisClient)
-cookieParser = CookieParser(Settings.security.sessionSecret)
-
-sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName)
-
-Metrics.injectMetricsRoute(app)
-app.use(Metrics.http.monitor(logger))
-
-io.configure ->
-	io.enable('browser client minification')
-	io.enable('browser client etag')
-
-	# Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch"
-	# See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with
-	io.set('match origin protocol', true)
-
-	# gzip uses a Node 0.8.x method of calling the gzip program which
-	# doesn't work with 0.6.x
-	#io.enable('browser client gzip')
-	io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling'])
-	io.set('log level', 1)
-
-app.get "/", (req, res, next) ->
-	res.send "real-time-sharelatex is alive"
-
-app.get "/status", (req, res, next) ->
-	if Settings.shutDownInProgress
-		res.send 503 # Service unavailable
-	else
-		res.send "real-time-sharelatex is alive"
-
-app.get "/debug/events", (req, res, next) ->
-	Settings.debugEvents = parseInt(req.query?.count,10) || 20
-	logger.log {count: Settings.debugEvents}, "starting debug mode"
-	res.send "debug mode will log next #{Settings.debugEvents} events"
-
-rclient = require("redis-sharelatex").createClient(Settings.redis.realtime)
-
-healthCheck = (req, res, next)->
-	rclient.healthCheck (error) ->
-		if error?
-			logger.err {err: error}, "failed redis health check"
-			res.sendStatus 500
-		else if HealthCheckManager.isFailing()
-			status = HealthCheckManager.status()
-			logger.err {pubSubErrors: status}, "failed pubsub health check"
-			res.sendStatus 500
-		else
-			res.sendStatus 200
-
-app.get "/health_check", healthCheck
-
-app.get "/health_check/redis", healthCheck
-
-
-
-Router = require "./app/js/Router"
-Router.configure(app, io, sessionSockets)
-
-WebsocketLoadBalancer = require "./app/js/WebsocketLoadBalancer"
-WebsocketLoadBalancer.listenForEditorEvents(io)
-
-DocumentUpdaterController = require "./app/js/DocumentUpdaterController"
-DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io)
-
-port = Settings.internal.realTime.port
-host = Settings.internal.realTime.host
-
-server.listen port, host, (error) ->
-	throw error if error?
-	logger.info "realtime starting up, listening on #{host}:#{port}"
-
-# Stop huge stack traces in logs from all the socket.io parsing steps.
-Error.stackTraceLimit = 10
-
-
-shutdownCleanly = (signal) ->
-	connectedClients = io.sockets.clients()?.length
-	if connectedClients == 0
-		logger.warn("no clients connected, exiting")
-		process.exit()
-	else
-		logger.warn {connectedClients}, "clients still connected, not shutting down yet"
-		setTimeout () ->
-			shutdownCleanly(signal)
-		, 30 * 1000
-
-drainAndShutdown = (signal) ->
-	if Settings.shutDownInProgress
-		logger.warn signal: signal, "shutdown already in progress, ignoring signal"
-		return
-	else
-		Settings.shutDownInProgress = true
-		statusCheckInterval = Settings.statusCheckInterval
-		if statusCheckInterval
-			logger.warn signal: signal, "received interrupt, delay drain by #{statusCheckInterval}ms"
-		setTimeout () ->
-			logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins"
-			DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
-			shutdownCleanly(signal)
-		, statusCheckInterval
-
-
-Settings.shutDownInProgress = false
-if Settings.shutdownDrainTimeWindow?
-	shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10)
-	logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled"
-	for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
-		process.on signal, drainAndShutdown  # signal is passed as argument to event handler
-
-	# global exception handler
-	if Settings.errors?.catchUncaughtErrors
-		process.removeAllListeners('uncaughtException')
-		process.on 'uncaughtException', (error) ->
-			if ['EPIPE', 'ECONNRESET'].includes(error.code)
-				Metrics.inc('disconnected_write', 1, {status: error.code})
-				return logger.warn err: error, 'attempted to write to disconnected client'
-			logger.error err: error, 'uncaught exception'
-			if Settings.errors?.shutdownOnUncaughtError
-				drainAndShutdown('SIGABRT')
-
-if Settings.continualPubsubTraffic
-	console.log "continualPubsubTraffic enabled"
-
-	pubsubClient = redis.createClient(Settings.redis.pubsub)
-	clusterClient = redis.createClient(Settings.redis.websessions)
-
-	publishJob = (channel, callback)->
-		checker = new HealthCheckManager(channel)
-		logger.debug {channel:channel}, "sending pub to keep connection alive"
-		json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()})
-		Metrics.summary "redis.publish.#{channel}", json.length
-		pubsubClient.publish channel, json, (err)->
-			if err?
-				logger.err {err, channel}, "error publishing pubsub traffic to redis"
-			blob = JSON.stringify({keep: "alive"})
-			Metrics.summary "redis.publish.cluster-continual-traffic", blob.length
-			clusterClient.publish "cluster-continual-traffic", blob, callback
-
-
-	runPubSubTraffic = ->
-		async.map ["applied-ops", "editor-events"], publishJob, (err)->
-			setTimeout(runPubSubTraffic, 1000 * 20)
-
-	runPubSubTraffic()
-
-
-

services/real-time/app.js

@@ -0,0 +1,252 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const Metrics = require('metrics-sharelatex')
+const Settings = require('settings-sharelatex')
+Metrics.initialize(Settings.appName || 'real-time')
+const async = require('async')
+
+const logger = require('logger-sharelatex')
+logger.initialize('real-time')
+Metrics.event_loop.monitor(logger)
+
+const express = require('express')
+const session = require('express-session')
+const redis = require('redis-sharelatex')
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
+  logger.initializeErrorReporting(Settings.sentry.dsn)
+}
+
+const sessionRedisClient = redis.createClient(Settings.redis.websessions)
+
+const RedisStore = require('connect-redis')(session)
+const SessionSockets = require('./app/js/SessionSockets')
+const CookieParser = require('cookie-parser')
+
+const DrainManager = require('./app/js/DrainManager')
+const HealthCheckManager = require('./app/js/HealthCheckManager')
+
+// work around frame handler bug in socket.io v0.9.16
+require('./socket.io.patch.js')
+// Set up socket.io server
+const app = express()
+
+const server = require('http').createServer(app)
+const io = require('socket.io').listen(server)
+
+// Bind to sessions
+const sessionStore = new RedisStore({ client: sessionRedisClient })
+const cookieParser = CookieParser(Settings.security.sessionSecret)
+
+const sessionSockets = new SessionSockets(
+  io,
+  sessionStore,
+  cookieParser,
+  Settings.cookieName
+)
+
+Metrics.injectMetricsRoute(app)
+app.use(Metrics.http.monitor(logger))
+
+io.configure(function () {
+  io.enable('browser client minification')
+  io.enable('browser client etag')
+
+  // Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch"
+  // See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with
+  io.set('match origin protocol', true)
+
+  // gzip uses a Node 0.8.x method of calling the gzip program which
+  // doesn't work with 0.6.x
+  // io.enable('browser client gzip')
+  io.set('transports', [
+    'websocket',
+    'flashsocket',
+    'htmlfile',
+    'xhr-polling',
+    'jsonp-polling'
+  ])
+  return io.set('log level', 1)
+})
+
+app.get('/', (req, res, next) => res.send('real-time-sharelatex is alive'))
+
+app.get('/status', function (req, res, next) {
+  if (Settings.shutDownInProgress) {
+    return res.send(503) // Service unavailable
+  } else {
+    return res.send('real-time-sharelatex is alive')
+  }
+})
+
+app.get('/debug/events', function (req, res, next) {
+  Settings.debugEvents =
+    parseInt(req.query != null ? req.query.count : undefined, 10) || 20
+  logger.log({ count: Settings.debugEvents }, 'starting debug mode')
+  return res.send(`debug mode will log next ${Settings.debugEvents} events`)
+})
+
+const rclient = require('redis-sharelatex').createClient(
+  Settings.redis.realtime
+)
+
+const healthCheck = (req, res, next) =>
+  rclient.healthCheck(function (error) {
+    if (error != null) {
+      logger.err({ err: error }, 'failed redis health check')
+      return res.sendStatus(500)
+    } else if (HealthCheckManager.isFailing()) {
+      const status = HealthCheckManager.status()
+      logger.err({ pubSubErrors: status }, 'failed pubsub health check')
+      return res.sendStatus(500)
+    } else {
+      return res.sendStatus(200)
+    }
+  })
+
+app.get('/health_check', healthCheck)
+
+app.get('/health_check/redis', healthCheck)
+
+const Router = require('./app/js/Router')
+Router.configure(app, io, sessionSockets)
+
+const WebsocketLoadBalancer = require('./app/js/WebsocketLoadBalancer')
+WebsocketLoadBalancer.listenForEditorEvents(io)
+
+const DocumentUpdaterController = require('./app/js/DocumentUpdaterController')
+DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io)
+
+const { port } = Settings.internal.realTime
+const { host } = Settings.internal.realTime
+
+server.listen(port, host, function (error) {
+  if (error != null) {
+    throw error
+  }
+  return logger.info(`realtime starting up, listening on ${host}:${port}`)
+})
+
+// Stop huge stack traces in logs from all the socket.io parsing steps.
+Error.stackTraceLimit = 10
+
+var shutdownCleanly = function (signal) {
+  const connectedClients = __guard__(io.sockets.clients(), (x) => x.length)
+  if (connectedClients === 0) {
+    logger.warn('no clients connected, exiting')
+    return process.exit()
+  } else {
+    logger.warn(
+      { connectedClients },
+      'clients still connected, not shutting down yet'
+    )
+    return setTimeout(() => shutdownCleanly(signal), 30 * 1000)
+  }
+}
+
+const drainAndShutdown = function (signal) {
+  if (Settings.shutDownInProgress) {
+    logger.warn({ signal }, 'shutdown already in progress, ignoring signal')
+  } else {
+    Settings.shutDownInProgress = true
+    const { statusCheckInterval } = Settings
+    if (statusCheckInterval) {
+      logger.warn(
+        { signal },
+        `received interrupt, delay drain by ${statusCheckInterval}ms`
+      )
+    }
+    return setTimeout(function () {
+      logger.warn(
+        { signal },
+        `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`
+      )
+      DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
+      return shutdownCleanly(signal)
+    }, statusCheckInterval)
+  }
+}
+
+Settings.shutDownInProgress = false
+if (Settings.shutdownDrainTimeWindow != null) {
+  var shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10)
+  logger.log({ shutdownDrainTimeWindow }, 'shutdownDrainTimeWindow enabled')
+  for (const signal of [
+    'SIGINT',
+    'SIGHUP',
+    'SIGQUIT',
+    'SIGUSR1',
+    'SIGUSR2',
+    'SIGTERM',
+    'SIGABRT'
+  ]) {
+    process.on(signal, drainAndShutdown)
+  } // signal is passed as argument to event handler
+
+  // global exception handler
+  if (
+    Settings.errors != null ? Settings.errors.catchUncaughtErrors : undefined
+  ) {
+    process.removeAllListeners('uncaughtException')
+    process.on('uncaughtException', function (error) {
+      if (['EPIPE', 'ECONNRESET'].includes(error.code)) {
+        Metrics.inc('disconnected_write', 1, { status: error.code })
+        return logger.warn(
+          { err: error },
+          'attempted to write to disconnected client'
+        )
+      }
+      logger.error({ err: error }, 'uncaught exception')
+      if (
+        Settings.errors != null
+          ? Settings.errors.shutdownOnUncaughtError
+          : undefined
+      ) {
+        return drainAndShutdown('SIGABRT')
+      }
+    })
+  }
+}
+
+if (Settings.continualPubsubTraffic) {
+  logger.warn('continualPubsubTraffic enabled')
+
+  const pubsubClient = redis.createClient(Settings.redis.pubsub)
+  const clusterClient = redis.createClient(Settings.redis.websessions)
+
+  const publishJob = function (channel, callback) {
+    const checker = new HealthCheckManager(channel)
+    logger.debug({ channel }, 'sending pub to keep connection alive')
+    const json = JSON.stringify({
+      health_check: true,
+      key: checker.id,
+      date: new Date().toString()
+    })
+    Metrics.summary(`redis.publish.${channel}`, json.length)
+    return pubsubClient.publish(channel, json, function (err) {
+      if (err != null) {
+        logger.err({ err, channel }, 'error publishing pubsub traffic to redis')
+      }
+      const blob = JSON.stringify({ keep: 'alive' })
+      Metrics.summary('redis.publish.cluster-continual-traffic', blob.length)
+      return clusterClient.publish('cluster-continual-traffic', blob, callback)
+    })
+  }
+
+  var runPubSubTraffic = () =>
+    async.map(['applied-ops', 'editor-events'], publishJob, () =>
+      setTimeout(runPubSubTraffic, 1000 * 20)
+    )
+
+  runPubSubTraffic()
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/real-time/app/coffee/AuthorizationManager.coffee

@@ -1,36 +0,0 @@
-module.exports = AuthorizationManager =
-	assertClientCanViewProject: (client, callback = (error) ->) ->
-		AuthorizationManager._assertClientHasPrivilegeLevel client, ["readOnly", "readAndWrite", "owner"], callback
-
-	assertClientCanEditProject: (client, callback = (error) ->) ->
-		AuthorizationManager._assertClientHasPrivilegeLevel client, ["readAndWrite", "owner"], callback
-				
-	_assertClientHasPrivilegeLevel: (client, allowedLevels, callback = (error) ->) ->
-		if client.ol_context["privilege_level"] in allowedLevels
-			callback null
-		else
-			callback new Error("not authorized")
-
-	assertClientCanViewProjectAndDoc: (client, doc_id, callback = (error) ->) ->
-		AuthorizationManager.assertClientCanViewProject client, (error) ->
-			return callback(error) if error?
-			AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback
-
-	assertClientCanEditProjectAndDoc: (client, doc_id, callback = (error) ->) ->
-		AuthorizationManager.assertClientCanEditProject client, (error) ->
-			return callback(error) if error?
-			AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback
-
-	_assertClientCanAccessDoc: (client, doc_id, callback = (error) ->) ->
-		if client.ol_context["doc:#{doc_id}"] is "allowed"
-			callback null
-		else
-			callback new Error("not authorized")
-
-	addAccessToDoc: (client, doc_id, callback = (error) ->) ->
-		client.ol_context["doc:#{doc_id}"] = "allowed"
-		callback(null)
-
-	removeAccessToDoc: (client, doc_id, callback = (error) ->) ->
-		delete client.ol_context["doc:#{doc_id}"]
-		callback(null)

services/real-time/app/coffee/ChannelManager.coffee

@@ -1,71 +0,0 @@
-logger = require 'logger-sharelatex'
-metrics = require "metrics-sharelatex"
-settings = require "settings-sharelatex"
-
-ClientMap = new Map() # for each redis client, store a Map of subscribed channels (channelname -> subscribe promise)
-
-# Manage redis pubsub subscriptions for individual projects and docs, ensuring
-# that we never subscribe to a channel multiple times. The socket.io side is
-# handled by RoomManager.
-
-module.exports = ChannelManager =
-    getClientMapEntry: (rclient) ->
-        # return the per-client channel map if it exists, otherwise create and
-        # return an empty map for the client.
-        ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient)
-
-    subscribe: (rclient, baseChannel, id) ->
-        clientChannelMap = @getClientMapEntry(rclient)
-        channel = "#{baseChannel}:#{id}"
-        actualSubscribe = () ->
-            # subscribe is happening in the foreground and it should reject
-            p = rclient.subscribe(channel)
-            p.finally () ->
-                if clientChannelMap.get(channel) is subscribePromise
-                    clientChannelMap.delete(channel)
-            .then () ->
-                logger.log {channel}, "subscribed to channel"
-                metrics.inc "subscribe.#{baseChannel}"
-            .catch (err) ->
-                logger.error {channel, err}, "failed to subscribe to channel"
-                metrics.inc "subscribe.failed.#{baseChannel}"
-            return p
-
-        pendingActions = clientChannelMap.get(channel) || Promise.resolve()
-        subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe)
-        clientChannelMap.set(channel, subscribePromise)
-        logger.log {channel}, "planned to subscribe to channel"
-        return subscribePromise
-
-    unsubscribe: (rclient, baseChannel, id) ->
-        clientChannelMap = @getClientMapEntry(rclient)
-        channel = "#{baseChannel}:#{id}"
-        actualUnsubscribe = () ->
-            # unsubscribe is happening in the background, it should not reject
-            p = rclient.unsubscribe(channel)
-            .finally () ->
-                if clientChannelMap.get(channel) is unsubscribePromise
-                    clientChannelMap.delete(channel)
-            .then () ->
-                logger.log {channel}, "unsubscribed from channel"
-                metrics.inc "unsubscribe.#{baseChannel}"
-            .catch (err) ->
-                logger.error {channel, err}, "unsubscribed from channel"
-                metrics.inc "unsubscribe.failed.#{baseChannel}"
-            return p
-
-        pendingActions = clientChannelMap.get(channel) || Promise.resolve()
-        unsubscribePromise = pendingActions.then(actualUnsubscribe, actualUnsubscribe)
-        clientChannelMap.set(channel, unsubscribePromise)
-        logger.log {channel}, "planned to unsubscribe from channel"
-        return unsubscribePromise
-
-    publish: (rclient, baseChannel, id, data) ->
-        metrics.summary "redis.publish.#{baseChannel}", data.length
-        if id is 'all' or !settings.publishOnIndividualChannels
-            channel = baseChannel
-        else
-            channel = "#{baseChannel}:#{id}"
-        # we publish on a different client to the subscribe, so we can't
-        # check for the channel existing here
-        rclient.publish channel, data

services/real-time/app/coffee/ConnectedUsersManager.coffee

@@ -1,91 +0,0 @@
-async = require("async")
-Settings = require('settings-sharelatex')
-logger = require("logger-sharelatex")
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.realtime)
-Keys = Settings.redis.realtime.key_schema
-
-ONE_HOUR_IN_S = 60 * 60
-ONE_DAY_IN_S = ONE_HOUR_IN_S * 24
-FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4
-
-USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4
-REFRESH_TIMEOUT_IN_S = 10  # only show clients which have responded to a refresh request in the last 10 seconds
-
-module.exports =
-	# Use the same method for when a user connects, and when a user sends a cursor
-	# update. This way we don't care if the connected_user key has expired when
-	# we receive a cursor update. 
-	updateUserPosition: (project_id, client_id, user, cursorData, callback = (err)->)->
-		logger.log project_id:project_id, client_id:client_id, "marking user as joined or connected"
-
-		multi = rclient.multi()
-		
-		multi.sadd   Keys.clientsInProject({project_id}), client_id
-		multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S
-		
-		multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()
-		multi.hset Keys.connectedUser({project_id, client_id}), "user_id", user._id
-		multi.hset Keys.connectedUser({project_id, client_id}), "first_name", user.first_name or ""
-		multi.hset Keys.connectedUser({project_id, client_id}), "last_name", user.last_name or ""
-		multi.hset Keys.connectedUser({project_id, client_id}), "email", user.email or ""
-		
-		if cursorData?
-			multi.hset Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData)
-		multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S
-		
-		multi.exec (err)->
-			if err?
-				logger.err err:err, project_id:project_id, client_id:client_id, "problem marking user as connected"
-			callback(err)
-
-	refreshClient: (project_id, client_id, callback = (err) ->) ->
-		logger.log project_id:project_id, client_id:client_id, "refreshing connected client"
-		multi = rclient.multi()
-		multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()
-		multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S
-		multi.exec (err)->
-			if err?
-				logger.err err:err, project_id:project_id, client_id:client_id, "problem refreshing connected client"
-			callback(err)
-
-	markUserAsDisconnected: (project_id, client_id, callback)->
-		logger.log project_id:project_id, client_id:client_id, "marking user as disconnected"
-		multi = rclient.multi()
-		multi.srem Keys.clientsInProject({project_id}), client_id
-		multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S
-		multi.del Keys.connectedUser({project_id, client_id})
-		multi.exec callback
-
-
-	_getConnectedUser: (project_id, client_id, callback)->
-		rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)->
-			if !result? or Object.keys(result).length == 0 or !result.user_id
-				result =
-					connected : false
-					client_id:client_id
-			else
-				result.connected = true
-				result.client_id = client_id
-				result.client_age = (Date.now() - parseInt(result.last_updated_at,10)) / 1000
-				if result.cursorData?
-					try
-						result.cursorData = JSON.parse(result.cursorData)
-					catch e
-						logger.error {err: e, project_id, client_id, cursorData: result.cursorData}, "error parsing cursorData JSON" 
-						return callback e
-			callback err, result
-
-	getConnectedUsers: (project_id, callback)->
-		self = @
-		rclient.smembers Keys.clientsInProject({project_id}), (err, results)->
-			return callback(err) if err?
-			jobs = results.map (client_id)->
-				(cb)->
-					self._getConnectedUser(project_id, client_id, cb)
-			async.series jobs, (err, users = [])->
-				return callback(err) if err?
-				users = users.filter (user) ->
-					user?.connected && user?.client_age < REFRESH_TIMEOUT_IN_S
-				callback null, users
-

services/real-time/app/coffee/DocumentUpdaterController.coffee

@@ -1,88 +0,0 @@
-logger = require "logger-sharelatex"
-settings = require 'settings-sharelatex'
-RedisClientManager = require "./RedisClientManager"
-SafeJsonParse = require "./SafeJsonParse"
-EventLogger = require "./EventLogger"
-HealthCheckManager = require "./HealthCheckManager"
-RoomManager = require "./RoomManager"
-ChannelManager = require "./ChannelManager"
-metrics = require "metrics-sharelatex"
-
-MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb
-
-module.exports = DocumentUpdaterController =
-	# DocumentUpdaterController is responsible for updates that come via Redis
-	# Pub/Sub from the document updater.
-	rclientList: RedisClientManager.createClientList(settings.redis.pubsub)
-
-	listenForUpdatesFromDocumentUpdater: (io) ->
-		logger.log {rclients: @rclientList.length}, "listening for applied-ops events"
-		for rclient, i in @rclientList
-			rclient.subscribe "applied-ops"
-			rclient.on "message", (channel, message) ->
-				metrics.inc "rclient", 0.001 # global event rate metric
-				EventLogger.debugEvent(channel, message) if settings.debugEvents > 0
-				DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message)
-		# create metrics for each redis instance only when we have multiple redis clients
-		if @rclientList.length > 1
-			for rclient, i in @rclientList
-				do (i) ->
-					rclient.on "message", () ->
-						metrics.inc "rclient-#{i}", 0.001 # per client event rate metric
-		@handleRoomUpdates(@rclientList)
-
-	handleRoomUpdates: (rclientSubList) ->
-		roomEvents = RoomManager.eventSource()
-		roomEvents.on 'doc-active', (doc_id) ->
-			subscribePromises = for rclient in rclientSubList
-				ChannelManager.subscribe rclient, "applied-ops", doc_id
-			RoomManager.emitOnCompletion(subscribePromises, "doc-subscribed-#{doc_id}")
-		roomEvents.on 'doc-empty', (doc_id) ->
-			for rclient in rclientSubList
-				ChannelManager.unsubscribe rclient, "applied-ops", doc_id
-
-	_processMessageFromDocumentUpdater: (io, channel, message) ->
-		SafeJsonParse.parse message, (error, message) ->
-			if error?
-				logger.error {err: error, channel}, "error parsing JSON"
-				return
-			if message.op?
-				if message._id? && settings.checkEventOrder
-					status = EventLogger.checkEventOrder("applied-ops", message._id, message)
-					if status is 'duplicate'
-						return # skip duplicate events
-				DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op)
-			else if message.error?
-				DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message)
-			else if message.health_check?
-				logger.debug {message}, "got health check message in applied ops channel"
-				HealthCheckManager.check channel, message.key
-
-	_applyUpdateFromDocumentUpdater: (io, doc_id, update) ->
-		clientList = io.sockets.clients(doc_id)
-		# avoid unnecessary work if no clients are connected
-		if clientList.length is 0
-			return
-		# send updates to clients
-		logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, socketIoClients: (client.id for client in clientList), "distributing updates to clients"
-		seen = {}
-		# send messages only to unique clients (due to duplicate entries in io.sockets.clients)
-		for client in clientList when not seen[client.id]
-			seen[client.id] = true
-			if client.publicId == update.meta.source
-				logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender"
-				client.emit "otUpdateApplied", v: update.v, doc: update.doc
-			else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement
-				logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator"
-				client.emit "otUpdateApplied", update
-		if Object.keys(seen).length < clientList.length
-			metrics.inc "socket-io.duplicate-clients", 0.1
-			logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients"
-
-	_processErrorFromDocumentUpdater: (io, doc_id, error, message) ->
-		for client in io.sockets.clients(doc_id)
-			logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client"
-			client.emit "otUpdateError", error, message
-			client.disconnect()
-
-

services/real-time/app/coffee/DocumentUpdaterManager.coffee

@@ -1,83 +0,0 @@
-request = require "request"
-_ = require "underscore"
-logger = require "logger-sharelatex"
-settings = require "settings-sharelatex"
-metrics = require("metrics-sharelatex")
-
-rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater)
-Keys = settings.redis.documentupdater.key_schema
-
-module.exports = DocumentUpdaterManager =
-	getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) ->
-		timer = new metrics.Timer("get-document")
-		url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}"
-		logger.log {project_id, doc_id, fromVersion}, "getting doc from document updater"
-		request.get url, (err, res, body) ->
-			timer.done()
-			if err?
-				logger.error {err, url, project_id, doc_id}, "error getting doc from doc updater"
-				return callback(err)
-			if 200 <= res.statusCode < 300
-				logger.log {project_id, doc_id}, "got doc from document document updater"
-				try
-					body = JSON.parse(body)
-				catch error
-					return callback(error)
-				callback null, body?.lines, body?.version, body?.ranges, body?.ops
-			else if res.statusCode in [404, 422]
-				err = new Error("doc updater could not load requested ops")
-				err.statusCode = res.statusCode
-				logger.warn {err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops"
-				callback err
-			else
-				err = new Error("doc updater returned a non-success status code: #{res.statusCode}")
-				err.statusCode = res.statusCode
-				logger.error {err, project_id, doc_id, url}, "doc updater returned a non-success status code: #{res.statusCode}"
-				callback err
-
-	flushProjectToMongoAndDelete: (project_id, callback = ()->) ->
-		# this method is called when the last connected user leaves the project
-		logger.log project_id:project_id, "deleting project from document updater"
-		timer = new metrics.Timer("delete.mongo.project")
-		# flush the project in the background when all users have left
-		url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" +
-			(if settings.shutDownInProgress then "&shutdown=true" else "")
-		request.del url, (err, res, body)->
-			timer.done()
-			if err?
-				logger.error {err, project_id}, "error deleting project from document updater"
-				return callback(err)
-			else if 200 <= res.statusCode  < 300
-				logger.log {project_id}, "deleted project from document updater"
-				return callback(null)
-			else
-				err = new Error("document updater returned a failure status code: #{res.statusCode}")
-				err.statusCode = res.statusCode
-				logger.error {err, project_id}, "document updater returned failure status code: #{res.statusCode}"
-				return callback(err)
-
-	queueChange: (project_id, doc_id, change, callback = ()->)->
-		allowedKeys = [ 'doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash']
-		change = _.pick change, allowedKeys
-		jsonChange = JSON.stringify change
-		if jsonChange.indexOf("\u0000") != -1
-			# memory corruption check
-			error = new Error("null bytes found in op")
-			logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message
-			return callback(error)
-
-		updateSize = jsonChange.length
-		if updateSize > settings.maxUpdateSize
-			error = new Error("update is too large")
-			error.updateSize = updateSize
-			return callback(error)
-
-		# record metric for each update added to queue
-		metrics.summary 'redis.pendingUpdates', updateSize, {status: 'push'}
-
-		doc_key = "#{project_id}:#{doc_id}"
-		# Push onto pendingUpdates for doc_id first, because once the doc updater
-		# gets an entry on pending-updates-list, it starts processing.
-		rclient.rpush Keys.pendingUpdates({doc_id}), jsonChange, (error) ->
-			return callback(error) if error?
-			rclient.rpush "pending-updates-list", doc_key, callback

services/real-time/app/coffee/DrainManager.coffee

@@ -1,39 +0,0 @@
-logger = require "logger-sharelatex"
-
-module.exports = DrainManager =
-
-	startDrainTimeWindow: (io, minsToDrain)->
-		drainPerMin = io.sockets.clients().length / minsToDrain
-		DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate
-
-	startDrain: (io, rate) ->
-		# Clear out any old interval
-		clearInterval @interval
-		logger.log  rate: rate, "starting drain"
-		if rate == 0
-			return
-		else if rate < 1
-			# allow lower drain rates
-			# e.g. rate=0.1 will drain one client every 10 seconds
-			pollingInterval = 1000 / rate
-			rate = 1
-		else
-			pollingInterval = 1000
-		@interval = setInterval () =>
-			@reconnectNClients(io, rate)
-		, pollingInterval
-
-	RECONNECTED_CLIENTS: {}
-	reconnectNClients: (io, N) ->
-		drainedCount = 0
-		for client in io.sockets.clients()
-			if !@RECONNECTED_CLIENTS[client.id]
-				@RECONNECTED_CLIENTS[client.id] = true
-				logger.log {client_id: client.id}, "Asking client to reconnect gracefully"
-				client.emit "reconnectGracefully"
-				drainedCount++
-			haveDrainedNClients = (drainedCount == N)
-			if haveDrainedNClients
-				break
-		if drainedCount < N
-			logger.log "All clients have been told to reconnectGracefully"

services/real-time/app/coffee/Errors.coffee

@@ -1,10 +0,0 @@
-CodedError = (message, code) ->
-	error = new Error(message)
-	error.name = "CodedError"
-	error.code = code
-	error.__proto__ = CodedError.prototype
-	return error
-CodedError.prototype.__proto__ = Error.prototype
-
-module.exports = Errors =
-	CodedError: CodedError

services/real-time/app/coffee/EventLogger.coffee

@@ -1,60 +0,0 @@
-logger = require 'logger-sharelatex'
-metrics = require 'metrics-sharelatex'
-settings = require 'settings-sharelatex'
-
-# keep track of message counters to detect duplicate and out of order events
-# messsage ids have the format "UNIQUEHOSTKEY-COUNTER"
-
-EVENT_LOG_COUNTER = {}
-EVENT_LOG_TIMESTAMP = {}
-EVENT_LAST_CLEAN_TIMESTAMP = 0
-
-# counter for debug logs
-COUNTER = 0
-
-module.exports = EventLogger =
-
-	MAX_STALE_TIME_IN_MS: 3600 * 1000
-
-	debugEvent: (channel, message) ->
-		if settings.debugEvents > 0
-			logger.log {channel:channel, message:message, counter: COUNTER++}, "logging event"
-			settings.debugEvents--
-
-	checkEventOrder: (channel, message_id, message) ->
-		return if typeof(message_id) isnt 'string'
-		return if !(result = message_id.match(/^(.*)-(\d+)$/))
-		key = result[1]
-		count = parseInt(result[2], 0)
-		if !(count >= 0)# ignore checks if counter is not present
-			return
-		# store the last count in a hash for each host
-		previous = EventLogger._storeEventCount(key, count)
-		if !previous? || count == (previous + 1)
-			metrics.inc "event.#{channel}.valid", 0.001 # downsample high rate docupdater events
-			return # order is ok
-		if (count == previous)
-			metrics.inc "event.#{channel}.duplicate"
-			logger.warn {channel:channel, message_id:message_id}, "duplicate event"
-			return "duplicate"
-		else
-			metrics.inc "event.#{channel}.out-of-order"
-			logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event"
-			return "out-of-order"
-
-	_storeEventCount: (key, count) ->
-		previous = EVENT_LOG_COUNTER[key]
-		now = Date.now()
-		EVENT_LOG_COUNTER[key] = count
-		EVENT_LOG_TIMESTAMP[key] = now
-		# periodically remove old counts
-		if (now - EVENT_LAST_CLEAN_TIMESTAMP) > EventLogger.MAX_STALE_TIME_IN_MS
-			EventLogger._cleanEventStream(now)
-			EVENT_LAST_CLEAN_TIMESTAMP = now
-		return previous
-
-	_cleanEventStream: (now) ->
-		for key, timestamp of EVENT_LOG_TIMESTAMP
-			if (now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS
-				delete EVENT_LOG_COUNTER[key]
-				delete EVENT_LOG_TIMESTAMP[key]

services/real-time/app/coffee/HealthCheckManager.coffee

@@ -1,52 +0,0 @@
-metrics = require "metrics-sharelatex"
-logger = require("logger-sharelatex")
-
-os = require "os"
-HOST = os.hostname()
-PID = process.pid
-COUNT = 0
-
-CHANNEL_MANAGER = {} # hash of event checkers by channel name
-CHANNEL_ERROR = {} # error status by channel name
-
-module.exports = class HealthCheckManager
-    # create an instance of this class which checks that an event with a unique
-    # id is received only once within a timeout
-    constructor: (@channel, timeout = 1000) ->
-        # unique event string
-        @id = "host=#{HOST}:pid=#{PID}:count=#{COUNT++}"
-        # count of number of times the event is received
-        @count = 0 
-        # after a timeout check the status of the count
-        @handler = setTimeout () =>
-            @setStatus()
-        , timeout
-        # use a timer to record the latency of the channel
-        @timer = new metrics.Timer("event.#{@channel}.latency")
-        # keep a record of these objects to dispatch on
-        CHANNEL_MANAGER[@channel] = @
-    processEvent: (id) ->
-        # if this is our event record it
-        if id == @id
-            @count++
-            @timer?.done()
-            @timer = null # only time the latency of the first event
-    setStatus: () ->
-        # if we saw the event anything other than a single time that is an error
-        if @count != 1
-            logger.err channel:@channel, count:@count, id:@id, "redis channel health check error"
-        error = (@count != 1)
-        CHANNEL_ERROR[@channel] = error
-
-    # class methods
-    @check: (channel, id) ->
-        # dispatch event to manager for channel
-        CHANNEL_MANAGER[channel]?.processEvent id
-    @status: () ->
-        # return status of all channels for logging
-        return CHANNEL_ERROR
-    @isFailing: () -> 
-        # check if any channel status is bad 
-        for channel, error of CHANNEL_ERROR
-            return true if error is true
-        return false

services/real-time/app/coffee/HttpApiController.coffee

@@ -1,35 +0,0 @@
-WebsocketLoadBalancer = require "./WebsocketLoadBalancer"
-DrainManager = require "./DrainManager"
-logger = require "logger-sharelatex"
-
-module.exports = HttpApiController =
-	sendMessage: (req, res, next) ->
-		logger.log {message: req.params.message}, "sending message"
-		if Array.isArray(req.body)
-			for payload in req.body
-				WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, payload
-		else
-			WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body
-		res.send 204 # No content
-	
-	startDrain: (req, res, next) ->
-		io = req.app.get("io")
-		rate = req.query.rate or "4"
-		rate = parseFloat(rate) || 0
-		logger.log {rate}, "setting client drain rate"
-		DrainManager.startDrain io, rate
-		res.send 204
-
-	disconnectClient: (req, res, next) ->
-		io = req.app.get("io")
-		client_id = req.params.client_id
-		client = io.sockets.sockets[client_id]
-
-		if !client
-			logger.info({client_id}, "api: client already disconnected")
-			res.sendStatus(404)
-			return
-		logger.warn({client_id}, "api: requesting client disconnect")
-		client.on "disconnect", () ->
-			res.sendStatus(204)
-		client.disconnect()

services/real-time/app/coffee/HttpController.coffee

@@ -1,35 +0,0 @@
-async = require "async"
-
-module.exports = HttpController =
-	# The code in this controller is hard to unit test because of a lot of 
-	# dependencies on internal socket.io methods. It is not critical to the running
-	# of ShareLaTeX, and is only used for getting stats about connected clients,
-	# and for checking internal state in acceptance tests. The acceptances tests
-	# should provide appropriate coverage.
-	_getConnectedClientView: (ioClient, callback = (error, client) ->) ->
-			client_id = ioClient.id
-			{project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context
-			client = {client_id, project_id, user_id, first_name, last_name, email, connected_time}
-			client.rooms = []
-			for name, joined of ioClient.manager.roomClients[client_id]
-				if joined and name != ""
-					client.rooms.push name.replace(/^\//, "") # Remove leading /
-			callback(null, client)
-
-	getConnectedClients: (req, res, next) ->
-		io = req.app.get("io")
-		ioClients = io.sockets.clients()
-		async.map ioClients, HttpController._getConnectedClientView, (error, clients) ->
-			return next(error) if error?
-			res.json clients
-			
-	getConnectedClient: (req, res, next) ->
-		{client_id} = req.params
-		io = req.app.get("io")
-		ioClient = io.sockets.sockets[client_id]
-		if !ioClient
-			res.sendStatus(404)
-			return
-		HttpController._getConnectedClientView ioClient, (error, client) ->
-			return next(error) if error?
-			res.json client

services/real-time/app/coffee/RedisClientManager.coffee

@@ -1,18 +0,0 @@
-redis = require("redis-sharelatex")
-logger = require 'logger-sharelatex'
-
-module.exports = RedisClientManager =
-    createClientList: (configs...) ->
-        # create a dynamic list of redis clients, excluding any configurations which are not defined
-        clientList = for x in configs when x?
-            redisType = if x.cluster?
-                "cluster"
-            else if x.sentinels?
-                "sentinel"
-            else if x.host?
-                "single"
-            else
-                "unknown"
-            logger.log {redis: redisType}, "creating redis client"
-            redis.createClient(x)
-        return clientList

services/real-time/app/coffee/RoomManager.coffee

@@ -1,110 +0,0 @@
-logger = require 'logger-sharelatex'
-metrics = require "metrics-sharelatex"
-{EventEmitter} = require 'events'
-
-IdMap = new Map() # keep track of whether ids are from projects or docs
-RoomEvents = new EventEmitter() # emits {project,doc}-active and {project,doc}-empty events
-
-# Manage socket.io rooms for individual projects and docs
-#
-# The first time someone joins a project or doc we emit a 'project-active' or
-# 'doc-active' event.
-#
-# When the last person leaves a project or doc, we emit 'project-empty' or
-# 'doc-empty' event.
-#
-# The pubsub side is handled by ChannelManager
-
-module.exports = RoomManager =
-
-    joinProject: (client, project_id, callback = () ->) ->
-        @joinEntity client, "project", project_id, callback
-
-    joinDoc: (client, doc_id, callback = () ->) ->
-        @joinEntity client, "doc", doc_id, callback
-
-    leaveDoc: (client, doc_id) ->
-        @leaveEntity client, "doc", doc_id
-
-    leaveProjectAndDocs: (client) ->
-        # what rooms is this client in? we need to leave them all. socket.io
-        # will cause us to leave the rooms, so we only need to manage our
-        # channel subscriptions... but it will be safer if we leave them
-        # explicitly, and then socket.io will just regard this as a client that
-        # has not joined any rooms and do a final disconnection.
-        roomsToLeave = @_roomsClientIsIn(client)
-        logger.log {client: client.id, roomsToLeave: roomsToLeave}, "client leaving project"
-        for id in roomsToLeave
-            entity = IdMap.get(id)
-            @leaveEntity client, entity, id
-
-    emitOnCompletion: (promiseList, eventName) ->
-        Promise.all(promiseList)
-          .then(() -> RoomEvents.emit(eventName))
-          .catch((err) -> RoomEvents.emit(eventName, err))
-
-    eventSource: () ->
-        return RoomEvents
-
-    joinEntity: (client, entity, id, callback) ->
-        beforeCount = @_clientsInRoom(client, id)
-        # client joins room immediately but joinDoc request does not complete
-        # until room is subscribed
-        client.join id
-        # is this a new room? if so, subscribe
-        if beforeCount == 0
-            logger.log {entity, id}, "room is now active"
-            RoomEvents.once "#{entity}-subscribed-#{id}", (err) ->
-                # only allow the client to join when all the relevant channels have subscribed
-                logger.log {client: client.id, entity, id, beforeCount}, "client joined new room and subscribed to channel"
-                callback(err)
-            RoomEvents.emit "#{entity}-active", id
-            IdMap.set(id, entity)
-            # keep track of the number of listeners
-            metrics.gauge "room-listeners", RoomEvents.eventNames().length
-        else
-            logger.log {client: client.id, entity, id, beforeCount}, "client joined existing room"
-            client.join id
-            callback()
-
-    leaveEntity: (client, entity, id) ->
-        # Ignore any requests to leave when the client is not actually in the
-        # room. This can happen if the client sends spurious leaveDoc requests
-        # for old docs after a reconnection.
-        # This can now happen all the time, as we skip the join for clients that
-        #  disconnect before joinProject/joinDoc completed.
-        if !@_clientAlreadyInRoom(client, id)
-            logger.log {client: client.id, entity, id}, "ignoring request from client to leave room it is not in"
-            return
-        client.leave id
-        afterCount = @_clientsInRoom(client, id)
-        logger.log {client: client.id, entity, id, afterCount}, "client left room"
-        # is the room now empty? if so, unsubscribe
-        if !entity?
-            logger.error {entity: id}, "unknown entity when leaving with id"
-            return
-        if afterCount == 0
-            logger.log {entity, id}, "room is now empty"
-            RoomEvents.emit "#{entity}-empty", id
-            IdMap.delete(id)
-            metrics.gauge "room-listeners", RoomEvents.eventNames().length
-
-    # internal functions below, these access socket.io rooms data directly and
-    # will need updating for socket.io v2
-
-    _clientsInRoom: (client, room) ->
-        nsp = client.namespace.name
-        name = (nsp + '/') + room;
-        return (client.manager?.rooms?[name] || []).length
-
-    _roomsClientIsIn: (client) ->
-        roomList = for fullRoomPath of client.manager.roomClients?[client.id] when fullRoomPath isnt ''
-            # strip socket.io prefix from room to get original id
-            [prefix, room] = fullRoomPath.split('/', 2)
-            room
-        return roomList
-
-    _clientAlreadyInRoom: (client, room) ->
-        nsp = client.namespace.name
-        name = (nsp + '/') + room;
-        return client.manager.roomClients?[client.id]?[name]

services/real-time/app/coffee/Router.coffee

@@ -1,188 +0,0 @@
-metrics = require "metrics-sharelatex"
-logger = require "logger-sharelatex"
-settings = require "settings-sharelatex"
-WebsocketController = require "./WebsocketController"
-HttpController = require "./HttpController"
-HttpApiController = require "./HttpApiController"
-bodyParser = require "body-parser"
-base64id = require("base64id")
-
-basicAuth = require('basic-auth-connect')
-httpAuth = basicAuth (user, pass)->
-	isValid = user == settings.internal.realTime.user and pass == settings.internal.realTime.pass
-	if !isValid
-		logger.err user:user, pass:pass, "invalid login details"
-	return isValid
-
-module.exports = Router =
-	_handleError: (callback = ((error) ->), error, client, method, attrs = {}) ->
-			for key in ["project_id", "doc_id", "user_id"]
-				attrs[key] = client.ol_context[key]
-			attrs.client_id = client.id
-			attrs.err = error
-			if error.name == "CodedError"
-				logger.warn attrs, error.message, code: error.code
-				return callback {message: error.message, code: error.code}
-			if error.message == 'unexpected arguments'
-				# the payload might be very large, put it on level info
-				logger.log attrs, 'unexpected arguments'
-				metrics.inc 'unexpected-arguments', 1, { status: method }
-				return callback { message: error.message }
-			if error.message in ["not authorized", "doc updater could not load requested ops", "no project_id found on client"]
-				logger.warn attrs, error.message
-				return callback {message: error.message}
-			else
-				logger.error attrs, "server side error in #{method}"
-				# Don't return raw error to prevent leaking server side info
-				return callback {message: "Something went wrong in real-time service"}
-
-	_handleInvalidArguments: (client, method, args) ->
-		error = new Error("unexpected arguments")
-		callback = args[args.length - 1]
-		if typeof callback != 'function'
-			callback = (() ->)
-		attrs = {arguments: args}
-		Router._handleError(callback, error, client, method, attrs)
-
-	configure: (app, io, session) ->
-		app.set("io", io)
-		app.get "/clients", HttpController.getConnectedClients
-		app.get "/clients/:client_id", HttpController.getConnectedClient
-
-		app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage
-		
-		app.post "/drain", httpAuth, HttpApiController.startDrain
-		app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient
-
-		session.on 'connection', (error, client, session) ->
-			# init client context, we may access it in Router._handleError before
-			#  setting any values
-			client.ol_context = {}
-
-			client?.on "error", (err) ->
-				logger.err { clientErr: err }, "socket.io client error"
-				if client.connected
-					client.emit("reconnectGracefully")
-					client.disconnect()
-
-			if settings.shutDownInProgress
-				client.emit("connectionRejected", {message: "retry"})
-				client.disconnect()
-				return
-
-			if client? and error?.message?.match(/could not look up session by key/)
-				logger.warn err: error, client: client?, session: session?, "invalid session"
-				# tell the client to reauthenticate if it has an invalid session key
-				client.emit("connectionRejected", {message: "invalid session"})
-				client.disconnect()
-				return
-
-			if error?
-				logger.err err: error, client: client?, session: session?, "error when client connected"
-				client?.emit("connectionRejected", {message: "error"})
-				client?.disconnect()
-				return
-
-			# send positive confirmation that the client has a valid connection
-			client.publicId = 'P.' + base64id.generateId()
-			client.emit("connectionAccepted", null, client.publicId)
-
-			metrics.inc('socket-io.connection')
-			metrics.gauge('socket-io.clients', io.sockets.clients()?.length)
-
-			logger.log session: session, client_id: client.id, "client connected"
-
-			if session?.passport?.user?
-				user = session.passport.user
-			else if session?.user?
-				user = session.user
-			else
-				user = {_id: "anonymous-user"}
-
-			client.on "joinProject", (data = {}, callback) ->
-				if typeof callback != 'function'
-					return Router._handleInvalidArguments(client, 'joinProject', arguments)
-
-				if data.anonymousAccessToken
-					user.anonymousAccessToken = data.anonymousAccessToken
-				WebsocketController.joinProject client, user, data.project_id, (err, args...) ->
-					if err?
-						Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id}
-					else
-						callback(null, args...)
-
-			client.on "disconnect", () ->
-				metrics.inc('socket-io.disconnect')
-				metrics.gauge('socket-io.clients', io.sockets.clients()?.length - 1)
-
-				WebsocketController.leaveProject io, client, (err) ->
-					if err?
-						Router._handleError (() ->), err, client, "leaveProject"
-
-			# Variadic. The possible arguments:
-			# doc_id, callback
-			# doc_id, fromVersion, callback
-			# doc_id, options, callback
-			# doc_id, fromVersion, options, callback
-			client.on "joinDoc", (doc_id, fromVersion, options, callback) ->
-				if typeof fromVersion == "function" and !options
-					callback = fromVersion
-					fromVersion = -1
-					options = {}
-				else if typeof fromVersion == "number" and typeof options == "function"
-					callback = options
-					options = {}
-				else if typeof fromVersion == "object" and typeof options == "function"
-					callback = options
-					options = fromVersion
-					fromVersion = -1
-				else if typeof fromVersion == "number" and typeof options == "object" and typeof callback == 'function'
-					# Called with 4 args, things are as expected
-				else
-					return Router._handleInvalidArguments(client, 'joinDoc', arguments)
-
-				WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) ->
-					if err?
-						Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion}
-					else
-						callback(null, args...)
-
-			client.on "leaveDoc", (doc_id, callback) ->
-				if typeof callback != 'function'
-					return Router._handleInvalidArguments(client, 'leaveDoc', arguments)
-
-				WebsocketController.leaveDoc client, doc_id, (err, args...) ->
-					if err?
-						Router._handleError callback, err, client, "leaveDoc"
-					else
-						callback(null, args...)
-
-			client.on "clientTracking.getConnectedUsers", (callback = (error, users) ->) ->
-				if typeof callback != 'function'
-					return Router._handleInvalidArguments(client, 'clientTracking.getConnectedUsers', arguments)
-
-				WebsocketController.getConnectedUsers client, (err, users) ->
-					if err?
-						Router._handleError callback, err, client, "clientTracking.getConnectedUsers"
-					else
-						callback(null, users)
-
-			client.on "clientTracking.updatePosition", (cursorData, callback = (error) ->) ->
-				if typeof callback != 'function'
-					return Router._handleInvalidArguments(client, 'clientTracking.updatePosition', arguments)
-
-				WebsocketController.updateClientPosition client, cursorData, (err) ->
-					if err?
-						Router._handleError callback, err, client, "clientTracking.updatePosition"
-					else
-						callback()
-
-			client.on "applyOtUpdate", (doc_id, update, callback = (error) ->) ->
-				if typeof callback != 'function'
-					return Router._handleInvalidArguments(client, 'applyOtUpdate', arguments)
-
-				WebsocketController.applyOtUpdate client, doc_id, update, (err) ->
-					if err?
-						Router._handleError callback, err, client, "applyOtUpdate", {doc_id, update}
-					else
-						callback()

services/real-time/app/coffee/SafeJsonParse.coffee

@@ -1,13 +0,0 @@
-Settings = require "settings-sharelatex"
-logger = require "logger-sharelatex"
-
-module.exports =
-	parse: (data, callback = (error, parsed) ->) ->
-		if data.length > Settings.maxUpdateSize
-			logger.error {head: data.slice(0,1024), length: data.length}, "data too large to parse"
-			return callback new Error("data too large to parse")
-		try
-			parsed = JSON.parse(data)
-		catch e
-			return callback e
-		callback null, parsed

services/real-time/app/coffee/SessionSockets.coffee

@@ -1,23 +0,0 @@
-{EventEmitter} = require('events')
-
-module.exports = (io, sessionStore, cookieParser, cookieName) ->
-	missingSessionError = new Error('could not look up session by key')
-
-	sessionSockets = new EventEmitter()
-	next = (error, socket, session) ->
-		sessionSockets.emit 'connection', error, socket, session
-
-	io.on 'connection', (socket) ->
-		req = socket.handshake
-		cookieParser req, {}, () ->
-			sessionId = req.signedCookies and req.signedCookies[cookieName]
-			if not sessionId
-				return next(missingSessionError, socket)
-			sessionStore.get sessionId, (error, session) ->
-				if error
-					return next(error, socket)
-				if not session
-					return next(missingSessionError, socket)
-				next(null, socket, session)
-
-	return sessionSockets

services/real-time/app/coffee/WebApiManager.coffee

@@ -1,38 +0,0 @@
-request = require "request"
-settings = require "settings-sharelatex"
-logger = require "logger-sharelatex"
-{ CodedError } = require "./Errors"
-
-module.exports = WebApiManager =
-	joinProject: (project_id, user, callback = (error, project, privilegeLevel, isRestrictedUser) ->) ->
-		user_id = user._id
-		logger.log {project_id, user_id}, "sending join project request to web"
-		url = "#{settings.apis.web.url}/project/#{project_id}/join"
-		headers = {}
-		if user.anonymousAccessToken?
-			headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken
-		request.post {
-			url: url
-			qs: {user_id}
-			auth:
-				user: settings.apis.web.user
-				pass: settings.apis.web.pass
-				sendImmediately: true
-			json: true
-			jar: false
-			headers: headers
-		}, (error, response, data) ->
-			return callback(error) if error?
-			if 200 <= response.statusCode < 300
-				if !data? || !data?.project?
-					err = new Error('no data returned from joinProject request')
-					logger.error {err, project_id, user_id}, "error accessing web api"
-					return callback(err)
-				callback null, data.project, data.privilegeLevel, data.isRestrictedUser
-			else if response.statusCode == 429
-				logger.log(project_id, user_id, "rate-limit hit when joining project")
-				callback(new CodedError("rate-limit hit when joining project", "TooManyRequests"))
-			else
-				err = new Error("non-success status code from web: #{response.statusCode}")
-				logger.error {err, project_id, user_id}, "error accessing web api"
-				callback err

services/real-time/app/coffee/WebsocketController.coffee

@@ -1,276 +0,0 @@
-logger = require "logger-sharelatex"
-metrics = require "metrics-sharelatex"
-settings = require "settings-sharelatex"
-WebApiManager = require "./WebApiManager"
-AuthorizationManager = require "./AuthorizationManager"
-DocumentUpdaterManager = require "./DocumentUpdaterManager"
-ConnectedUsersManager = require "./ConnectedUsersManager"
-WebsocketLoadBalancer = require "./WebsocketLoadBalancer"
-RoomManager = require "./RoomManager"
-
-module.exports = WebsocketController =
-	# If the protocol version changes when the client reconnects,
-	# it will force a full refresh of the page. Useful for non-backwards
-	# compatible protocol changes. Use only in extreme need.
-	PROTOCOL_VERSION: 2
-
-	joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) ->
-		if client.disconnected
-			metrics.inc('editor.join-project.disconnected', 1, {status: 'immediately'})
-			return callback()
-
-		user_id = user?._id
-		logger.log {user_id, project_id, client_id: client.id}, "user joining project"
-		metrics.inc "editor.join-project"
-		WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) ->
-			return callback(error) if error?
-			if client.disconnected
-				metrics.inc('editor.join-project.disconnected', 1, {status: 'after-web-api-call'})
-				return callback()
-
-			if !privilegeLevel or privilegeLevel == ""
-				err = new Error("not authorized")
-				logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project"
-				return callback(err)
-
-			client.ol_context = {}
-			client.ol_context["privilege_level"] = privilegeLevel
-			client.ol_context["user_id"] = user_id
-			client.ol_context["project_id"] = project_id
-			client.ol_context["owner_id"] = project?.owner?._id
-			client.ol_context["first_name"] = user?.first_name
-			client.ol_context["last_name"] = user?.last_name
-			client.ol_context["email"] = user?.email
-			client.ol_context["connected_time"] = new Date()
-			client.ol_context["signup_date"] = user?.signUpDate
-			client.ol_context["login_count"] = user?.loginCount
-			client.ol_context["is_restricted_user"] = !!(isRestrictedUser)
-
-			RoomManager.joinProject client, project_id, (err) ->
-				return callback(err) if err
-				logger.log {user_id, project_id, client_id: client.id}, "user joined project"
-				callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION
-
-			# No need to block for setting the user as connected in the cursor tracking
-			ConnectedUsersManager.updateUserPosition project_id, client.publicId, user, null, () ->
-
-	# We want to flush a project if there are no more (local) connected clients
-	# but we need to wait for the triggering client to disconnect. How long we wait
-	# is determined by FLUSH_IF_EMPTY_DELAY.
-	FLUSH_IF_EMPTY_DELAY: 500 #ms
-	leaveProject: (io, client, callback = (error) ->) ->
-			{project_id, user_id} = client.ol_context
-			return callback() unless project_id # client did not join project
-
-			metrics.inc "editor.leave-project"
-			logger.log {project_id, user_id, client_id: client.id}, "client leaving project"
-			WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.publicId
-
-			# We can do this in the background
-			ConnectedUsersManager.markUserAsDisconnected project_id, client.publicId, (err) ->
-				if err?
-					logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected"
-
-			RoomManager.leaveProjectAndDocs(client)
-			setTimeout () ->
-				remainingClients = io.sockets.clients(project_id)
-				if remainingClients.length == 0
-					# Flush project in the background
-					DocumentUpdaterManager.flushProjectToMongoAndDelete project_id, (err) ->
-						if err?
-							logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project"
-				callback()
-			, WebsocketController.FLUSH_IF_EMPTY_DELAY
-
-	joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) ->
-			if client.disconnected
-				metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'})
-				return callback()
-
-			metrics.inc "editor.join-doc"
-			{project_id, user_id, is_restricted_user} = client.ol_context
-			return callback(new Error("no project_id found on client")) if !project_id?
-			logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc"
-
-			AuthorizationManager.assertClientCanViewProject client, (error) ->
-				return callback(error) if error?
-				# ensure the per-doc applied-ops channel is subscribed before sending the
-				# doc to the client, so that no events are missed.
-				RoomManager.joinDoc client, doc_id, (error) ->
-					return callback(error) if error?
-					if client.disconnected
-						metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-joining-room'})
-						# the client will not read the response anyways
-						return callback()
-
-					DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) ->
-						return callback(error) if error?
-						if client.disconnected
-							metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'})
-							# the client will not read the response anyways
-							return callback()
-
-						if is_restricted_user and ranges?.comments?
-							ranges.comments = []
-
-						# Encode any binary bits of data so it can go via WebSockets
-						# See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html
-						encodeForWebsockets = (text) -> unescape(encodeURIComponent(text))
-						escapedLines = []
-						for line in lines
-							try
-								line = encodeForWebsockets(line)
-							catch err
-								logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component"
-								return callback(err)
-							escapedLines.push line
-						if options.encodeRanges
-							try
-								for comment in ranges?.comments or []
-									comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c?
-								for change in ranges?.changes or []
-									change.op.i = encodeForWebsockets(change.op.i) if change.op.i?
-									change.op.d = encodeForWebsockets(change.op.d) if change.op.d?
-							catch err
-								logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component"
-								return callback(err)
-
-						AuthorizationManager.addAccessToDoc client, doc_id
-						logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc"
-						callback null, escapedLines, version, ops, ranges
-
-	leaveDoc: (client, doc_id, callback = (error) ->) ->
-			# client may have disconnected, but we have to cleanup internal state.
-			metrics.inc "editor.leave-doc"
-			{project_id, user_id} = client.ol_context
-			logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc"
-			RoomManager.leaveDoc(client, doc_id)
-			# we could remove permission when user leaves a doc, but because
-			# the connection is per-project, we continue to allow access
-			# after the initial joinDoc since we know they are already authorised.
-			## AuthorizationManager.removeAccessToDoc client, doc_id
-			callback()
-	updateClientPosition: (client, cursorData, callback = (error) ->) ->
-			if client.disconnected
-				# do not create a ghost entry in redis
-				return callback()
-
-			metrics.inc "editor.update-client-position", 0.1
-			{project_id, first_name, last_name, email, user_id} = client.ol_context
-			logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position"
-
-			AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) ->
-				if error?
-					logger.warn {err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet."
-					return callback()
-				cursorData.id      = client.publicId
-				cursorData.user_id = user_id if user_id?
-				cursorData.email   = email   if email?
-				# Don't store anonymous users in redis to avoid influx
-				if !user_id or user_id == 'anonymous-user'
-					cursorData.name = ""
-					callback()
-				else
-					cursorData.name = if first_name && last_name
-						"#{first_name} #{last_name}"
-					else if first_name
-						first_name
-					else if last_name
-						last_name
-					else
-						""
-					ConnectedUsersManager.updateUserPosition(project_id, client.publicId, {
-						first_name: first_name,
-						last_name:  last_name,
-						email:      email,
-						_id:        user_id
-					}, {
-						row:    cursorData.row,
-						column: cursorData.column,
-						doc_id: cursorData.doc_id
-					}, callback)
-				WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData)
-
-	CLIENT_REFRESH_DELAY: 1000
-	getConnectedUsers: (client, callback = (error, users) ->) ->
-			if client.disconnected
-				# they are not interested anymore, skip the redis lookups
-				return callback()
-
-			metrics.inc "editor.get-connected-users"
-			{project_id, user_id, is_restricted_user} = client.ol_context
-			if is_restricted_user
-				return callback(null, [])
-			return callback(new Error("no project_id found on client")) if !project_id?
-			logger.log {user_id, project_id, client_id: client.id}, "getting connected users"
-			AuthorizationManager.assertClientCanViewProject client, (error) ->
-				return callback(error) if error?
-				WebsocketLoadBalancer.emitToRoom project_id, 'clientTracking.refresh'
-				setTimeout () ->
-					ConnectedUsersManager.getConnectedUsers project_id, (error, users) ->
-						return callback(error) if error?
-						callback null, users
-						logger.log {user_id, project_id, client_id: client.id}, "got connected users"
-				, WebsocketController.CLIENT_REFRESH_DELAY
-
-	applyOtUpdate: (client, doc_id, update, callback = (error) ->) ->
-			# client may have disconnected, but we can submit their update to doc-updater anyways.
-			{user_id, project_id} = client.ol_context
-			return callback(new Error("no project_id found on client")) if !project_id?
-
-			WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) ->
-				if error?
-					logger.warn {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update"
-					setTimeout () ->
-						# Disconnect, but give the client the chance to receive the error
-						client.disconnect()
-					, 100
-					return callback(error)
-				update.meta ||= {}
-				update.meta.source = client.publicId
-				update.meta.user_id = user_id
-				metrics.inc "editor.doc-update", 0.3
-
-				logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater"
-
-				DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) ->
-					if error?.message == "update is too large"
-						metrics.inc "update_too_large"
-						updateSize = error.updateSize
-						logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large")
-
-						# mark the update as received -- the client should not send it again!
-						callback()
-
-						# trigger an out-of-sync error
-						message = {project_id, doc_id, error: "update is too large"}
-						setTimeout () ->
-							if client.disconnected
-								# skip the message broadcast, the client has moved on
-								return metrics.inc('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'})
-							client.emit "otUpdateError", message.error, message
-							client.disconnect()
-						, 100
-						return
-
-					if error?
-						logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update"
-						client.disconnect()
-					callback(error)
-
-	_assertClientCanApplyUpdate: (client, doc_id, update, callback) ->
-		AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) ->
-			if error?
-				if error.message == "not authorized" and WebsocketController._isCommentUpdate(update)
-					# This might be a comment op, which we only need read-only priveleges for
-					AuthorizationManager.assertClientCanViewProjectAndDoc client, doc_id, callback
-				else
-					return callback(error)
-			else
-				return callback(null)
-
-	_isCommentUpdate: (update) ->
-		for op in update.op
-			if !op.c?
-				return false
-		return true

services/real-time/app/coffee/WebsocketLoadBalancer.coffee

@@ -1,104 +0,0 @@
-Settings = require 'settings-sharelatex'
-logger = require 'logger-sharelatex'
-RedisClientManager = require "./RedisClientManager"
-SafeJsonParse = require "./SafeJsonParse"
-EventLogger = require "./EventLogger"
-HealthCheckManager = require "./HealthCheckManager"
-RoomManager = require "./RoomManager"
-ChannelManager = require "./ChannelManager"
-ConnectedUsersManager = require "./ConnectedUsersManager"
-
-RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [
-	'connectionAccepted',
-	'otUpdateApplied',
-	'otUpdateError',
-	'joinDoc',
-	'reciveNewDoc',
-	'reciveNewFile',
-	'reciveNewFolder',
-	'removeEntity'
-]
-
-module.exports = WebsocketLoadBalancer =
-	rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub)
-	rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub)
-
-	emitToRoom: (room_id, message, payload...) ->
-		if !room_id?
-			logger.warn {message, payload}, "no room_id provided, ignoring emitToRoom"
-			return
-		data = JSON.stringify
-			room_id: room_id
-			message: message
-			payload: payload
-		logger.log {room_id, message, payload, length: data.length}, "emitting to room"
-
-		for rclientPub in @rclientPubList
-			ChannelManager.publish rclientPub, "editor-events", room_id, data
-
-	emitToAll: (message, payload...) ->
-		@emitToRoom "all", message, payload...
-
-	listenForEditorEvents: (io) ->
-		logger.log {rclients: @rclientPubList.length}, "publishing editor events"
-		logger.log {rclients: @rclientSubList.length}, "listening for editor events"
-		for rclientSub in @rclientSubList
-			rclientSub.subscribe "editor-events"
-			rclientSub.on "message", (channel, message) ->
-				EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0
-				WebsocketLoadBalancer._processEditorEvent io, channel, message
-		@handleRoomUpdates(@rclientSubList)
-
-	handleRoomUpdates: (rclientSubList) ->
-		roomEvents = RoomManager.eventSource()
-		roomEvents.on 'project-active', (project_id) ->
-			subscribePromises = for rclient in rclientSubList
-				ChannelManager.subscribe rclient, "editor-events", project_id
-			RoomManager.emitOnCompletion(subscribePromises, "project-subscribed-#{project_id}")
-		roomEvents.on 'project-empty', (project_id) ->
-			for rclient in rclientSubList
-				ChannelManager.unsubscribe rclient, "editor-events", project_id
-
-	_processEditorEvent: (io, channel, message) ->
-		SafeJsonParse.parse message, (error, message) ->
-			if error?
-				logger.error {err: error, channel}, "error parsing JSON"
-				return
-			if message.room_id == "all"
-				io.sockets.emit(message.message, message.payload...)
-			else if message.message is 'clientTracking.refresh' && message.room_id?
-				clientList = io.sockets.clients(message.room_id)
-				logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list"
-				for client in clientList
-					ConnectedUsersManager.refreshClient(message.room_id, client.publicId)
-			else if message.room_id?
-				if message._id? && Settings.checkEventOrder
-					status = EventLogger.checkEventOrder("editor-events", message._id, message)
-					if status is "duplicate"
-						return # skip duplicate events
-
-				is_restricted_message = message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST
-
-				# send messages only to unique clients (due to duplicate entries in io.sockets.clients)
-				clientList = io.sockets.clients(message.room_id)
-				.filter((client) ->
-					!(is_restricted_message && client.ol_context['is_restricted_user'])
-				)
-
-				# avoid unnecessary work if no clients are connected
-				return if clientList.length is 0
-				logger.log {
-					channel: channel,
-					message: message.message,
-					room_id: message.room_id,
-					message_id: message._id,
-					socketIoClients: (client.id for client in clientList)
-				}, "distributing event to clients"
-				seen = {}
-				for client in clientList
-							if !seen[client.id]
-								seen[client.id] = true
-								client.emit(message.message, message.payload...)
-			else if message.health_check?
-				logger.debug {message}, "got health check message in editor events channel"
-				HealthCheckManager.check channel, message.key

services/real-time/app/js/AuthorizationManager.js

@@ -0,0 +1,111 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let AuthorizationManager
+module.exports = AuthorizationManager = {
+  assertClientCanViewProject(client, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    return AuthorizationManager._assertClientHasPrivilegeLevel(
+      client,
+      ['readOnly', 'readAndWrite', 'owner'],
+      callback
+    )
+  },
+
+  assertClientCanEditProject(client, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    return AuthorizationManager._assertClientHasPrivilegeLevel(
+      client,
+      ['readAndWrite', 'owner'],
+      callback
+    )
+  },
+
+  _assertClientHasPrivilegeLevel(client, allowedLevels, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (Array.from(allowedLevels).includes(client.ol_context.privilege_level)) {
+      return callback(null)
+    } else {
+      return callback(new Error('not authorized'))
+    }
+  },
+
+  assertClientCanViewProjectAndDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    return AuthorizationManager.assertClientCanViewProject(client, function (
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return AuthorizationManager._assertClientCanAccessDoc(
+        client,
+        doc_id,
+        callback
+      )
+    })
+  },
+
+  assertClientCanEditProjectAndDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    return AuthorizationManager.assertClientCanEditProject(client, function (
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return AuthorizationManager._assertClientCanAccessDoc(
+        client,
+        doc_id,
+        callback
+      )
+    })
+  },
+
+  _assertClientCanAccessDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (client.ol_context[`doc:${doc_id}`] === 'allowed') {
+      return callback(null)
+    } else {
+      return callback(new Error('not authorized'))
+    }
+  },
+
+  addAccessToDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    client.ol_context[`doc:${doc_id}`] = 'allowed'
+    return callback(null)
+  },
+
+  removeAccessToDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    delete client.ol_context[`doc:${doc_id}`]
+    return callback(null)
+  }
+}

services/real-time/app/js/ChannelManager.js

@@ -0,0 +1,105 @@
+/* eslint-disable
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let ChannelManager
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const settings = require('settings-sharelatex')
+
+const ClientMap = new Map() // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise)
+
+// Manage redis pubsub subscriptions for individual projects and docs, ensuring
+// that we never subscribe to a channel multiple times. The socket.io side is
+// handled by RoomManager.
+
+module.exports = ChannelManager = {
+  getClientMapEntry(rclient) {
+    // return the per-client channel map if it exists, otherwise create and
+    // return an empty map for the client.
+    return (
+      ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient)
+    )
+  },
+
+  subscribe(rclient, baseChannel, id) {
+    const clientChannelMap = this.getClientMapEntry(rclient)
+    const channel = `${baseChannel}:${id}`
+    const actualSubscribe = function () {
+      // subscribe is happening in the foreground and it should reject
+      const p = rclient.subscribe(channel)
+      p.finally(function () {
+        if (clientChannelMap.get(channel) === subscribePromise) {
+          return clientChannelMap.delete(channel)
+        }
+      })
+        .then(function () {
+          logger.log({ channel }, 'subscribed to channel')
+          return metrics.inc(`subscribe.${baseChannel}`)
+        })
+        .catch(function (err) {
+          logger.error({ channel, err }, 'failed to subscribe to channel')
+          return metrics.inc(`subscribe.failed.${baseChannel}`)
+        })
+      return p
+    }
+
+    const pendingActions = clientChannelMap.get(channel) || Promise.resolve()
+    var subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe)
+    clientChannelMap.set(channel, subscribePromise)
+    logger.log({ channel }, 'planned to subscribe to channel')
+    return subscribePromise
+  },
+
+  unsubscribe(rclient, baseChannel, id) {
+    const clientChannelMap = this.getClientMapEntry(rclient)
+    const channel = `${baseChannel}:${id}`
+    const actualUnsubscribe = function () {
+      // unsubscribe is happening in the background, it should not reject
+      const p = rclient
+        .unsubscribe(channel)
+        .finally(function () {
+          if (clientChannelMap.get(channel) === unsubscribePromise) {
+            return clientChannelMap.delete(channel)
+          }
+        })
+        .then(function () {
+          logger.log({ channel }, 'unsubscribed from channel')
+          return metrics.inc(`unsubscribe.${baseChannel}`)
+        })
+        .catch(function (err) {
+          logger.error({ channel, err }, 'unsubscribed from channel')
+          return metrics.inc(`unsubscribe.failed.${baseChannel}`)
+        })
+      return p
+    }
+
+    const pendingActions = clientChannelMap.get(channel) || Promise.resolve()
+    var unsubscribePromise = pendingActions.then(
+      actualUnsubscribe,
+      actualUnsubscribe
+    )
+    clientChannelMap.set(channel, unsubscribePromise)
+    logger.log({ channel }, 'planned to unsubscribe from channel')
+    return unsubscribePromise
+  },
+
+  publish(rclient, baseChannel, id, data) {
+    let channel
+    metrics.summary(`redis.publish.${baseChannel}`, data.length)
+    if (id === 'all' || !settings.publishOnIndividualChannels) {
+      channel = baseChannel
+    } else {
+      channel = `${baseChannel}:${id}`
+    }
+    // we publish on a different client to the subscribe, so we can't
+    // check for the channel existing here
+    return rclient.publish(channel, data)
+  }
+}

services/real-time/app/js/ConnectedUsersManager.js

@@ -0,0 +1,194 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const async = require('async')
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(Settings.redis.realtime)
+const Keys = Settings.redis.realtime.key_schema
+
+const ONE_HOUR_IN_S = 60 * 60
+const ONE_DAY_IN_S = ONE_HOUR_IN_S * 24
+const FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4
+
+const USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4
+const REFRESH_TIMEOUT_IN_S = 10 // only show clients which have responded to a refresh request in the last 10 seconds
+
+module.exports = {
+  // Use the same method for when a user connects, and when a user sends a cursor
+  // update. This way we don't care if the connected_user key has expired when
+  // we receive a cursor update.
+  updateUserPosition(project_id, client_id, user, cursorData, callback) {
+    if (callback == null) {
+      callback = function (err) {}
+    }
+    logger.log({ project_id, client_id }, 'marking user as joined or connected')
+
+    const multi = rclient.multi()
+
+    multi.sadd(Keys.clientsInProject({ project_id }), client_id)
+    multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S)
+
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'last_updated_at',
+      Date.now()
+    )
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'user_id',
+      user._id
+    )
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'first_name',
+      user.first_name || ''
+    )
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'last_name',
+      user.last_name || ''
+    )
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'email',
+      user.email || ''
+    )
+
+    if (cursorData != null) {
+      multi.hset(
+        Keys.connectedUser({ project_id, client_id }),
+        'cursorData',
+        JSON.stringify(cursorData)
+      )
+    }
+    multi.expire(
+      Keys.connectedUser({ project_id, client_id }),
+      USER_TIMEOUT_IN_S
+    )
+
+    return multi.exec(function (err) {
+      if (err != null) {
+        logger.err(
+          { err, project_id, client_id },
+          'problem marking user as connected'
+        )
+      }
+      return callback(err)
+    })
+  },
+
+  refreshClient(project_id, client_id, callback) {
+    if (callback == null) {
+      callback = function (err) {}
+    }
+    logger.log({ project_id, client_id }, 'refreshing connected client')
+    const multi = rclient.multi()
+    multi.hset(
+      Keys.connectedUser({ project_id, client_id }),
+      'last_updated_at',
+      Date.now()
+    )
+    multi.expire(
+      Keys.connectedUser({ project_id, client_id }),
+      USER_TIMEOUT_IN_S
+    )
+    return multi.exec(function (err) {
+      if (err != null) {
+        logger.err(
+          { err, project_id, client_id },
+          'problem refreshing connected client'
+        )
+      }
+      return callback(err)
+    })
+  },
+
+  markUserAsDisconnected(project_id, client_id, callback) {
+    logger.log({ project_id, client_id }, 'marking user as disconnected')
+    const multi = rclient.multi()
+    multi.srem(Keys.clientsInProject({ project_id }), client_id)
+    multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S)
+    multi.del(Keys.connectedUser({ project_id, client_id }))
+    return multi.exec(callback)
+  },
+
+  _getConnectedUser(project_id, client_id, callback) {
+    return rclient.hgetall(
+      Keys.connectedUser({ project_id, client_id }),
+      function (err, result) {
+        if (
+          result == null ||
+          Object.keys(result).length === 0 ||
+          !result.user_id
+        ) {
+          result = {
+            connected: false,
+            client_id
+          }
+        } else {
+          result.connected = true
+          result.client_id = client_id
+          result.client_age =
+            (Date.now() - parseInt(result.last_updated_at, 10)) / 1000
+          if (result.cursorData != null) {
+            try {
+              result.cursorData = JSON.parse(result.cursorData)
+            } catch (e) {
+              logger.error(
+                {
+                  err: e,
+                  project_id,
+                  client_id,
+                  cursorData: result.cursorData
+                },
+                'error parsing cursorData JSON'
+              )
+              return callback(e)
+            }
+          }
+        }
+        return callback(err, result)
+      }
+    )
+  },
+
+  getConnectedUsers(project_id, callback) {
+    const self = this
+    return rclient.smembers(Keys.clientsInProject({ project_id }), function (
+      err,
+      results
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      const jobs = results.map((client_id) => (cb) =>
+        self._getConnectedUser(project_id, client_id, cb)
+      )
+      return async.series(jobs, function (err, users) {
+        if (users == null) {
+          users = []
+        }
+        if (err != null) {
+          return callback(err)
+        }
+        users = users.filter(
+          (user) =>
+            (user != null ? user.connected : undefined) &&
+            (user != null ? user.client_age : undefined) < REFRESH_TIMEOUT_IN_S
+        )
+        return callback(null, users)
+      })
+    })
+  }
+}

services/real-time/app/js/DocumentUpdaterController.js

@@ -0,0 +1,208 @@
+/* eslint-disable
+    camelcase,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocumentUpdaterController
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+const RedisClientManager = require('./RedisClientManager')
+const SafeJsonParse = require('./SafeJsonParse')
+const EventLogger = require('./EventLogger')
+const HealthCheckManager = require('./HealthCheckManager')
+const RoomManager = require('./RoomManager')
+const ChannelManager = require('./ChannelManager')
+const metrics = require('metrics-sharelatex')
+
+const MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 // 1Mb
+
+module.exports = DocumentUpdaterController = {
+  // DocumentUpdaterController is responsible for updates that come via Redis
+  // Pub/Sub from the document updater.
+  rclientList: RedisClientManager.createClientList(settings.redis.pubsub),
+
+  listenForUpdatesFromDocumentUpdater(io) {
+    let i, rclient
+    logger.log(
+      { rclients: this.rclientList.length },
+      'listening for applied-ops events'
+    )
+    for (i = 0; i < this.rclientList.length; i++) {
+      rclient = this.rclientList[i]
+      rclient.subscribe('applied-ops')
+      rclient.on('message', function (channel, message) {
+        metrics.inc('rclient', 0.001) // global event rate metric
+        if (settings.debugEvents > 0) {
+          EventLogger.debugEvent(channel, message)
+        }
+        return DocumentUpdaterController._processMessageFromDocumentUpdater(
+          io,
+          channel,
+          message
+        )
+      })
+    }
+    // create metrics for each redis instance only when we have multiple redis clients
+    if (this.rclientList.length > 1) {
+      for (i = 0; i < this.rclientList.length; i++) {
+        rclient = this.rclientList[i]
+        ;((
+          i // per client event rate metric
+        ) => rclient.on('message', () => metrics.inc(`rclient-${i}`, 0.001)))(i)
+      }
+    }
+    return this.handleRoomUpdates(this.rclientList)
+  },
+
+  handleRoomUpdates(rclientSubList) {
+    const roomEvents = RoomManager.eventSource()
+    roomEvents.on('doc-active', function (doc_id) {
+      const subscribePromises = Array.from(rclientSubList).map((rclient) =>
+        ChannelManager.subscribe(rclient, 'applied-ops', doc_id)
+      )
+      return RoomManager.emitOnCompletion(
+        subscribePromises,
+        `doc-subscribed-${doc_id}`
+      )
+    })
+    return roomEvents.on('doc-empty', (doc_id) =>
+      Array.from(rclientSubList).map((rclient) =>
+        ChannelManager.unsubscribe(rclient, 'applied-ops', doc_id)
+      )
+    )
+  },
+
+  _processMessageFromDocumentUpdater(io, channel, message) {
+    return SafeJsonParse.parse(message, function (error, message) {
+      if (error != null) {
+        logger.error({ err: error, channel }, 'error parsing JSON')
+        return
+      }
+      if (message.op != null) {
+        if (message._id != null && settings.checkEventOrder) {
+          const status = EventLogger.checkEventOrder(
+            'applied-ops',
+            message._id,
+            message
+          )
+          if (status === 'duplicate') {
+            return // skip duplicate events
+          }
+        }
+        return DocumentUpdaterController._applyUpdateFromDocumentUpdater(
+          io,
+          message.doc_id,
+          message.op
+        )
+      } else if (message.error != null) {
+        return DocumentUpdaterController._processErrorFromDocumentUpdater(
+          io,
+          message.doc_id,
+          message.error,
+          message
+        )
+      } else if (message.health_check != null) {
+        logger.debug(
+          { message },
+          'got health check message in applied ops channel'
+        )
+        return HealthCheckManager.check(channel, message.key)
+      }
+    })
+  },
+
+  _applyUpdateFromDocumentUpdater(io, doc_id, update) {
+    let client
+    const clientList = io.sockets.clients(doc_id)
+    // avoid unnecessary work if no clients are connected
+    if (clientList.length === 0) {
+      return
+    }
+    // send updates to clients
+    logger.log(
+      {
+        doc_id,
+        version: update.v,
+        source: update.meta != null ? update.meta.source : undefined,
+        socketIoClients: (() => {
+          const result = []
+          for (client of Array.from(clientList)) {
+            result.push(client.id)
+          }
+          return result
+        })()
+      },
+      'distributing updates to clients'
+    )
+    const seen = {}
+    // send messages only to unique clients (due to duplicate entries in io.sockets.clients)
+    for (client of Array.from(clientList)) {
+      if (!seen[client.id]) {
+        seen[client.id] = true
+        if (client.publicId === update.meta.source) {
+          logger.log(
+            {
+              doc_id,
+              version: update.v,
+              source: update.meta != null ? update.meta.source : undefined
+            },
+            'distributing update to sender'
+          )
+          client.emit('otUpdateApplied', { v: update.v, doc: update.doc })
+        } else if (!update.dup) {
+          // Duplicate ops should just be sent back to sending client for acknowledgement
+          logger.log(
+            {
+              doc_id,
+              version: update.v,
+              source: update.meta != null ? update.meta.source : undefined,
+              client_id: client.id
+            },
+            'distributing update to collaborator'
+          )
+          client.emit('otUpdateApplied', update)
+        }
+      }
+    }
+    if (Object.keys(seen).length < clientList.length) {
+      metrics.inc('socket-io.duplicate-clients', 0.1)
+      return logger.log(
+        {
+          doc_id,
+          socketIoClients: (() => {
+            const result1 = []
+            for (client of Array.from(clientList)) {
+              result1.push(client.id)
+            }
+            return result1
+          })()
+        },
+        'discarded duplicate clients'
+      )
+    }
+  },
+
+  _processErrorFromDocumentUpdater(io, doc_id, error, message) {
+    return (() => {
+      const result = []
+      for (const client of Array.from(io.sockets.clients(doc_id))) {
+        logger.warn(
+          { err: error, doc_id, client_id: client.id },
+          'error from document updater, disconnecting client'
+        )
+        client.emit('otUpdateError', error, message)
+        result.push(client.disconnect())
+      }
+      return result
+    })()
+  }
+}

services/real-time/app/js/DocumentUpdaterManager.js

@@ -0,0 +1,169 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DocumentUpdaterManager
+const request = require('request')
+const _ = require('underscore')
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+const rclient = require('redis-sharelatex').createClient(
+  settings.redis.documentupdater
+)
+const Keys = settings.redis.documentupdater.key_schema
+
+module.exports = DocumentUpdaterManager = {
+  getDocument(project_id, doc_id, fromVersion, callback) {
+    if (callback == null) {
+      callback = function (error, exists, doclines, version) {}
+    }
+    const timer = new metrics.Timer('get-document')
+    const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}`
+    logger.log(
+      { project_id, doc_id, fromVersion },
+      'getting doc from document updater'
+    )
+    return request.get(url, function (err, res, body) {
+      timer.done()
+      if (err != null) {
+        logger.error(
+          { err, url, project_id, doc_id },
+          'error getting doc from doc updater'
+        )
+        return callback(err)
+      }
+      if (res.statusCode >= 200 && res.statusCode < 300) {
+        logger.log(
+          { project_id, doc_id },
+          'got doc from document document updater'
+        )
+        try {
+          body = JSON.parse(body)
+        } catch (error) {
+          return callback(error)
+        }
+        return callback(
+          null,
+          body != null ? body.lines : undefined,
+          body != null ? body.version : undefined,
+          body != null ? body.ranges : undefined,
+          body != null ? body.ops : undefined
+        )
+      } else if ([404, 422].includes(res.statusCode)) {
+        err = new Error('doc updater could not load requested ops')
+        err.statusCode = res.statusCode
+        logger.warn(
+          { err, project_id, doc_id, url, fromVersion },
+          'doc updater could not load requested ops'
+        )
+        return callback(err)
+      } else {
+        err = new Error(
+          `doc updater returned a non-success status code: ${res.statusCode}`
+        )
+        err.statusCode = res.statusCode
+        logger.error(
+          { err, project_id, doc_id, url },
+          `doc updater returned a non-success status code: ${res.statusCode}`
+        )
+        return callback(err)
+      }
+    })
+  },
+
+  flushProjectToMongoAndDelete(project_id, callback) {
+    // this method is called when the last connected user leaves the project
+    if (callback == null) {
+      callback = function () {}
+    }
+    logger.log({ project_id }, 'deleting project from document updater')
+    const timer = new metrics.Timer('delete.mongo.project')
+    // flush the project in the background when all users have left
+    const url =
+      `${settings.apis.documentupdater.url}/project/${project_id}?background=true` +
+      (settings.shutDownInProgress ? '&shutdown=true' : '')
+    return request.del(url, function (err, res, body) {
+      timer.done()
+      if (err != null) {
+        logger.error(
+          { err, project_id },
+          'error deleting project from document updater'
+        )
+        return callback(err)
+      } else if (res.statusCode >= 200 && res.statusCode < 300) {
+        logger.log({ project_id }, 'deleted project from document updater')
+        return callback(null)
+      } else {
+        err = new Error(
+          `document updater returned a failure status code: ${res.statusCode}`
+        )
+        err.statusCode = res.statusCode
+        logger.error(
+          { err, project_id },
+          `document updater returned failure status code: ${res.statusCode}`
+        )
+        return callback(err)
+      }
+    })
+  },
+
+  queueChange(project_id, doc_id, change, callback) {
+    let error
+    if (callback == null) {
+      callback = function () {}
+    }
+    const allowedKeys = [
+      'doc',
+      'op',
+      'v',
+      'dupIfSource',
+      'meta',
+      'lastV',
+      'hash'
+    ]
+    change = _.pick(change, allowedKeys)
+    const jsonChange = JSON.stringify(change)
+    if (jsonChange.indexOf('\u0000') !== -1) {
+      // memory corruption check
+      error = new Error('null bytes found in op')
+      logger.error(
+        { err: error, project_id, doc_id, jsonChange },
+        error.message
+      )
+      return callback(error)
+    }
+
+    const updateSize = jsonChange.length
+    if (updateSize > settings.maxUpdateSize) {
+      error = new Error('update is too large')
+      error.updateSize = updateSize
+      return callback(error)
+    }
+
+    // record metric for each update added to queue
+    metrics.summary('redis.pendingUpdates', updateSize, { status: 'push' })
+
+    const doc_key = `${project_id}:${doc_id}`
+    // Push onto pendingUpdates for doc_id first, because once the doc updater
+    // gets an entry on pending-updates-list, it starts processing.
+    return rclient.rpush(Keys.pendingUpdates({ doc_id }), jsonChange, function (
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      return rclient.rpush('pending-updates-list', doc_key, callback)
+    })
+  }
+}

services/real-time/app/js/DrainManager.js

@@ -0,0 +1,63 @@
+/* eslint-disable
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let DrainManager
+const logger = require('logger-sharelatex')
+
+module.exports = DrainManager = {
+  startDrainTimeWindow(io, minsToDrain) {
+    const drainPerMin = io.sockets.clients().length / minsToDrain
+    return DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4))
+  }, // enforce minimum drain rate
+
+  startDrain(io, rate) {
+    // Clear out any old interval
+    let pollingInterval
+    clearInterval(this.interval)
+    logger.log({ rate }, 'starting drain')
+    if (rate === 0) {
+      return
+    } else if (rate < 1) {
+      // allow lower drain rates
+      // e.g. rate=0.1 will drain one client every 10 seconds
+      pollingInterval = 1000 / rate
+      rate = 1
+    } else {
+      pollingInterval = 1000
+    }
+    return (this.interval = setInterval(() => {
+      return this.reconnectNClients(io, rate)
+    }, pollingInterval))
+  },
+
+  RECONNECTED_CLIENTS: {},
+  reconnectNClients(io, N) {
+    let drainedCount = 0
+    for (const client of Array.from(io.sockets.clients())) {
+      if (!this.RECONNECTED_CLIENTS[client.id]) {
+        this.RECONNECTED_CLIENTS[client.id] = true
+        logger.log(
+          { client_id: client.id },
+          'Asking client to reconnect gracefully'
+        )
+        client.emit('reconnectGracefully')
+        drainedCount++
+      }
+      const haveDrainedNClients = drainedCount === N
+      if (haveDrainedNClients) {
+        break
+      }
+    }
+    if (drainedCount < N) {
+      return logger.log('All clients have been told to reconnectGracefully')
+    }
+  }
+}

services/real-time/app/js/Errors.js

@@ -0,0 +1,17 @@
+/* eslint-disable
+    no-proto,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+let Errors
+var CodedError = function (message, code) {
+  const error = new Error(message)
+  error.name = 'CodedError'
+  error.code = code
+  error.__proto__ = CodedError.prototype
+  return error
+}
+CodedError.prototype.__proto__ = Error.prototype
+
+module.exports = Errors = { CodedError }

services/real-time/app/js/EventLogger.js

@@ -0,0 +1,100 @@
+/* eslint-disable
+    camelcase,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let EventLogger
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const settings = require('settings-sharelatex')
+
+// keep track of message counters to detect duplicate and out of order events
+// messsage ids have the format "UNIQUEHOSTKEY-COUNTER"
+
+const EVENT_LOG_COUNTER = {}
+const EVENT_LOG_TIMESTAMP = {}
+let EVENT_LAST_CLEAN_TIMESTAMP = 0
+
+// counter for debug logs
+let COUNTER = 0
+
+module.exports = EventLogger = {
+  MAX_STALE_TIME_IN_MS: 3600 * 1000,
+
+  debugEvent(channel, message) {
+    if (settings.debugEvents > 0) {
+      logger.log({ channel, message, counter: COUNTER++ }, 'logging event')
+      return settings.debugEvents--
+    }
+  },
+
+  checkEventOrder(channel, message_id, message) {
+    let result
+    if (typeof message_id !== 'string') {
+      return
+    }
+    if (!(result = message_id.match(/^(.*)-(\d+)$/))) {
+      return
+    }
+    const key = result[1]
+    const count = parseInt(result[2], 0)
+    if (!(count >= 0)) {
+      // ignore checks if counter is not present
+      return
+    }
+    // store the last count in a hash for each host
+    const previous = EventLogger._storeEventCount(key, count)
+    if (previous == null || count === previous + 1) {
+      metrics.inc(`event.${channel}.valid`, 0.001) // downsample high rate docupdater events
+      return // order is ok
+    }
+    if (count === previous) {
+      metrics.inc(`event.${channel}.duplicate`)
+      logger.warn({ channel, message_id }, 'duplicate event')
+      return 'duplicate'
+    } else {
+      metrics.inc(`event.${channel}.out-of-order`)
+      logger.warn(
+        { channel, message_id, key, previous, count },
+        'out of order event'
+      )
+      return 'out-of-order'
+    }
+  },
+
+  _storeEventCount(key, count) {
+    const previous = EVENT_LOG_COUNTER[key]
+    const now = Date.now()
+    EVENT_LOG_COUNTER[key] = count
+    EVENT_LOG_TIMESTAMP[key] = now
+    // periodically remove old counts
+    if (now - EVENT_LAST_CLEAN_TIMESTAMP > EventLogger.MAX_STALE_TIME_IN_MS) {
+      EventLogger._cleanEventStream(now)
+      EVENT_LAST_CLEAN_TIMESTAMP = now
+    }
+    return previous
+  },
+
+  _cleanEventStream(now) {
+    return (() => {
+      const result = []
+      for (const key in EVENT_LOG_TIMESTAMP) {
+        const timestamp = EVENT_LOG_TIMESTAMP[key]
+        if (now - timestamp > EventLogger.MAX_STALE_TIME_IN_MS) {
+          delete EVENT_LOG_COUNTER[key]
+          result.push(delete EVENT_LOG_TIMESTAMP[key])
+        } else {
+          result.push(undefined)
+        }
+      }
+      return result
+    })()
+  }
+}

services/real-time/app/js/HealthCheckManager.js

@@ -0,0 +1,93 @@
+/* eslint-disable
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HealthCheckManager
+const metrics = require('metrics-sharelatex')
+const logger = require('logger-sharelatex')
+
+const os = require('os')
+const HOST = os.hostname()
+const PID = process.pid
+let COUNT = 0
+
+const CHANNEL_MANAGER = {} // hash of event checkers by channel name
+const CHANNEL_ERROR = {} // error status by channel name
+
+module.exports = HealthCheckManager = class HealthCheckManager {
+  // create an instance of this class which checks that an event with a unique
+  // id is received only once within a timeout
+  constructor(channel, timeout) {
+    // unique event string
+    this.channel = channel
+    if (timeout == null) {
+      timeout = 1000
+    }
+    this.id = `host=${HOST}:pid=${PID}:count=${COUNT++}`
+    // count of number of times the event is received
+    this.count = 0
+    // after a timeout check the status of the count
+    this.handler = setTimeout(() => {
+      return this.setStatus()
+    }, timeout)
+    // use a timer to record the latency of the channel
+    this.timer = new metrics.Timer(`event.${this.channel}.latency`)
+    // keep a record of these objects to dispatch on
+    CHANNEL_MANAGER[this.channel] = this
+  }
+
+  processEvent(id) {
+    // if this is our event record it
+    if (id === this.id) {
+      this.count++
+      if (this.timer != null) {
+        this.timer.done()
+      }
+      return (this.timer = null) // only time the latency of the first event
+    }
+  }
+
+  setStatus() {
+    // if we saw the event anything other than a single time that is an error
+    if (this.count !== 1) {
+      logger.err(
+        { channel: this.channel, count: this.count, id: this.id },
+        'redis channel health check error'
+      )
+    }
+    const error = this.count !== 1
+    return (CHANNEL_ERROR[this.channel] = error)
+  }
+
+  // class methods
+  static check(channel, id) {
+    // dispatch event to manager for channel
+    return CHANNEL_MANAGER[channel] != null
+      ? CHANNEL_MANAGER[channel].processEvent(id)
+      : undefined
+  }
+
+  static status() {
+    // return status of all channels for logging
+    return CHANNEL_ERROR
+  }
+
+  static isFailing() {
+    // check if any channel status is bad
+    for (const channel in CHANNEL_ERROR) {
+      const error = CHANNEL_ERROR[channel]
+      if (error === true) {
+        return true
+      }
+    }
+    return false
+  }
+}

services/real-time/app/js/HttpApiController.js

@@ -0,0 +1,62 @@
+/* eslint-disable
+    camelcase,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HttpApiController
+const WebsocketLoadBalancer = require('./WebsocketLoadBalancer')
+const DrainManager = require('./DrainManager')
+const logger = require('logger-sharelatex')
+
+module.exports = HttpApiController = {
+  sendMessage(req, res, next) {
+    logger.log({ message: req.params.message }, 'sending message')
+    if (Array.isArray(req.body)) {
+      for (const payload of Array.from(req.body)) {
+        WebsocketLoadBalancer.emitToRoom(
+          req.params.project_id,
+          req.params.message,
+          payload
+        )
+      }
+    } else {
+      WebsocketLoadBalancer.emitToRoom(
+        req.params.project_id,
+        req.params.message,
+        req.body
+      )
+    }
+    return res.send(204)
+  }, // No content
+
+  startDrain(req, res, next) {
+    const io = req.app.get('io')
+    let rate = req.query.rate || '4'
+    rate = parseFloat(rate) || 0
+    logger.log({ rate }, 'setting client drain rate')
+    DrainManager.startDrain(io, rate)
+    return res.send(204)
+  },
+
+  disconnectClient(req, res, next) {
+    const io = req.app.get('io')
+    const { client_id } = req.params
+    const client = io.sockets.sockets[client_id]
+
+    if (!client) {
+      logger.info({ client_id }, 'api: client already disconnected')
+      res.sendStatus(404)
+      return
+    }
+    logger.warn({ client_id }, 'api: requesting client disconnect')
+    client.on('disconnect', () => res.sendStatus(204))
+    return client.disconnect()
+  }
+}

services/real-time/app/js/HttpController.js

@@ -0,0 +1,87 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let HttpController
+const async = require('async')
+
+module.exports = HttpController = {
+  // The code in this controller is hard to unit test because of a lot of
+  // dependencies on internal socket.io methods. It is not critical to the running
+  // of ShareLaTeX, and is only used for getting stats about connected clients,
+  // and for checking internal state in acceptance tests. The acceptances tests
+  // should provide appropriate coverage.
+  _getConnectedClientView(ioClient, callback) {
+    if (callback == null) {
+      callback = function (error, client) {}
+    }
+    const client_id = ioClient.id
+    const {
+      project_id,
+      user_id,
+      first_name,
+      last_name,
+      email,
+      connected_time
+    } = ioClient.ol_context
+    const client = {
+      client_id,
+      project_id,
+      user_id,
+      first_name,
+      last_name,
+      email,
+      connected_time
+    }
+    client.rooms = []
+    for (const name in ioClient.manager.roomClients[client_id]) {
+      const joined = ioClient.manager.roomClients[client_id][name]
+      if (joined && name !== '') {
+        client.rooms.push(name.replace(/^\//, '')) // Remove leading /
+      }
+    }
+    return callback(null, client)
+  },
+
+  getConnectedClients(req, res, next) {
+    const io = req.app.get('io')
+    const ioClients = io.sockets.clients()
+    return async.map(
+      ioClients,
+      HttpController._getConnectedClientView,
+      function (error, clients) {
+        if (error != null) {
+          return next(error)
+        }
+        return res.json(clients)
+      }
+    )
+  },
+
+  getConnectedClient(req, res, next) {
+    const { client_id } = req.params
+    const io = req.app.get('io')
+    const ioClient = io.sockets.sockets[client_id]
+    if (!ioClient) {
+      res.sendStatus(404)
+      return
+    }
+    return HttpController._getConnectedClientView(ioClient, function (
+      error,
+      client
+    ) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.json(client)
+    })
+  }
+}

services/real-time/app/js/RedisClientManager.js

@@ -0,0 +1,40 @@
+/* eslint-disable
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RedisClientManager
+const redis = require('redis-sharelatex')
+const logger = require('logger-sharelatex')
+
+module.exports = RedisClientManager = {
+  createClientList(...configs) {
+    // create a dynamic list of redis clients, excluding any configurations which are not defined
+    const clientList = (() => {
+      const result = []
+      for (const x of Array.from(configs)) {
+        if (x != null) {
+          const redisType =
+            x.cluster != null
+              ? 'cluster'
+              : x.sentinels != null
+              ? 'sentinel'
+              : x.host != null
+              ? 'single'
+              : 'unknown'
+          logger.log({ redis: redisType }, 'creating redis client')
+          result.push(redis.createClient(x))
+        }
+      }
+      return result
+    })()
+    return clientList
+  }
+}

services/real-time/app/js/RoomManager.js

@@ -0,0 +1,189 @@
+/* eslint-disable
+    camelcase,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let RoomManager
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const { EventEmitter } = require('events')
+
+const IdMap = new Map() // keep track of whether ids are from projects or docs
+const RoomEvents = new EventEmitter() // emits {project,doc}-active and {project,doc}-empty events
+
+// Manage socket.io rooms for individual projects and docs
+//
+// The first time someone joins a project or doc we emit a 'project-active' or
+// 'doc-active' event.
+//
+// When the last person leaves a project or doc, we emit 'project-empty' or
+// 'doc-empty' event.
+//
+// The pubsub side is handled by ChannelManager
+
+module.exports = RoomManager = {
+  joinProject(client, project_id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    return this.joinEntity(client, 'project', project_id, callback)
+  },
+
+  joinDoc(client, doc_id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    return this.joinEntity(client, 'doc', doc_id, callback)
+  },
+
+  leaveDoc(client, doc_id) {
+    return this.leaveEntity(client, 'doc', doc_id)
+  },
+
+  leaveProjectAndDocs(client) {
+    // what rooms is this client in? we need to leave them all. socket.io
+    // will cause us to leave the rooms, so we only need to manage our
+    // channel subscriptions... but it will be safer if we leave them
+    // explicitly, and then socket.io will just regard this as a client that
+    // has not joined any rooms and do a final disconnection.
+    const roomsToLeave = this._roomsClientIsIn(client)
+    logger.log({ client: client.id, roomsToLeave }, 'client leaving project')
+    return (() => {
+      const result = []
+      for (const id of Array.from(roomsToLeave)) {
+        const entity = IdMap.get(id)
+        result.push(this.leaveEntity(client, entity, id))
+      }
+      return result
+    })()
+  },
+
+  emitOnCompletion(promiseList, eventName) {
+    return Promise.all(promiseList)
+      .then(() => RoomEvents.emit(eventName))
+      .catch((err) => RoomEvents.emit(eventName, err))
+  },
+
+  eventSource() {
+    return RoomEvents
+  },
+
+  joinEntity(client, entity, id, callback) {
+    const beforeCount = this._clientsInRoom(client, id)
+    // client joins room immediately but joinDoc request does not complete
+    // until room is subscribed
+    client.join(id)
+    // is this a new room? if so, subscribe
+    if (beforeCount === 0) {
+      logger.log({ entity, id }, 'room is now active')
+      RoomEvents.once(`${entity}-subscribed-${id}`, function (err) {
+        // only allow the client to join when all the relevant channels have subscribed
+        logger.log(
+          { client: client.id, entity, id, beforeCount },
+          'client joined new room and subscribed to channel'
+        )
+        return callback(err)
+      })
+      RoomEvents.emit(`${entity}-active`, id)
+      IdMap.set(id, entity)
+      // keep track of the number of listeners
+      return metrics.gauge('room-listeners', RoomEvents.eventNames().length)
+    } else {
+      logger.log(
+        { client: client.id, entity, id, beforeCount },
+        'client joined existing room'
+      )
+      client.join(id)
+      return callback()
+    }
+  },
+
+  leaveEntity(client, entity, id) {
+    // Ignore any requests to leave when the client is not actually in the
+    // room. This can happen if the client sends spurious leaveDoc requests
+    // for old docs after a reconnection.
+    // This can now happen all the time, as we skip the join for clients that
+    //  disconnect before joinProject/joinDoc completed.
+    if (!this._clientAlreadyInRoom(client, id)) {
+      logger.log(
+        { client: client.id, entity, id },
+        'ignoring request from client to leave room it is not in'
+      )
+      return
+    }
+    client.leave(id)
+    const afterCount = this._clientsInRoom(client, id)
+    logger.log(
+      { client: client.id, entity, id, afterCount },
+      'client left room'
+    )
+    // is the room now empty? if so, unsubscribe
+    if (entity == null) {
+      logger.error({ entity: id }, 'unknown entity when leaving with id')
+      return
+    }
+    if (afterCount === 0) {
+      logger.log({ entity, id }, 'room is now empty')
+      RoomEvents.emit(`${entity}-empty`, id)
+      IdMap.delete(id)
+      return metrics.gauge('room-listeners', RoomEvents.eventNames().length)
+    }
+  },
+
+  // internal functions below, these access socket.io rooms data directly and
+  // will need updating for socket.io v2
+
+  _clientsInRoom(client, room) {
+    const nsp = client.namespace.name
+    const name = nsp + '/' + room
+    return (
+      __guard__(
+        client.manager != null ? client.manager.rooms : undefined,
+        (x) => x[name]
+      ) || []
+    ).length
+  },
+
+  _roomsClientIsIn(client) {
+    const roomList = (() => {
+      const result = []
+      for (const fullRoomPath in client.manager.roomClients != null
+        ? client.manager.roomClients[client.id]
+        : undefined) {
+        // strip socket.io prefix from room to get original id
+        if (fullRoomPath !== '') {
+          const [prefix, room] = Array.from(fullRoomPath.split('/', 2))
+          result.push(room)
+        }
+      }
+      return result
+    })()
+    return roomList
+  },
+
+  _clientAlreadyInRoom(client, room) {
+    const nsp = client.namespace.name
+    const name = nsp + '/' + room
+    return __guard__(
+      client.manager.roomClients != null
+        ? client.manager.roomClients[client.id]
+        : undefined,
+      (x) => x[name]
+    )
+  }
+}
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/real-time/app/js/Router.js

@@ -0,0 +1,402 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    standard/no-callback-literal,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let Router
+const metrics = require('metrics-sharelatex')
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+const WebsocketController = require('./WebsocketController')
+const HttpController = require('./HttpController')
+const HttpApiController = require('./HttpApiController')
+const bodyParser = require('body-parser')
+const base64id = require('base64id')
+
+const basicAuth = require('basic-auth-connect')
+const httpAuth = basicAuth(function (user, pass) {
+  const isValid =
+    user === settings.internal.realTime.user &&
+    pass === settings.internal.realTime.pass
+  if (!isValid) {
+    logger.err({ user, pass }, 'invalid login details')
+  }
+  return isValid
+})
+
+module.exports = Router = {
+  _handleError(callback, error, client, method, attrs) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (attrs == null) {
+      attrs = {}
+    }
+    for (const key of ['project_id', 'doc_id', 'user_id']) {
+      attrs[key] = client.ol_context[key]
+    }
+    attrs.client_id = client.id
+    attrs.err = error
+    if (error.name === 'CodedError') {
+      logger.warn(attrs, error.message, { code: error.code })
+      return callback({ message: error.message, code: error.code })
+    }
+    if (error.message === 'unexpected arguments') {
+      // the payload might be very large, put it on level info
+      logger.log(attrs, 'unexpected arguments')
+      metrics.inc('unexpected-arguments', 1, { status: method })
+      return callback({ message: error.message })
+    }
+    if (
+      [
+        'not authorized',
+        'doc updater could not load requested ops',
+        'no project_id found on client'
+      ].includes(error.message)
+    ) {
+      logger.warn(attrs, error.message)
+      return callback({ message: error.message })
+    } else {
+      logger.error(attrs, `server side error in ${method}`)
+      // Don't return raw error to prevent leaking server side info
+      return callback({ message: 'Something went wrong in real-time service' })
+    }
+  },
+
+  _handleInvalidArguments(client, method, args) {
+    const error = new Error('unexpected arguments')
+    let callback = args[args.length - 1]
+    if (typeof callback !== 'function') {
+      callback = function () {}
+    }
+    const attrs = { arguments: args }
+    return Router._handleError(callback, error, client, method, attrs)
+  },
+
+  configure(app, io, session) {
+    app.set('io', io)
+    app.get('/clients', HttpController.getConnectedClients)
+    app.get('/clients/:client_id', HttpController.getConnectedClient)
+
+    app.post(
+      '/project/:project_id/message/:message',
+      httpAuth,
+      bodyParser.json({ limit: '5mb' }),
+      HttpApiController.sendMessage
+    )
+
+    app.post('/drain', httpAuth, HttpApiController.startDrain)
+    app.post(
+      '/client/:client_id/disconnect',
+      httpAuth,
+      HttpApiController.disconnectClient
+    )
+
+    return session.on('connection', function (error, client, session) {
+      // init client context, we may access it in Router._handleError before
+      //  setting any values
+      let user
+      client.ol_context = {}
+
+      if (client != null) {
+        client.on('error', function (err) {
+          logger.err({ clientErr: err }, 'socket.io client error')
+          if (client.connected) {
+            client.emit('reconnectGracefully')
+            return client.disconnect()
+          }
+        })
+      }
+
+      if (settings.shutDownInProgress) {
+        client.emit('connectionRejected', { message: 'retry' })
+        client.disconnect()
+        return
+      }
+
+      if (
+        client != null &&
+        __guard__(error != null ? error.message : undefined, (x) =>
+          x.match(/could not look up session by key/)
+        )
+      ) {
+        logger.warn(
+          { err: error, client: client != null, session: session != null },
+          'invalid session'
+        )
+        // tell the client to reauthenticate if it has an invalid session key
+        client.emit('connectionRejected', { message: 'invalid session' })
+        client.disconnect()
+        return
+      }
+
+      if (error != null) {
+        logger.err(
+          { err: error, client: client != null, session: session != null },
+          'error when client connected'
+        )
+        if (client != null) {
+          client.emit('connectionRejected', { message: 'error' })
+        }
+        if (client != null) {
+          client.disconnect()
+        }
+        return
+      }
+
+      // send positive confirmation that the client has a valid connection
+      client.publicId = 'P.' + base64id.generateId()
+      client.emit('connectionAccepted', null, client.publicId)
+
+      metrics.inc('socket-io.connection')
+      metrics.gauge(
+        'socket-io.clients',
+        __guard__(io.sockets.clients(), (x1) => x1.length)
+      )
+
+      logger.log({ session, client_id: client.id }, 'client connected')
+
+      if (
+        __guard__(
+          session != null ? session.passport : undefined,
+          (x2) => x2.user
+        ) != null
+      ) {
+        ;({ user } = session.passport)
+      } else if ((session != null ? session.user : undefined) != null) {
+        ;({ user } = session)
+      } else {
+        user = { _id: 'anonymous-user' }
+      }
+
+      client.on('joinProject', function (data, callback) {
+        if (data == null) {
+          data = {}
+        }
+        if (typeof callback !== 'function') {
+          return Router._handleInvalidArguments(
+            client,
+            'joinProject',
+            arguments
+          )
+        }
+
+        if (data.anonymousAccessToken) {
+          user.anonymousAccessToken = data.anonymousAccessToken
+        }
+        return WebsocketController.joinProject(
+          client,
+          user,
+          data.project_id,
+          function (err, ...args) {
+            if (err != null) {
+              return Router._handleError(callback, err, client, 'joinProject', {
+                project_id: data.project_id,
+                user_id: user != null ? user.id : undefined
+              })
+            } else {
+              return callback(null, ...Array.from(args))
+            }
+          }
+        )
+      })
+
+      client.on('disconnect', function () {
+        metrics.inc('socket-io.disconnect')
+        metrics.gauge(
+          'socket-io.clients',
+          __guard__(io.sockets.clients(), (x3) => x3.length) - 1
+        )
+
+        return WebsocketController.leaveProject(io, client, function (err) {
+          if (err != null) {
+            return Router._handleError(
+              function () {},
+              err,
+              client,
+              'leaveProject'
+            )
+          }
+        })
+      })
+
+      // Variadic. The possible arguments:
+      // doc_id, callback
+      // doc_id, fromVersion, callback
+      // doc_id, options, callback
+      // doc_id, fromVersion, options, callback
+      client.on('joinDoc', function (doc_id, fromVersion, options, callback) {
+        if (typeof fromVersion === 'function' && !options) {
+          callback = fromVersion
+          fromVersion = -1
+          options = {}
+        } else if (
+          typeof fromVersion === 'number' &&
+          typeof options === 'function'
+        ) {
+          callback = options
+          options = {}
+        } else if (
+          typeof fromVersion === 'object' &&
+          typeof options === 'function'
+        ) {
+          callback = options
+          options = fromVersion
+          fromVersion = -1
+        } else if (
+          typeof fromVersion === 'number' &&
+          typeof options === 'object' &&
+          typeof callback === 'function'
+        ) {
+          // Called with 4 args, things are as expected
+        } else {
+          return Router._handleInvalidArguments(client, 'joinDoc', arguments)
+        }
+
+        return WebsocketController.joinDoc(
+          client,
+          doc_id,
+          fromVersion,
+          options,
+          function (err, ...args) {
+            if (err != null) {
+              return Router._handleError(callback, err, client, 'joinDoc', {
+                doc_id,
+                fromVersion
+              })
+            } else {
+              return callback(null, ...Array.from(args))
+            }
+          }
+        )
+      })
+
+      client.on('leaveDoc', function (doc_id, callback) {
+        if (typeof callback !== 'function') {
+          return Router._handleInvalidArguments(client, 'leaveDoc', arguments)
+        }
+
+        return WebsocketController.leaveDoc(client, doc_id, function (
+          err,
+          ...args
+        ) {
+          if (err != null) {
+            return Router._handleError(callback, err, client, 'leaveDoc')
+          } else {
+            return callback(null, ...Array.from(args))
+          }
+        })
+      })
+
+      client.on('clientTracking.getConnectedUsers', function (callback) {
+        if (callback == null) {
+          callback = function (error, users) {}
+        }
+        if (typeof callback !== 'function') {
+          return Router._handleInvalidArguments(
+            client,
+            'clientTracking.getConnectedUsers',
+            arguments
+          )
+        }
+
+        return WebsocketController.getConnectedUsers(client, function (
+          err,
+          users
+        ) {
+          if (err != null) {
+            return Router._handleError(
+              callback,
+              err,
+              client,
+              'clientTracking.getConnectedUsers'
+            )
+          } else {
+            return callback(null, users)
+          }
+        })
+      })
+
+      client.on('clientTracking.updatePosition', function (
+        cursorData,
+        callback
+      ) {
+        if (callback == null) {
+          callback = function (error) {}
+        }
+        if (typeof callback !== 'function') {
+          return Router._handleInvalidArguments(
+            client,
+            'clientTracking.updatePosition',
+            arguments
+          )
+        }
+
+        return WebsocketController.updateClientPosition(
+          client,
+          cursorData,
+          function (err) {
+            if (err != null) {
+              return Router._handleError(
+                callback,
+                err,
+                client,
+                'clientTracking.updatePosition'
+              )
+            } else {
+              return callback()
+            }
+          }
+        )
+      })
+
+      return client.on('applyOtUpdate', function (doc_id, update, callback) {
+        if (callback == null) {
+          callback = function (error) {}
+        }
+        if (typeof callback !== 'function') {
+          return Router._handleInvalidArguments(
+            client,
+            'applyOtUpdate',
+            arguments
+          )
+        }
+
+        return WebsocketController.applyOtUpdate(
+          client,
+          doc_id,
+          update,
+          function (err) {
+            if (err != null) {
+              return Router._handleError(
+                callback,
+                err,
+                client,
+                'applyOtUpdate',
+                { doc_id, update }
+              )
+            } else {
+              return callback()
+            }
+          }
+        )
+      })
+    })
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/real-time/app/js/SafeJsonParse.js

@@ -0,0 +1,35 @@
+/* eslint-disable
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+
+module.exports = {
+  parse(data, callback) {
+    let parsed
+    if (callback == null) {
+      callback = function (error, parsed) {}
+    }
+    if (data.length > Settings.maxUpdateSize) {
+      logger.error(
+        { head: data.slice(0, 1024), length: data.length },
+        'data too large to parse'
+      )
+      return callback(new Error('data too large to parse'))
+    }
+    try {
+      parsed = JSON.parse(data)
+    } catch (e) {
+      return callback(e)
+    }
+    return callback(null, parsed)
+  }
+}

services/real-time/app/js/SessionSockets.js

@@ -0,0 +1,37 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { EventEmitter } = require('events')
+
+module.exports = function (io, sessionStore, cookieParser, cookieName) {
+  const missingSessionError = new Error('could not look up session by key')
+
+  const sessionSockets = new EventEmitter()
+  const next = (error, socket, session) =>
+    sessionSockets.emit('connection', error, socket, session)
+
+  io.on('connection', function (socket) {
+    const req = socket.handshake
+    return cookieParser(req, {}, function () {
+      const sessionId = req.signedCookies && req.signedCookies[cookieName]
+      if (!sessionId) {
+        return next(missingSessionError, socket)
+      }
+      return sessionStore.get(sessionId, function (error, session) {
+        if (error) {
+          return next(error, socket)
+        }
+        if (!session) {
+          return next(missingSessionError, socket)
+        }
+        return next(null, socket, session)
+      })
+    })
+  })
+
+  return sessionSockets
+}

services/real-time/app/js/WebApiManager.js

@@ -0,0 +1,86 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let WebApiManager
+const request = require('request')
+const settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const { CodedError } = require('./Errors')
+
+module.exports = WebApiManager = {
+  joinProject(project_id, user, callback) {
+    if (callback == null) {
+      callback = function (error, project, privilegeLevel, isRestrictedUser) {}
+    }
+    const user_id = user._id
+    logger.log({ project_id, user_id }, 'sending join project request to web')
+    const url = `${settings.apis.web.url}/project/${project_id}/join`
+    const headers = {}
+    if (user.anonymousAccessToken != null) {
+      headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken
+    }
+    return request.post(
+      {
+        url,
+        qs: { user_id },
+        auth: {
+          user: settings.apis.web.user,
+          pass: settings.apis.web.pass,
+          sendImmediately: true
+        },
+        json: true,
+        jar: false,
+        headers
+      },
+      function (error, response, data) {
+        let err
+        if (error != null) {
+          return callback(error)
+        }
+        if (response.statusCode >= 200 && response.statusCode < 300) {
+          if (
+            data == null ||
+            (data != null ? data.project : undefined) == null
+          ) {
+            err = new Error('no data returned from joinProject request')
+            logger.error(
+              { err, project_id, user_id },
+              'error accessing web api'
+            )
+            return callback(err)
+          }
+          return callback(
+            null,
+            data.project,
+            data.privilegeLevel,
+            data.isRestrictedUser
+          )
+        } else if (response.statusCode === 429) {
+          logger.log(project_id, user_id, 'rate-limit hit when joining project')
+          return callback(
+            new CodedError(
+              'rate-limit hit when joining project',
+              'TooManyRequests'
+            )
+          )
+        } else {
+          err = new Error(
+            `non-success status code from web: ${response.statusCode}`
+          )
+          logger.error({ err, project_id, user_id }, 'error accessing web api')
+          return callback(err)
+        }
+      }
+    )
+  }
+}

services/real-time/app/js/WebsocketController.js

@@ -0,0 +1,608 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let WebsocketController
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const settings = require('settings-sharelatex')
+const WebApiManager = require('./WebApiManager')
+const AuthorizationManager = require('./AuthorizationManager')
+const DocumentUpdaterManager = require('./DocumentUpdaterManager')
+const ConnectedUsersManager = require('./ConnectedUsersManager')
+const WebsocketLoadBalancer = require('./WebsocketLoadBalancer')
+const RoomManager = require('./RoomManager')
+
+module.exports = WebsocketController = {
+  // If the protocol version changes when the client reconnects,
+  // it will force a full refresh of the page. Useful for non-backwards
+  // compatible protocol changes. Use only in extreme need.
+  PROTOCOL_VERSION: 2,
+
+  joinProject(client, user, project_id, callback) {
+    if (callback == null) {
+      callback = function (error, project, privilegeLevel, protocolVersion) {}
+    }
+    if (client.disconnected) {
+      metrics.inc('editor.join-project.disconnected', 1, {
+        status: 'immediately'
+      })
+      return callback()
+    }
+
+    const user_id = user != null ? user._id : undefined
+    logger.log(
+      { user_id, project_id, client_id: client.id },
+      'user joining project'
+    )
+    metrics.inc('editor.join-project')
+    return WebApiManager.joinProject(project_id, user, function (
+      error,
+      project,
+      privilegeLevel,
+      isRestrictedUser
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      if (client.disconnected) {
+        metrics.inc('editor.join-project.disconnected', 1, {
+          status: 'after-web-api-call'
+        })
+        return callback()
+      }
+
+      if (!privilegeLevel || privilegeLevel === '') {
+        const err = new Error('not authorized')
+        logger.warn(
+          { err, project_id, user_id, client_id: client.id },
+          'user is not authorized to join project'
+        )
+        return callback(err)
+      }
+
+      client.ol_context = {}
+      client.ol_context.privilege_level = privilegeLevel
+      client.ol_context.user_id = user_id
+      client.ol_context.project_id = project_id
+      client.ol_context.owner_id = __guard__(
+        project != null ? project.owner : undefined,
+        (x) => x._id
+      )
+      client.ol_context.first_name = user != null ? user.first_name : undefined
+      client.ol_context.last_name = user != null ? user.last_name : undefined
+      client.ol_context.email = user != null ? user.email : undefined
+      client.ol_context.connected_time = new Date()
+      client.ol_context.signup_date = user != null ? user.signUpDate : undefined
+      client.ol_context.login_count = user != null ? user.loginCount : undefined
+      client.ol_context.is_restricted_user = !!isRestrictedUser
+
+      RoomManager.joinProject(client, project_id, function (err) {
+        if (err) {
+          return callback(err)
+        }
+        logger.log(
+          { user_id, project_id, client_id: client.id },
+          'user joined project'
+        )
+        return callback(
+          null,
+          project,
+          privilegeLevel,
+          WebsocketController.PROTOCOL_VERSION
+        )
+      })
+
+      // No need to block for setting the user as connected in the cursor tracking
+      return ConnectedUsersManager.updateUserPosition(
+        project_id,
+        client.publicId,
+        user,
+        null,
+        function () {}
+      )
+    })
+  },
+
+  // We want to flush a project if there are no more (local) connected clients
+  // but we need to wait for the triggering client to disconnect. How long we wait
+  // is determined by FLUSH_IF_EMPTY_DELAY.
+  FLUSH_IF_EMPTY_DELAY: 500, // ms
+  leaveProject(io, client, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    const { project_id, user_id } = client.ol_context
+    if (!project_id) {
+      return callback()
+    } // client did not join project
+
+    metrics.inc('editor.leave-project')
+    logger.log(
+      { project_id, user_id, client_id: client.id },
+      'client leaving project'
+    )
+    WebsocketLoadBalancer.emitToRoom(
+      project_id,
+      'clientTracking.clientDisconnected',
+      client.publicId
+    )
+
+    // We can do this in the background
+    ConnectedUsersManager.markUserAsDisconnected(
+      project_id,
+      client.publicId,
+      function (err) {
+        if (err != null) {
+          return logger.error(
+            { err, project_id, user_id, client_id: client.id },
+            'error marking client as disconnected'
+          )
+        }
+      }
+    )
+
+    RoomManager.leaveProjectAndDocs(client)
+    return setTimeout(function () {
+      const remainingClients = io.sockets.clients(project_id)
+      if (remainingClients.length === 0) {
+        // Flush project in the background
+        DocumentUpdaterManager.flushProjectToMongoAndDelete(
+          project_id,
+          function (err) {
+            if (err != null) {
+              return logger.error(
+                { err, project_id, user_id, client_id: client.id },
+                'error flushing to doc updater after leaving project'
+              )
+            }
+          }
+        )
+      }
+      return callback()
+    }, WebsocketController.FLUSH_IF_EMPTY_DELAY)
+  },
+
+  joinDoc(client, doc_id, fromVersion, options, callback) {
+    if (fromVersion == null) {
+      fromVersion = -1
+    }
+    if (callback == null) {
+      callback = function (error, doclines, version, ops, ranges) {}
+    }
+    if (client.disconnected) {
+      metrics.inc('editor.join-doc.disconnected', 1, { status: 'immediately' })
+      return callback()
+    }
+
+    metrics.inc('editor.join-doc')
+    const { project_id, user_id, is_restricted_user } = client.ol_context
+    if (project_id == null) {
+      return callback(new Error('no project_id found on client'))
+    }
+    logger.log(
+      { user_id, project_id, doc_id, fromVersion, client_id: client.id },
+      'client joining doc'
+    )
+
+    return AuthorizationManager.assertClientCanViewProject(client, function (
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      // ensure the per-doc applied-ops channel is subscribed before sending the
+      // doc to the client, so that no events are missed.
+      return RoomManager.joinDoc(client, doc_id, function (error) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (client.disconnected) {
+          metrics.inc('editor.join-doc.disconnected', 1, {
+            status: 'after-joining-room'
+          })
+          // the client will not read the response anyways
+          return callback()
+        }
+
+        return DocumentUpdaterManager.getDocument(
+          project_id,
+          doc_id,
+          fromVersion,
+          function (error, lines, version, ranges, ops) {
+            let err
+            if (error != null) {
+              return callback(error)
+            }
+            if (client.disconnected) {
+              metrics.inc('editor.join-doc.disconnected', 1, {
+                status: 'after-doc-updater-call'
+              })
+              // the client will not read the response anyways
+              return callback()
+            }
+
+            if (
+              is_restricted_user &&
+              (ranges != null ? ranges.comments : undefined) != null
+            ) {
+              ranges.comments = []
+            }
+
+            // Encode any binary bits of data so it can go via WebSockets
+            // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html
+            const encodeForWebsockets = (text) =>
+              unescape(encodeURIComponent(text))
+            const escapedLines = []
+            for (let line of Array.from(lines)) {
+              try {
+                line = encodeForWebsockets(line)
+              } catch (error1) {
+                err = error1
+                logger.err(
+                  {
+                    err,
+                    project_id,
+                    doc_id,
+                    fromVersion,
+                    line,
+                    client_id: client.id
+                  },
+                  'error encoding line uri component'
+                )
+                return callback(err)
+              }
+              escapedLines.push(line)
+            }
+            if (options.encodeRanges) {
+              try {
+                for (const comment of Array.from(
+                  (ranges != null ? ranges.comments : undefined) || []
+                )) {
+                  if (comment.op.c != null) {
+                    comment.op.c = encodeForWebsockets(comment.op.c)
+                  }
+                }
+                for (const change of Array.from(
+                  (ranges != null ? ranges.changes : undefined) || []
+                )) {
+                  if (change.op.i != null) {
+                    change.op.i = encodeForWebsockets(change.op.i)
+                  }
+                  if (change.op.d != null) {
+                    change.op.d = encodeForWebsockets(change.op.d)
+                  }
+                }
+              } catch (error2) {
+                err = error2
+                logger.err(
+                  {
+                    err,
+                    project_id,
+                    doc_id,
+                    fromVersion,
+                    ranges,
+                    client_id: client.id
+                  },
+                  'error encoding range uri component'
+                )
+                return callback(err)
+              }
+            }
+
+            AuthorizationManager.addAccessToDoc(client, doc_id)
+            logger.log(
+              {
+                user_id,
+                project_id,
+                doc_id,
+                fromVersion,
+                client_id: client.id
+              },
+              'client joined doc'
+            )
+            return callback(null, escapedLines, version, ops, ranges)
+          }
+        )
+      })
+    })
+  },
+
+  leaveDoc(client, doc_id, callback) {
+    // client may have disconnected, but we have to cleanup internal state.
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    metrics.inc('editor.leave-doc')
+    const { project_id, user_id } = client.ol_context
+    logger.log(
+      { user_id, project_id, doc_id, client_id: client.id },
+      'client leaving doc'
+    )
+    RoomManager.leaveDoc(client, doc_id)
+    // we could remove permission when user leaves a doc, but because
+    // the connection is per-project, we continue to allow access
+    // after the initial joinDoc since we know they are already authorised.
+    // # AuthorizationManager.removeAccessToDoc client, doc_id
+    return callback()
+  },
+  updateClientPosition(client, cursorData, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (client.disconnected) {
+      // do not create a ghost entry in redis
+      return callback()
+    }
+
+    metrics.inc('editor.update-client-position', 0.1)
+    const {
+      project_id,
+      first_name,
+      last_name,
+      email,
+      user_id
+    } = client.ol_context
+    logger.log(
+      { user_id, project_id, client_id: client.id, cursorData },
+      'updating client position'
+    )
+
+    return AuthorizationManager.assertClientCanViewProjectAndDoc(
+      client,
+      cursorData.doc_id,
+      function (error) {
+        if (error != null) {
+          logger.warn(
+            { err: error, client_id: client.id, project_id, user_id },
+            "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet."
+          )
+          return callback()
+        }
+        cursorData.id = client.publicId
+        if (user_id != null) {
+          cursorData.user_id = user_id
+        }
+        if (email != null) {
+          cursorData.email = email
+        }
+        // Don't store anonymous users in redis to avoid influx
+        if (!user_id || user_id === 'anonymous-user') {
+          cursorData.name = ''
+          callback()
+        } else {
+          cursorData.name =
+            first_name && last_name
+              ? `${first_name} ${last_name}`
+              : first_name || last_name || ''
+          ConnectedUsersManager.updateUserPosition(
+            project_id,
+            client.publicId,
+            {
+              first_name,
+              last_name,
+              email,
+              _id: user_id
+            },
+            {
+              row: cursorData.row,
+              column: cursorData.column,
+              doc_id: cursorData.doc_id
+            },
+            callback
+          )
+        }
+        return WebsocketLoadBalancer.emitToRoom(
+          project_id,
+          'clientTracking.clientUpdated',
+          cursorData
+        )
+      }
+    )
+  },
+
+  CLIENT_REFRESH_DELAY: 1000,
+  getConnectedUsers(client, callback) {
+    if (callback == null) {
+      callback = function (error, users) {}
+    }
+    if (client.disconnected) {
+      // they are not interested anymore, skip the redis lookups
+      return callback()
+    }
+
+    metrics.inc('editor.get-connected-users')
+    const { project_id, user_id, is_restricted_user } = client.ol_context
+    if (is_restricted_user) {
+      return callback(null, [])
+    }
+    if (project_id == null) {
+      return callback(new Error('no project_id found on client'))
+    }
+    logger.log(
+      { user_id, project_id, client_id: client.id },
+      'getting connected users'
+    )
+    return AuthorizationManager.assertClientCanViewProject(client, function (
+      error
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh')
+      return setTimeout(
+        () =>
+          ConnectedUsersManager.getConnectedUsers(project_id, function (
+            error,
+            users
+          ) {
+            if (error != null) {
+              return callback(error)
+            }
+            callback(null, users)
+            return logger.log(
+              { user_id, project_id, client_id: client.id },
+              'got connected users'
+            )
+          }),
+        WebsocketController.CLIENT_REFRESH_DELAY
+      )
+    })
+  },
+
+  applyOtUpdate(client, doc_id, update, callback) {
+    // client may have disconnected, but we can submit their update to doc-updater anyways.
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    const { user_id, project_id } = client.ol_context
+    if (project_id == null) {
+      return callback(new Error('no project_id found on client'))
+    }
+
+    return WebsocketController._assertClientCanApplyUpdate(
+      client,
+      doc_id,
+      update,
+      function (error) {
+        if (error != null) {
+          logger.warn(
+            { err: error, doc_id, client_id: client.id, version: update.v },
+            'client is not authorized to make update'
+          )
+          setTimeout(
+            () =>
+              // Disconnect, but give the client the chance to receive the error
+              client.disconnect(),
+            100
+          )
+          return callback(error)
+        }
+        if (!update.meta) {
+          update.meta = {}
+        }
+        update.meta.source = client.publicId
+        update.meta.user_id = user_id
+        metrics.inc('editor.doc-update', 0.3)
+
+        logger.log(
+          {
+            user_id,
+            doc_id,
+            project_id,
+            client_id: client.id,
+            version: update.v
+          },
+          'sending update to doc updater'
+        )
+
+        return DocumentUpdaterManager.queueChange(
+          project_id,
+          doc_id,
+          update,
+          function (error) {
+            if (
+              (error != null ? error.message : undefined) ===
+              'update is too large'
+            ) {
+              metrics.inc('update_too_large')
+              const { updateSize } = error
+              logger.warn(
+                { user_id, project_id, doc_id, updateSize },
+                'update is too large'
+              )
+
+              // mark the update as received -- the client should not send it again!
+              callback()
+
+              // trigger an out-of-sync error
+              const message = {
+                project_id,
+                doc_id,
+                error: 'update is too large'
+              }
+              setTimeout(function () {
+                if (client.disconnected) {
+                  // skip the message broadcast, the client has moved on
+                  return metrics.inc('editor.doc-update.disconnected', 1, {
+                    status: 'at-otUpdateError'
+                  })
+                }
+                client.emit('otUpdateError', message.error, message)
+                return client.disconnect()
+              }, 100)
+              return
+            }
+
+            if (error != null) {
+              logger.error(
+                {
+                  err: error,
+                  project_id,
+                  doc_id,
+                  client_id: client.id,
+                  version: update.v
+                },
+                'document was not available for update'
+              )
+              client.disconnect()
+            }
+            return callback(error)
+          }
+        )
+      }
+    )
+  },
+
+  _assertClientCanApplyUpdate(client, doc_id, update, callback) {
+    return AuthorizationManager.assertClientCanEditProjectAndDoc(
+      client,
+      doc_id,
+      function (error) {
+        if (error != null) {
+          if (
+            error.message === 'not authorized' &&
+            WebsocketController._isCommentUpdate(update)
+          ) {
+            // This might be a comment op, which we only need read-only priveleges for
+            return AuthorizationManager.assertClientCanViewProjectAndDoc(
+              client,
+              doc_id,
+              callback
+            )
+          } else {
+            return callback(error)
+          }
+        } else {
+          return callback(null)
+        }
+      }
+    )
+  },
+
+  _isCommentUpdate(update) {
+    for (const op of Array.from(update.op)) {
+      if (op.c == null) {
+        return false
+      }
+    }
+    return true
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/real-time/app/js/WebsocketLoadBalancer.js

@@ -0,0 +1,217 @@
+/* eslint-disable
+    camelcase,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let WebsocketLoadBalancer
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const RedisClientManager = require('./RedisClientManager')
+const SafeJsonParse = require('./SafeJsonParse')
+const EventLogger = require('./EventLogger')
+const HealthCheckManager = require('./HealthCheckManager')
+const RoomManager = require('./RoomManager')
+const ChannelManager = require('./ChannelManager')
+const ConnectedUsersManager = require('./ConnectedUsersManager')
+
+const RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [
+  'connectionAccepted',
+  'otUpdateApplied',
+  'otUpdateError',
+  'joinDoc',
+  'reciveNewDoc',
+  'reciveNewFile',
+  'reciveNewFolder',
+  'removeEntity'
+]
+
+module.exports = WebsocketLoadBalancer = {
+  rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub),
+  rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub),
+
+  emitToRoom(room_id, message, ...payload) {
+    if (room_id == null) {
+      logger.warn(
+        { message, payload },
+        'no room_id provided, ignoring emitToRoom'
+      )
+      return
+    }
+    const data = JSON.stringify({
+      room_id,
+      message,
+      payload
+    })
+    logger.log(
+      { room_id, message, payload, length: data.length },
+      'emitting to room'
+    )
+
+    return Array.from(this.rclientPubList).map((rclientPub) =>
+      ChannelManager.publish(rclientPub, 'editor-events', room_id, data)
+    )
+  },
+
+  emitToAll(message, ...payload) {
+    return this.emitToRoom('all', message, ...Array.from(payload))
+  },
+
+  listenForEditorEvents(io) {
+    logger.log(
+      { rclients: this.rclientPubList.length },
+      'publishing editor events'
+    )
+    logger.log(
+      { rclients: this.rclientSubList.length },
+      'listening for editor events'
+    )
+    for (const rclientSub of Array.from(this.rclientSubList)) {
+      rclientSub.subscribe('editor-events')
+      rclientSub.on('message', function (channel, message) {
+        if (Settings.debugEvents > 0) {
+          EventLogger.debugEvent(channel, message)
+        }
+        return WebsocketLoadBalancer._processEditorEvent(io, channel, message)
+      })
+    }
+    return this.handleRoomUpdates(this.rclientSubList)
+  },
+
+  handleRoomUpdates(rclientSubList) {
+    const roomEvents = RoomManager.eventSource()
+    roomEvents.on('project-active', function (project_id) {
+      const subscribePromises = Array.from(rclientSubList).map((rclient) =>
+        ChannelManager.subscribe(rclient, 'editor-events', project_id)
+      )
+      return RoomManager.emitOnCompletion(
+        subscribePromises,
+        `project-subscribed-${project_id}`
+      )
+    })
+    return roomEvents.on('project-empty', (project_id) =>
+      Array.from(rclientSubList).map((rclient) =>
+        ChannelManager.unsubscribe(rclient, 'editor-events', project_id)
+      )
+    )
+  },
+
+  _processEditorEvent(io, channel, message) {
+    return SafeJsonParse.parse(message, function (error, message) {
+      let clientList
+      let client
+      if (error != null) {
+        logger.error({ err: error, channel }, 'error parsing JSON')
+        return
+      }
+      if (message.room_id === 'all') {
+        return io.sockets.emit(message.message, ...Array.from(message.payload))
+      } else if (
+        message.message === 'clientTracking.refresh' &&
+        message.room_id != null
+      ) {
+        clientList = io.sockets.clients(message.room_id)
+        logger.log(
+          {
+            channel,
+            message: message.message,
+            room_id: message.room_id,
+            message_id: message._id,
+            socketIoClients: (() => {
+              const result = []
+              for (client of Array.from(clientList)) {
+                result.push(client.id)
+              }
+              return result
+            })()
+          },
+          'refreshing client list'
+        )
+        return (() => {
+          const result1 = []
+          for (client of Array.from(clientList)) {
+            result1.push(
+              ConnectedUsersManager.refreshClient(
+                message.room_id,
+                client.publicId
+              )
+            )
+          }
+          return result1
+        })()
+      } else if (message.room_id != null) {
+        if (message._id != null && Settings.checkEventOrder) {
+          const status = EventLogger.checkEventOrder(
+            'editor-events',
+            message._id,
+            message
+          )
+          if (status === 'duplicate') {
+            return // skip duplicate events
+          }
+        }
+
+        const is_restricted_message = !Array.from(
+          RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST
+        ).includes(message.message)
+
+        // send messages only to unique clients (due to duplicate entries in io.sockets.clients)
+        clientList = io.sockets
+          .clients(message.room_id)
+          .filter(
+            (client) =>
+              !(is_restricted_message && client.ol_context.is_restricted_user)
+          )
+
+        // avoid unnecessary work if no clients are connected
+        if (clientList.length === 0) {
+          return
+        }
+        logger.log(
+          {
+            channel,
+            message: message.message,
+            room_id: message.room_id,
+            message_id: message._id,
+            socketIoClients: (() => {
+              const result2 = []
+              for (client of Array.from(clientList)) {
+                result2.push(client.id)
+              }
+              return result2
+            })()
+          },
+          'distributing event to clients'
+        )
+        const seen = {}
+        return (() => {
+          const result3 = []
+          for (client of Array.from(clientList)) {
+            if (!seen[client.id]) {
+              seen[client.id] = true
+              result3.push(
+                client.emit(message.message, ...Array.from(message.payload))
+              )
+            } else {
+              result3.push(undefined)
+            }
+          }
+          return result3
+        })()
+      } else if (message.health_check != null) {
+        logger.debug(
+          { message },
+          'got health check message in editor events channel'
+        )
+        return HealthCheckManager.check(channel, message.key)
+      }
+    })
+  }
+}

services/real-time/buildscript.txt

@@ -1,10 +1,10 @@
 real-time
---public-repo=True
---language=coffeescript
---env-add=
---node-version=10.21.0
 --acceptance-creds=None
 --dependencies=redis
 --docker-repos=gcr.io/overleaf-ops
+--env-add=
 --env-pass-through=
---script-version=1.3.5
+--language=es
+--node-version=10.21.0
+--public-repo=True
+--script-version=2.3.0

services/real-time/config/settings.defaults.coffee

@@ -1,79 +0,0 @@
-settings =
-	redis:
-
-		pubsub:
-			host: process.env['PUBSUB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost"
-			port: process.env['PUBSUB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379"
-			password: process.env["PUBSUB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
-			maxRetriesPerRequest: parseInt(process.env["PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20")
-
-		realtime:
-			host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost"
-			port: process.env['REAL_TIME_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379"
-			password: process.env["REAL_TIME_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
-			key_schema:
-				clientsInProject: ({project_id}) -> "clients_in_project:{#{project_id}}"
-				connectedUser: ({project_id, client_id})-> "connected_user:{#{project_id}}:#{client_id}"
-			maxRetriesPerRequest: parseInt(process.env["REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20")
-
-		documentupdater:
-			host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost"
-			port: process.env['DOC_UPDATER_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379"
-			password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
-			key_schema:
-				pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
-			maxRetriesPerRequest: parseInt(process.env["DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20")
-
-		websessions: 			
-			host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost"
-			port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379"
-			password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
-			maxRetriesPerRequest: parseInt(process.env["WEB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20")
-
-	internal:
-		realTime:
-			port: 3026
-			host: process.env['LISTEN_ADDRESS'] or "localhost"
-			user: "sharelatex"
-			pass: "password"
-			
-	apis:
-		web:
-			url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}"
-			user: process.env['WEB_API_USER'] or "sharelatex"
-			pass: process.env['WEB_API_PASSWORD'] or "password"
-		documentupdater:
-			url: "http://#{process.env['DOCUMENT_UPDATER_HOST'] or process.env['DOCUPDATER_HOST'] or "localhost"}:3003"
-			
-	security:
-		sessionSecret: process.env['SESSION_SECRET'] or "secret-please-change"
-		
-	cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid"
-	
-	max_doc_length: 2 * 1024 * 1024 # 2mb
-
-	# combine
-	# max_doc_length (2mb see above) * 2 (delete + insert)
-	# max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater)
-	# overhead for JSON serialization
-	maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) or 7 * 1024 * 1024 + 64 * 1024
-
-	shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9
-
-	continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false
-
-	checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false
-	
-	publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false
-
-	statusCheckInterval: parseInt(process.env['STATUS_CHECK_INTERVAL'] or '0')
-
-	sentry:
-		dsn: process.env.SENTRY_DSN
-
-	errors:
-		catchUncaughtErrors: true
-		shutdownOnUncaughtError: true
-	
-# console.log settings.redis
-module.exports = settings

services/real-time/config/settings.defaults.js

@@ -0,0 +1,143 @@
+/* eslint-disable camelcase */
+
+const settings = {
+  redis: {
+    pubsub: {
+      host:
+        process.env.PUBSUB_REDIS_HOST || process.env.REDIS_HOST || 'localhost',
+      port: process.env.PUBSUB_REDIS_PORT || process.env.REDIS_PORT || '6379',
+      password:
+        process.env.PUBSUB_REDIS_PASSWORD || process.env.REDIS_PASSWORD || '',
+      maxRetriesPerRequest: parseInt(
+        process.env.PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST ||
+          process.env.REDIS_MAX_RETRIES_PER_REQUEST ||
+          '20'
+      )
+    },
+
+    realtime: {
+      host:
+        process.env.REAL_TIME_REDIS_HOST ||
+        process.env.REDIS_HOST ||
+        'localhost',
+      port:
+        process.env.REAL_TIME_REDIS_PORT || process.env.REDIS_PORT || '6379',
+      password:
+        process.env.REAL_TIME_REDIS_PASSWORD ||
+        process.env.REDIS_PASSWORD ||
+        '',
+      key_schema: {
+        clientsInProject({ project_id }) {
+          return `clients_in_project:{${project_id}}`
+        },
+        connectedUser({ project_id, client_id }) {
+          return `connected_user:{${project_id}}:${client_id}`
+        }
+      },
+      maxRetriesPerRequest: parseInt(
+        process.env.REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST ||
+          process.env.REDIS_MAX_RETRIES_PER_REQUEST ||
+          '20'
+      )
+    },
+
+    documentupdater: {
+      host:
+        process.env.DOC_UPDATER_REDIS_HOST ||
+        process.env.REDIS_HOST ||
+        'localhost',
+      port:
+        process.env.DOC_UPDATER_REDIS_PORT || process.env.REDIS_PORT || '6379',
+      password:
+        process.env.DOC_UPDATER_REDIS_PASSWORD ||
+        process.env.REDIS_PASSWORD ||
+        '',
+      key_schema: {
+        pendingUpdates({ doc_id }) {
+          return `PendingUpdates:{${doc_id}}`
+        }
+      },
+      maxRetriesPerRequest: parseInt(
+        process.env.DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST ||
+          process.env.REDIS_MAX_RETRIES_PER_REQUEST ||
+          '20'
+      )
+    },
+
+    websessions: {
+      host: process.env.WEB_REDIS_HOST || process.env.REDIS_HOST || 'localhost',
+      port: process.env.WEB_REDIS_PORT || process.env.REDIS_PORT || '6379',
+      password:
+        process.env.WEB_REDIS_PASSWORD || process.env.REDIS_PASSWORD || '',
+      maxRetriesPerRequest: parseInt(
+        process.env.WEB_REDIS_MAX_RETRIES_PER_REQUEST ||
+          process.env.REDIS_MAX_RETRIES_PER_REQUEST ||
+          '20'
+      )
+    }
+  },
+
+  internal: {
+    realTime: {
+      port: 3026,
+      host: process.env.LISTEN_ADDRESS || 'localhost',
+      user: 'sharelatex',
+      pass: 'password'
+    }
+  },
+
+  apis: {
+    web: {
+      url: `http://${
+        process.env.WEB_API_HOST || process.env.WEB_HOST || 'localhost'
+      }:${process.env.WEB_API_PORT || process.env.WEB_PORT || 3000}`,
+      user: process.env.WEB_API_USER || 'sharelatex',
+      pass: process.env.WEB_API_PASSWORD || 'password'
+    },
+    documentupdater: {
+      url: `http://${
+        process.env.DOCUMENT_UPDATER_HOST ||
+        process.env.DOCUPDATER_HOST ||
+        'localhost'
+      }:3003`
+    }
+  },
+
+  security: {
+    sessionSecret: process.env.SESSION_SECRET || 'secret-please-change'
+  },
+
+  cookieName: process.env.COOKIE_NAME || 'sharelatex.sid',
+
+  max_doc_length: 2 * 1024 * 1024, // 2mb
+
+  // combine
+  // max_doc_length (2mb see above) * 2 (delete + insert)
+  // max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater)
+  // overhead for JSON serialization
+  maxUpdateSize:
+    parseInt(process.env.MAX_UPDATE_SIZE) || 7 * 1024 * 1024 + 64 * 1024,
+
+  shutdownDrainTimeWindow: process.env.SHUTDOWN_DRAIN_TIME_WINDOW || 9,
+
+  continualPubsubTraffic: process.env.CONTINUAL_PUBSUB_TRAFFIC || false,
+
+  checkEventOrder: process.env.CHECK_EVENT_ORDER || false,
+
+  publishOnIndividualChannels:
+    process.env.PUBLISH_ON_INDIVIDUAL_CHANNELS || false,
+
+  statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'),
+
+  sentry: {
+    dsn: process.env.SENTRY_DSN
+  },
+
+  errors: {
+    catchUncaughtErrors: true,
+    shutdownOnUncaughtError: true
+  }
+}
+
+// console.log settings.redis
+module.exports = settings

services/real-time/docker-compose.ci.yml

@@ -1,7 +1,6 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.3.5
 
 version: "2.3"
 

services/real-time/docker-compose.yml

@@ -1,7 +1,6 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.3.5
 
 version: "2.3"
 

services/real-time/nodemon.json

@@ -10,10 +10,9 @@
   },
 
   "watch": [
-    "app/coffee/",
-    "app.coffee",
+    "app/js/",
+    "app.js",
     "config/"
   ],
-  "ext": "coffee"
-
+  "ext": "js"
 }

services/real-time/package.json

@@ -8,17 +8,15 @@
     "url": "https://github.com/sharelatex/real-time-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
-    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
-    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
-    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
-    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "start": "node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
+    "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "nodemon": "nodemon --config nodemon.json",
-    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+    "lint": "node_modules/.bin/eslint .",
+    "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different",
+    "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write"
   },
   "dependencies": {
     "async": "^0.9.0",
@@ -41,10 +39,23 @@
     "bunyan": "~0.22.3",
     "chai": "~1.9.1",
     "cookie-signature": "^1.1.0",
+    "eslint": "^6.8.0",
+    "eslint-config-prettier": "^6.10.0",
+    "eslint-config-standard": "^14.1.0",
+    "eslint-plugin-chai-expect": "^2.1.0",
+    "eslint-plugin-chai-friendly": "^0.5.0",
+    "eslint-plugin-import": "^2.20.1",
+    "eslint-plugin-mocha": "^6.3.0",
+    "eslint-plugin-node": "^11.0.0",
+    "eslint-plugin-prettier": "^3.1.2",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.1",
+    "mocha": "^4.0.1",
+    "prettier": "^2.0.0",
+    "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "~0.3.0",
     "sinon": "^2.4.1",
-    "mocha": "^4.0.1",
-    "uid-safe": "^2.1.5",
-    "timekeeper": "0.0.4"
+    "timekeeper": "0.0.4",
+    "uid-safe": "^2.1.5"
   }
 }

services/real-time/socket.io.patch.js

@@ -1,51 +1,66 @@
 // EventEmitter has been removed from process in node >= 7
 // https://github.com/nodejs/node/commit/62b544290a075fe38e233887a06c408ba25a1c71
-if(process.versions.node.split('.')[0] >= 7) {
+/*
+  A socket.io dependency expects the EventEmitter to be available at
+    `process.EventEmitter`.
+  See this trace:
+  ---
+
+  /app/node_modules/policyfile/lib/server.js:254
+  Object.keys(process.EventEmitter.prototype).forEach(function proxy (key){
+                                   ^
+
+  TypeError: Cannot read property 'prototype' of undefined
+    at Object.<anonymous> (/app/node_modules/policyfile/lib/server.js:254:34)
+ */
+if (process.versions.node.split('.')[0] >= 7) {
+  // eslint-disable-next-line node/no-deprecated-api
   process.EventEmitter = require('events')
 }
 
-var io = require("socket.io");
+var io = require('socket.io')
+const logger = require('logger-sharelatex')
 
-if (io.version === "0.9.16" || io.version === "0.9.19") {
-  console.log("patching socket.io hybi-16 transport frame prototype");
-  var transports = require("socket.io/lib/transports/websocket/hybi-16.js");
-  transports.prototype.frame = patchedFrameHandler;
+if (io.version === '0.9.16' || io.version === '0.9.19') {
+  logger.warn('patching socket.io hybi-16 transport frame prototype')
+  var transports = require('socket.io/lib/transports/websocket/hybi-16.js')
+  transports.prototype.frame = patchedFrameHandler
   // file hybi-07-12 has the same problem but no browsers are using that protocol now
 }
 
 function patchedFrameHandler(opcode, str) {
-  var dataBuffer = new Buffer(str),
-    dataLength = dataBuffer.length,
-    startOffset = 2,
-    secondByte = dataLength;
+  var dataBuffer = Buffer.from(str)
+  var dataLength = dataBuffer.length
+  var startOffset = 2
+  var secondByte = dataLength
   if (dataLength === 65536) {
-    console.log("fixing invalid frame length in socket.io");
+    logger.log('fixing invalid frame length in socket.io')
   }
   if (dataLength > 65535) {
     // original code had > 65536
-    startOffset = 10;
-    secondByte = 127;
+    startOffset = 10
+    secondByte = 127
   } else if (dataLength > 125) {
-    startOffset = 4;
-    secondByte = 126;
+    startOffset = 4
+    secondByte = 126
   }
-  var outputBuffer = new Buffer(dataLength + startOffset);
-  outputBuffer[0] = opcode;
-  outputBuffer[1] = secondByte;
-  dataBuffer.copy(outputBuffer, startOffset);
+  var outputBuffer = Buffer.alloc(dataLength + startOffset)
+  outputBuffer[0] = opcode
+  outputBuffer[1] = secondByte
+  dataBuffer.copy(outputBuffer, startOffset)
   switch (secondByte) {
     case 126:
-      outputBuffer[2] = dataLength >>> 8;
-      outputBuffer[3] = dataLength % 256;
-      break;
+      outputBuffer[2] = dataLength >>> 8
+      outputBuffer[3] = dataLength % 256
+      break
     case 127:
-      var l = dataLength;
+      var l = dataLength
       for (var i = 1; i <= 8; ++i) {
-        outputBuffer[startOffset - i] = l & 0xff;
-        l >>>= 8;
+        outputBuffer[startOffset - i] = l & 0xff
+        l >>>= 8
       }
   }
-  return outputBuffer;
+  return outputBuffer
 }
 
 const parser = require('socket.io/lib/parser')

services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee

@@ -1,222 +0,0 @@
-async = require "async"
-chai = require("chai")
-expect = chai.expect
-chai.should()
-
-RealTimeClient = require "./helpers/RealTimeClient"
-FixturesManager = require "./helpers/FixturesManager"
-
-settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(settings.redis.documentupdater)
-
-redisSettings = settings.redis
-
-describe "applyOtUpdate", ->
-	before ->
-		@update = {
-			op: [{i: "foo", p: 42}]
-		}
-	describe "when authorized", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, cb
-					
-				(cb) =>
-					@client.emit "applyOtUpdate", @doc_id, @update, cb
-			], done
-		
-		it "should push the doc into the pending updates list", (done) ->
-			rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) =>
-				doc_id.should.equal "#{@project_id}:#{@doc_id}"
-				done()
-			return null
-
-		it "should push the update into redis", (done) ->
-			rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) =>
-				update = JSON.parse(update)
-				update.op.should.deep.equal @update.op
-				update.meta.should.deep.equal {
-					source: @client.publicId
-					user_id: @user_id
-				}
-				done()
-			return null
-
-		after (done) ->
-			async.series [
-				(cb) => rclient.del "pending-updates-list", cb
-				(cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb
-				(cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates(@doc_id), cb
-			], done
-		
-	describe "when authorized with a huge edit update", ->
-		before (done) ->
-			@update = {
-				op: {
-					p: 12,
-					t: "update is too large".repeat(1024 * 400) # >7MB
-				}
-			}
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-					@client.on "otUpdateError", (@otUpdateError) =>
-
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, cb
-
-				(cb) =>
-					@client.emit "applyOtUpdate", @doc_id, @update, (@error) =>
-						cb()
-			], done
-
-		it "should not return an error", ->
-			expect(@error).to.not.exist
-
-		it "should send an otUpdateError to the client", (done) ->
-			setTimeout () =>
-				expect(@otUpdateError).to.exist
-				done()
-			, 300
-
-		it "should disconnect the client", (done) ->
-			setTimeout () =>
-				@client.socket.connected.should.equal false
-				done()
-			, 300
-
-		it "should not put the update in redis", (done) ->
-			rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) =>
-				len.should.equal 0
-				done()
-			return null
-
-	describe "when authorized to read-only with an edit update", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readOnly"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, cb
-					
-				(cb) =>
-					@client.emit "applyOtUpdate", @doc_id, @update, (@error) =>
-						cb()
-			], done
-		
-		it "should return an error", ->
-			expect(@error).to.exist
-		
-		it "should disconnect the client", (done) ->
-			setTimeout () =>
-				@client.socket.connected.should.equal false
-				done()
-			, 300
-			
-		it "should not put the update in redis", (done) ->
-			rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) =>
-				len.should.equal 0
-				done()
-			return null
-				
-	describe "when authorized to read-only with a comment update", ->
-		before (done) ->
-			@comment_update = {
-				op: [{c: "foo", p: 42}]
-			}
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readOnly"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, cb
-					
-				(cb) =>
-					@client.emit "applyOtUpdate", @doc_id, @comment_update, cb
-			], done
-		
-		it "should push the doc into the pending updates list", (done) ->
-			rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) =>
-				doc_id.should.equal "#{@project_id}:#{@doc_id}"
-				done()
-			return null
-
-		it "should push the update into redis", (done) ->
-			rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) =>
-				update = JSON.parse(update)
-				update.op.should.deep.equal @comment_update.op
-				update.meta.should.deep.equal {
-					source: @client.publicId
-					user_id: @user_id
-				}
-				done()
-			return null
-
-		after (done) ->
-			async.series [
-				(cb) => rclient.del "pending-updates-list", cb
-				(cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb
-				(cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), cb
-			], done

services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee

@@ -1,146 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-chai.should()
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockWebServer = require "./helpers/MockWebServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-describe "clientTracking", ->
-	describe "when a client updates its cursor location", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: { name: "Test Project" }
-					}, (error, {@user_id, @project_id}) => cb()
-				
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-				
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-					
-				(cb) =>
-					@clientB = RealTimeClient.connect()
-					@clientB.on "connectionAccepted", cb
-					
-				(cb) =>
-					@clientA.emit "joinProject", {
-						project_id: @project_id
-					}, cb
-				
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, cb
-					
-				(cb) =>
-					@clientB.emit "joinProject", {
-						project_id: @project_id
-					}, cb
-					
-				(cb) =>
-					@updates = []
-					@clientB.on "clientTracking.clientUpdated", (data) =>
-						@updates.push data
-
-					@clientA.emit "clientTracking.updatePosition", {
-						row: @row = 42
-						column: @column = 36
-						doc_id: @doc_id
-					}, (error) ->
-						throw error if error?
-						setTimeout cb, 300 # Give the message a chance to reach client B.
-			], done
-			
-		it "should tell other clients about the update", ->
-			@updates.should.deep.equal [
-				{
-					row: @row
-					column: @column
-					doc_id: @doc_id
-					id: @clientA.publicId
-					user_id: @user_id
-					name: "Joe Bloggs"
-				}
-			]
-		
-		it "should record the update in getConnectedUsers", (done) ->
-			@clientB.emit "clientTracking.getConnectedUsers", (error, users) =>
-				for user in users
-					if user.client_id == @clientA.publicId
-						expect(user.cursorData).to.deep.equal({
-							row: @row
-							column: @column
-							doc_id: @doc_id 
-						})
-						return done()
-				throw new Error("user was never found")
-				
-	describe "when an anonymous client updates its cursor location", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: { name: "Test Project"	}
-						publicAccess: "readAndWrite"
-					}, (error, {@user_id, @project_id}) => cb()
-				
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-				
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", {
-						project_id: @project_id
-					}, cb
-			
-				(cb) =>
-					RealTimeClient.setSession({}, cb)
-					
-				(cb) =>
-					@anonymous = RealTimeClient.connect()
-					@anonymous.on "connectionAccepted", cb	
-					
-				(cb) =>
-					@anonymous.emit "joinProject", {
-						project_id: @project_id
-					}, cb
-				
-				(cb) =>
-					@anonymous.emit "joinDoc", @doc_id, cb
-					
-				(cb) =>
-					@updates = []
-					@clientA.on "clientTracking.clientUpdated", (data) =>
-						@updates.push data
-
-					@anonymous.emit "clientTracking.updatePosition", {
-						row: @row = 42
-						column: @column = 36
-						doc_id: @doc_id
-					}, (error) ->
-						throw error if error?
-						setTimeout cb, 300 # Give the message a chance to reach client B.
-			], done
-			
-		it "should tell other clients about the update", ->
-			@updates.should.deep.equal [
-				{
-					row: @row
-					column: @column
-					doc_id: @doc_id
-					id: @anonymous.publicId
-					user_id: "anonymous-user"
-					name: ""
-				}
-			]

services/real-time/test/acceptance/coffee/DrainManagerTests.coffee

@@ -1,82 +0,0 @@
-RealTimeClient = require "./helpers/RealTimeClient"
-FixturesManager = require "./helpers/FixturesManager"
-
-expect = require("chai").expect
-
-async = require "async"
-request = require "request"
-
-Settings = require "settings-sharelatex"
-
-drain = (rate, callback) ->
-	request.post {
-		url: "http://localhost:3026/drain?rate=#{rate}"
-		auth: {
-			user: Settings.internal.realTime.user,
-			pass: Settings.internal.realTime.pass
-		}
-	}, (error, response, data) ->
-		callback error, data
-	return null
-
-describe "DrainManagerTests", ->
-	before (done) ->
-		FixturesManager.setUpProject {
-			privilegeLevel: "owner"
-			project: {
-				name: "Test Project"
-			}
-		}, (e, {@project_id, @user_id}) => done()
-		return null
-
-	before (done) ->
-		# cleanup to speedup reconnecting
-		@timeout(10000)
-		RealTimeClient.disconnectAllClients done
-
-	# trigger and check cleanup
-	it "should have disconnected all previous clients", (done) ->
-		RealTimeClient.getConnectedClients (error, data) ->
-			return done(error) if error
-			expect(data.length).to.equal(0)
-			done()
-
-	describe "with two clients in the project", ->
-		beforeEach (done) ->
-			async.series [
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientB = RealTimeClient.connect()
-					@clientB.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, cb
-
-				(cb) =>
-					@clientB.emit "joinProject", project_id: @project_id, cb
-			], done
-
-		describe "starting to drain", () ->
-			beforeEach (done) ->
-				async.parallel [
-					(cb) =>
-						@clientA.on "reconnectGracefully", cb
-					(cb) =>
-						@clientB.on "reconnectGracefully", cb
-
-					(cb) -> drain(2, cb)
-				], done
-
-			afterEach (done) ->
-				# reset drain
-				drain(0, done)
-
-			it "should not timeout", ->
-				expect(true).to.equal(true)
-
-			it "should not have disconnected", ->
-				expect(@clientA.socket.connected).to.equal true
-				expect(@clientB.socket.connected).to.equal true

services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee

@@ -1,160 +0,0 @@
-async = require "async"
-{expect} = require("chai")
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer"
-MockWebServer = require "./helpers/MockWebServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(settings.redis.pubsub)
-rclientRT = redis.createClient(settings.redis.realtime)
-KeysRT = settings.redis.realtime.key_schema
-
-describe "EarlyDisconnect", ->
-	before (done) ->
-		MockDocUpdaterServer.run done
-
-	describe "when the client disconnects before joinProject completes", ->
-		before () ->
-			# slow down web-api requests to force the race condition
-			@actualWebAPIjoinProject = joinProject = MockWebServer.joinProject
-			MockWebServer.joinProject = (project_id, user_id, cb) ->
-				setTimeout () ->
-					joinProject(project_id, user_id, cb)
-				, 300
-
-		after () ->
-			MockWebServer.joinProject = @actualWebAPIjoinProject
-
-		beforeEach (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (() ->)
-					# disconnect before joinProject completes
-					@clientA.on "disconnect", () -> cb()
-					@clientA.disconnect()
-
-				(cb) =>
-					# wait for joinDoc and subscribe
-					setTimeout cb, 500
-			], done
-
-		# we can force the race condition, there is no need to repeat too often
-		for attempt in Array.from(length: 5).map((_, i) -> i+1)
-			it "should not subscribe to the pub/sub channel anymore (race #{attempt})", (done) ->
-				rclient.pubsub 'CHANNELS', (err, resp) =>
-					return done(err) if err
-					expect(resp).to.not.include "editor-events:#{@project_id}"
-					done()
-				return null
-
-	describe "when the client disconnects before joinDoc completes", ->
-		beforeEach (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, (() ->)
-					# disconnect before joinDoc completes
-					@clientA.on "disconnect", () -> cb()
-					@clientA.disconnect()
-
-				(cb) =>
-					# wait for subscribe and unsubscribe
-					setTimeout cb, 100
-			], done
-
-		# we can not force the race condition, so we have to try many times
-		for attempt in Array.from(length: 20).map((_, i) -> i+1)
-			it "should not subscribe to the pub/sub channels anymore (race #{attempt})", (done) ->
-				rclient.pubsub 'CHANNELS', (err, resp) =>
-					return done(err) if err
-					expect(resp).to.not.include "editor-events:#{@project_id}"
-
-					rclient.pubsub 'CHANNELS', (err, resp) =>
-						return done(err) if err
-						expect(resp).to.not.include "applied-ops:#{@doc_id}"
-						done()
-				return null
-
-	describe "when the client disconnects before clientTracking.updatePosition starts", ->
-		beforeEach (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, cb
-
-				(cb) =>
-					@clientA.emit "clientTracking.updatePosition", {
-							row: 42
-							column: 36
-							doc_id: @doc_id
-						}, (() ->)
-					# disconnect before updateClientPosition completes
-					@clientA.on "disconnect", () -> cb()
-					@clientA.disconnect()
-
-				(cb) =>
-					# wait for updateClientPosition
-					setTimeout cb, 100
-			], done
-
-		# we can not force the race condition, so we have to try many times
-		for attempt in Array.from(length: 20).map((_, i) -> i+1)
-			it "should not show the client as connected (race #{attempt})", (done) ->
-				rclientRT.smembers KeysRT.clientsInProject({project_id: @project_id}), (err, results) ->
-					return done(err) if err
-					expect(results).to.deep.equal([])
-					done()
-				return null

services/real-time/test/acceptance/coffee/HttpControllerTests.coffee

@@ -1,68 +0,0 @@
-async = require('async')
-expect = require('chai').expect
-request = require('request').defaults({
-	baseUrl: 'http://localhost:3026'
-})
-
-RealTimeClient = require "./helpers/RealTimeClient"
-FixturesManager = require "./helpers/FixturesManager"
-
-describe 'HttpControllerTests', ->
-	describe 'without a user', ->
-		it 'should return 404 for the client view', (done) ->
-			client_id = 'not-existing'
-			request.get {
-				url: "/clients/#{client_id}"
-				json: true
-			}, (error, response, data) ->
-				return done(error) if error
-				expect(response.statusCode).to.equal(404)
-				done()
-
-	describe 'with a user and after joining a project', ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-					}, (error, {@project_id, @user_id}) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {}, (error, {@doc_id}) =>
-						cb(error)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-
-				(cb) =>
-					@client.emit "joinProject", {@project_id}, cb
-
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, cb
-			], done
-
-		it 'should send a client view', (done) ->
-			request.get {
-				url: "/clients/#{@client.socket.sessionid}"
-				json: true
-			}, (error, response, data) =>
-				return done(error) if error
-				expect(response.statusCode).to.equal(200)
-				expect(data.connected_time).to.exist
-				delete data.connected_time
-				# .email is not set in the session
-				delete data.email
-				expect(data).to.deep.equal({
-					client_id: @client.socket.sessionid,
-					first_name: 'Joe',
-					last_name: 'Bloggs',
-					project_id: @project_id,
-					user_id: @user_id,
-					rooms: [
-						@project_id,
-						@doc_id,
-					]
-				})
-				done()

services/real-time/test/acceptance/coffee/JoinDocTests.coffee

@@ -1,246 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-chai.should()
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-describe "joinDoc", ->
-	before ->
-		@lines = ["test", "doc", "lines"]
-		@version = 42
-		@ops = ["mock", "doc", "ops"]
-		@ranges = {"mock": "ranges"}
-			
-	describe "when authorised readAndWrite", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, -1)
-				.should.equal true
-		
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-			
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()
-		
-	describe "when authorised readOnly", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readOnly"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, -1)
-				.should.equal true
-		
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-			
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()
-		
-	describe "when authorised as owner", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, -1)
-				.should.equal true
-		
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-			
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()
-
-	# It is impossible to write an acceptance test to test joining an unauthorized
-	# project, since joinProject already catches that. If you can join a project,
-	# then you can join a doc in that project.
-			
-	describe "with a fromVersion", ->
-		before (done) ->
-			@fromVersion = 36
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, @fromVersion, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater with the fromVersion", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, @fromVersion)
-				.should.equal true
-		
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-			
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()
-
-	describe "with options", ->
-		before (done) ->
-			@options = { encodeRanges: true }
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, @options, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater with the default fromVersion", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, -1)
-				.should.equal true
-
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()
-
-	describe "with fromVersion and options", ->
-		before (done) ->
-			@fromVersion = 36
-			@options = { encodeRanges: true }
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, @fromVersion, @options, (error, @returnedArgs...) => cb(error)
-			], done
-
-		it "should get the doc from the doc updater with the fromVersion", ->
-			MockDocUpdaterServer.getDocument
-				.calledWith(@project_id, @doc_id, @fromVersion)
-				.should.equal true
-
-		it "should return the doc lines, version, ranges and ops", ->
-			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
-
-		it "should have joined the doc room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@doc_id in client.rooms).to.equal true
-				done()

services/real-time/test/acceptance/coffee/JoinProjectTests.coffee

@@ -1,108 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-chai.should()
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockWebServer = require "./helpers/MockWebServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-describe "joinProject", ->
-	describe "when authorized", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-			], done
-					
-		it "should get the project from web", ->
-			MockWebServer.joinProject
-				.calledWith(@project_id, @user_id)
-				.should.equal true
-					
-		it "should return the project", ->
-			@project.should.deep.equal {
-				name: "Test Project"
-			}
-		
-		it "should return the privilege level", ->
-			@privilegeLevel.should.equal "owner"
-		
-		it "should return the protocolVersion", ->
-			@protocolVersion.should.equal 2
-			
-		it "should have joined the project room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@project_id in client.rooms).to.equal true
-				done()
-				
-		it "should have marked the user as connected", (done) ->
-			@client.emit "clientTracking.getConnectedUsers", (error, users) =>
-				connected = false
-				for user in users
-					if user.client_id == @client.publicId and user.user_id == @user_id
-						connected = true
-						break
-				expect(connected).to.equal true
-				done()
-			
-	describe "when not authorized", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: null
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, (@error, @project, @privilegeLevel, @protocolVersion) =>
-						cb()
-			], done
-		
-		it "should return an error", ->
-			@error.message.should.equal "not authorized"
-			
-		it "should not have joined the project room", (done) ->
-			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-				expect(@project_id in client.rooms).to.equal false
-				done()
-
-	describe "when over rate limit", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-
-				(cb) =>
-					@client.emit "joinProject", project_id: 'rate-limited', (@error) =>
-						cb()
-			], done
-
-		it "should return a TooManyRequests error code", ->
-			@error.message.should.equal "rate-limit hit when joining project"
-			@error.code.should.equal "TooManyRequests"
-

services/real-time/test/acceptance/coffee/LeaveDocTests.coffee

@@ -1,86 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-chai.should()
-sinon = require("sinon")
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer"
-FixturesManager = require "./helpers/FixturesManager"
-logger = require("logger-sharelatex")
-
-async = require "async"
-
-describe "leaveDoc", ->
-	before ->
-		@lines = ["test", "doc", "lines"]
-		@version = 42
-		@ops = ["mock", "doc", "ops"]
-		sinon.spy(logger, "error")
-		sinon.spy(logger, "warn")
-		sinon.spy(logger, "log")
-		@other_doc_id = FixturesManager.getRandomId()
-	
-	after ->
-		logger.error.restore() # remove the spy
-		logger.warn.restore()
-		logger.log.restore()
-
-	describe "when joined to a doc", ->
-		beforeEach (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "readAndWrite"
-					}, (e, {@project_id, @user_id}) =>
-						cb(e)
-					
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-						
-				(cb) =>
-					@client = RealTimeClient.connect()
-					@client.on "connectionAccepted", cb
-						
-				(cb) =>
-					@client.emit "joinProject", project_id: @project_id, cb
-				
-				(cb) =>
-					@client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error)
-			], done
-							
-		describe "then leaving the doc", ->
-			beforeEach (done) ->
-				@client.emit "leaveDoc", @doc_id, (error) ->
-					throw error if error?
-					done()
-			
-			it "should have left the doc room", (done) ->
-				RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-					expect(@doc_id in client.rooms).to.equal false
-					done()
-
-		describe "when sending a leaveDoc request before the previous joinDoc request has completed", ->
-			beforeEach (done) ->
-				@client.emit "leaveDoc", @doc_id, () ->
-				@client.emit "joinDoc", @doc_id, () ->
-				@client.emit "leaveDoc", @doc_id, (error) ->
-					throw error if error?
-					done()
-
-			it "should not trigger an error", ->
-				sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen")
-
-			it "should have left the doc room", (done) ->
-				RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
-					expect(@doc_id in client.rooms).to.equal false
-					done()
-
-		describe "when sending a leaveDoc for a room the client has not joined ", ->
-			beforeEach (done) ->
-				@client.emit "leaveDoc", @other_doc_id, (error) ->
-					throw error if error?
-					done()
-
-			it "should trigger a low level message only", ->
-				sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in")

services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee

@@ -1,147 +0,0 @@
-RealTimeClient = require "./helpers/RealTimeClient"
-MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(settings.redis.pubsub)
-
-describe "leaveProject", ->
-	before (done) ->
-		MockDocUpdaterServer.run done
-
-	describe "with other clients in the project", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connectionAccepted", cb
-
-				(cb) =>
-					@clientB = RealTimeClient.connect()
-					@clientB.on "connectionAccepted", cb
-
-					@clientBDisconnectMessages = []
-					@clientB.on "clientTracking.clientDisconnected", (data) =>
-						@clientBDisconnectMessages.push data
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					@clientB.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, cb
-				(cb) =>
-					@clientB.emit "joinDoc", @doc_id, cb
-
-				(cb) =>
-					# leaveProject is called when the client disconnects
-					@clientA.on "disconnect", () -> cb()
-					@clientA.disconnect()
-
-				(cb) =>
-					# The API waits a little while before flushing changes
-					setTimeout done, 1000
-
-			], done
-
-		it "should emit a disconnect message to the room", ->
-			@clientBDisconnectMessages.should.deep.equal [@clientA.publicId]
-
-		it "should no longer list the client in connected users", (done) ->
-			@clientB.emit "clientTracking.getConnectedUsers", (error, users) =>
-				for user in users
-					if user.client_id == @clientA.publicId
-						throw "Expected clientA to not be listed in connected users"
-				return done()
-
-		it "should not flush the project to the document updater", ->
-			MockDocUpdaterServer.deleteProject
-				.calledWith(@project_id)
-				.should.equal false
-
-		it "should remain subscribed to the editor-events channels", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.include "editor-events:#{@project_id}"
-				done()
-			return null
-
-		it "should remain subscribed to the applied-ops channels", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.include "applied-ops:#{@doc_id}"
-				done()
-			return null
-
-	describe "with no other clients in the project", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connect", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, cb
-
-				(cb) =>
-					# leaveProject is called when the client disconnects
-					@clientA.on "disconnect", () -> cb()
-					@clientA.disconnect()
-
-				(cb) =>
-					# The API waits a little while before flushing changes
-					setTimeout done, 1000
-			], done
-
-		it "should flush the project to the document updater", ->
-			MockDocUpdaterServer.deleteProject
-				.calledWith(@project_id)
-				.should.equal true
-
-		it "should not subscribe to the editor-events channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "editor-events:#{@project_id}"
-				done()
-			return null
-
-		it "should not subscribe to the applied-ops channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "applied-ops:#{@doc_id}"
-				done()
-			return null

services/real-time/test/acceptance/coffee/PubSubRace.coffee

@@ -1,205 +0,0 @@
-RealTimeClient = require "./helpers/RealTimeClient"
-MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(settings.redis.pubsub)
-
-describe "PubSubRace", ->
-	before (done) ->
-		MockDocUpdaterServer.run done
-
-	describe "when the client leaves a doc before joinDoc completes", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connect", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, () ->
-					# leave before joinDoc completes
-					@clientA.emit "leaveDoc", @doc_id, cb
-
-				(cb) =>
-					# wait for subscribe and unsubscribe
-					setTimeout cb, 100
-			], done
-
-		it "should not subscribe to the applied-ops channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "applied-ops:#{@doc_id}"
-				done()
-			return null
-
-	describe "when the client emits joinDoc and leaveDoc requests frequently and leaves eventually", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connect", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, cb
-
-				(cb) =>
-					# wait for subscribe and unsubscribe
-					setTimeout cb, 100
-			], done
-
-		it "should not subscribe to the applied-ops channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "applied-ops:#{@doc_id}"
-				done()
-			return null
-
-	describe "when the client emits joinDoc and leaveDoc requests frequently and remains in the doc", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connect", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, () ->
-					@clientA.emit "leaveDoc", @doc_id, () ->
-					@clientA.emit "joinDoc", @doc_id, cb
-
-				(cb) =>
-					# wait for subscribe and unsubscribe
-					setTimeout cb, 100
-			], done
-
-		it "should subscribe to the applied-ops channels", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.include "applied-ops:#{@doc_id}"
-				done()
-			return null
-
-	describe "when the client disconnects before joinDoc completes", ->
-		before (done) ->
-			async.series [
-				(cb) =>
-					FixturesManager.setUpProject {
-						privilegeLevel: "owner"
-						project: {
-							name: "Test Project"
-						}
-					}, (e, {@project_id, @user_id}) => cb()
-
-				(cb) =>
-					@clientA = RealTimeClient.connect()
-					@clientA.on "connect", cb
-
-				(cb) =>
-					@clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) =>
-						cb(error)
-
-				(cb) =>
-					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-						cb(e)
-
-				(cb) =>
-					joinDocCompleted = false
-					@clientA.emit "joinDoc", @doc_id, () ->
-						joinDocCompleted = true
-					# leave before joinDoc completes
-					setTimeout () =>
-						if joinDocCompleted
-							return cb(new Error('joinDocCompleted -- lower timeout'))
-						@clientA.on "disconnect", () -> cb()
-						@clientA.disconnect()
-					# socket.io processes joinDoc and disconnect with different delays:
-					#  - joinDoc goes through two process.nextTick
-					#  - disconnect goes through one process.nextTick
-					# We have to inject the disconnect event into a different event loop
-					#  cycle.
-					, 3
-
-				(cb) =>
-					# wait for subscribe and unsubscribe
-					setTimeout cb, 100
-			], done
-
-		it "should not subscribe to the editor-events channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "editor-events:#{@project_id}"
-				done()
-			return null
-
-		it "should not subscribe to the applied-ops channels anymore", (done) ->
-			rclient.pubsub 'CHANNELS', (err, resp) =>
-				return done(err) if err
-				resp.should.not.include "applied-ops:#{@doc_id}"
-				done()
-			return null

services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee

@@ -1,208 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-chai.should()
-
-RealTimeClient = require "./helpers/RealTimeClient"
-MockWebServer = require "./helpers/MockWebServer"
-FixturesManager = require "./helpers/FixturesManager"
-
-async = require "async"
-
-settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(settings.redis.pubsub)
-
-describe "receiveUpdate", ->
-	beforeEach (done) ->
-		@lines = ["test", "doc", "lines"]
-		@version = 42
-		@ops = ["mock", "doc", "ops"]
-		
-		async.series [
-			(cb) =>
-				FixturesManager.setUpProject {
-					privilegeLevel: "owner"
-					project: { name: "Test Project"	}
-				}, (error, {@user_id, @project_id}) => cb()
-			
-			(cb) =>
-				FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) =>
-					cb(e)
-			
-			(cb) =>
-				@clientA = RealTimeClient.connect()
-				@clientA.on "connectionAccepted", cb
-				
-			(cb) =>
-				@clientB = RealTimeClient.connect()
-				@clientB.on "connectionAccepted", cb
-				
-			(cb) =>
-				@clientA.emit "joinProject", {
-					project_id: @project_id
-				}, cb
-			
-			(cb) =>
-				@clientA.emit "joinDoc", @doc_id, cb
-				
-			(cb) =>
-				@clientB.emit "joinProject", {
-					project_id: @project_id
-				}, cb
-			
-			(cb) =>
-				@clientB.emit "joinDoc", @doc_id, cb
-
-			(cb) =>
-				FixturesManager.setUpProject {
-					privilegeLevel: "owner"
-					project: {name: "Test Project"}
-				}, (error, {user_id: @user_id_second, project_id: @project_id_second}) => cb()
-
-			(cb) =>
-				FixturesManager.setUpDoc @project_id_second, {@lines, @version, @ops}, (e, {doc_id: @doc_id_second}) =>
-					cb(e)
-
-			(cb) =>
-				@clientC = RealTimeClient.connect()
-				@clientC.on "connectionAccepted", cb
-
-			(cb) =>
-				@clientC.emit "joinProject", {
-					project_id: @project_id_second
-				}, cb
-			(cb) =>
-				@clientC.emit "joinDoc", @doc_id_second, cb
-
-			(cb) =>
-				@clientAUpdates = []
-				@clientA.on "otUpdateApplied", (update) => @clientAUpdates.push(update)
-				@clientBUpdates = []
-				@clientB.on "otUpdateApplied", (update) => @clientBUpdates.push(update)
-				@clientCUpdates = []
-				@clientC.on "otUpdateApplied", (update) => @clientCUpdates.push(update)
-
-				@clientAErrors = []
-				@clientA.on "otUpdateError", (error) => @clientAErrors.push(error)
-				@clientBErrors = []
-				@clientB.on "otUpdateError", (error) => @clientBErrors.push(error)
-				@clientCErrors = []
-				@clientC.on "otUpdateError", (error) => @clientCErrors.push(error)
-				cb()
-		], done
-
-	afterEach () ->
-		@clientA?.disconnect()
-		@clientB?.disconnect()
-		@clientC?.disconnect()
-
-	describe "with an update from clientA", ->
-		beforeEach (done) ->
-			@update = {
-				doc_id: @doc_id
-				op:
-					meta:
-						source: @clientA.publicId
-					v: @version
-					doc: @doc_id
-					op: [{i: "foo", p: 50}]				
-			}
-			rclient.publish "applied-ops", JSON.stringify(@update)
-			setTimeout done, 200 # Give clients time to get message
-			
-		it "should send the full op to clientB", ->
-			@clientBUpdates.should.deep.equal [@update.op]
-			
-		it "should send an ack to clientA", ->
-			@clientAUpdates.should.deep.equal [{
-				v: @version, doc: @doc_id
-			}]
-
-		it "should send nothing to clientC", ->
-			@clientCUpdates.should.deep.equal []
-
-	describe "with an update from clientC", ->
-		beforeEach (done) ->
-			@update = {
-				doc_id: @doc_id_second
-				op:
-					meta:
-						source: @clientC.publicId
-					v: @version
-					doc: @doc_id_second
-					op: [{i: "update from clientC", p: 50}]
-			}
-			rclient.publish "applied-ops", JSON.stringify(@update)
-			setTimeout done, 200 # Give clients time to get message
-
-		it "should send nothing to clientA", ->
-			@clientAUpdates.should.deep.equal []
-
-		it "should send nothing to clientB", ->
-			@clientBUpdates.should.deep.equal []
-
-		it "should send an ack to clientC", ->
-			@clientCUpdates.should.deep.equal [{
-				v: @version, doc: @doc_id_second
-			}]
-
-	describe "with an update from a remote client for project 1", ->
-		beforeEach (done) ->
-			@update = {
-				doc_id: @doc_id
-				op:
-					meta:
-						source: 'this-is-a-remote-client-id'
-					v: @version
-					doc: @doc_id
-					op: [{i: "foo", p: 50}]
-			}
-			rclient.publish "applied-ops", JSON.stringify(@update)
-			setTimeout done, 200 # Give clients time to get message
-
-		it "should send the full op to clientA", ->
-			@clientAUpdates.should.deep.equal [@update.op]
-			
-		it "should send the full op to clientB", ->
-			@clientBUpdates.should.deep.equal [@update.op]
-
-		it "should send nothing to clientC", ->
-			@clientCUpdates.should.deep.equal []
-
-	describe "with an error for the first project", ->
-		beforeEach (done) ->
-			rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id, error: @error = "something went wrong"})
-			setTimeout done, 200 # Give clients time to get message
-
-		it "should send the error to the clients in the first project", ->
-			@clientAErrors.should.deep.equal [@error]
-			@clientBErrors.should.deep.equal [@error]
-
-		it "should not send any errors to the client in the second project", ->
-			@clientCErrors.should.deep.equal []
-
-		it "should disconnect the clients of the first project", ->
-			@clientA.socket.connected.should.equal false
-			@clientB.socket.connected.should.equal false
-
-		it "should not disconnect the client in the second project", ->
-			@clientC.socket.connected.should.equal true
-
-	describe "with an error for the second project", ->
-		beforeEach (done) ->
-			rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id_second, error: @error = "something went wrong"})
-			setTimeout done, 200 # Give clients time to get message
-
-		it "should not send any errors to the clients in the first project", ->
-			@clientAErrors.should.deep.equal []
-			@clientBErrors.should.deep.equal []
-
-		it "should send the error to the client in the second project", ->
-			@clientCErrors.should.deep.equal [@error]
-
-		it "should not disconnect the clients of the first project", ->
-			@clientA.socket.connected.should.equal true
-			@clientB.socket.connected.should.equal true
-
-		it "should disconnect the client in the second project", ->
-			@clientC.socket.connected.should.equal false

services/real-time/test/acceptance/coffee/RouterTests.coffee

@@ -1,76 +0,0 @@
-async = require "async"
-{expect} = require("chai")
-
-RealTimeClient = require "./helpers/RealTimeClient"
-FixturesManager = require "./helpers/FixturesManager"
-
-
-describe "Router", ->
-	describe "joinProject", ->
-		describe "when there is no callback provided", ->
-			after () ->
-				process.removeListener('unhandledRejection', @onUnhandled)
-
-			before (done) ->
-				@onUnhandled = (error) ->
-					done(error)
-				process.on('unhandledRejection', @onUnhandled)
-				async.series [
-					(cb) =>
-						FixturesManager.setUpProject {
-							privilegeLevel: "owner"
-							project: {
-								name: "Test Project"
-							}
-						}, (e, {@project_id, @user_id}) =>
-							cb(e)
-
-					(cb) =>
-						@client = RealTimeClient.connect()
-						@client.on "connectionAccepted", cb
-
-					(cb) =>
-						@client = RealTimeClient.connect()
-						@client.on "connectionAccepted", cb
-
-					(cb) =>
-						@client.emit "joinProject", project_id: @project_id
-						setTimeout(cb, 100)
-				], done
-
-			it "should keep on going", ->
-				expect('still running').to.exist
-
-		describe "when there are too many arguments", ->
-			after () ->
-				process.removeListener('unhandledRejection', @onUnhandled)
-
-			before (done) ->
-				@onUnhandled = (error) ->
-					done(error)
-				process.on('unhandledRejection', @onUnhandled)
-				async.series [
-					(cb) =>
-						FixturesManager.setUpProject {
-							privilegeLevel: "owner"
-							project: {
-								name: "Test Project"
-							}
-						}, (e, {@project_id, @user_id}) =>
-							cb(e)
-
-					(cb) =>
-						@client = RealTimeClient.connect()
-						@client.on "connectionAccepted", cb
-
-					(cb) =>
-						@client = RealTimeClient.connect()
-						@client.on "connectionAccepted", cb
-
-					(cb) =>
-						@client.emit "joinProject", 1, 2, 3, 4, 5, (@error) =>
-							cb()
-				], done
-
-			it "should return an error message", ->
-				expect(@error.message).to.equal('unexpected arguments')

services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee

@@ -1,67 +0,0 @@
-RealTimeClient = require("./helpers/RealTimeClient")
-Settings = require("settings-sharelatex")
-{expect} = require('chai')
-
-describe 'SessionSockets', ->
-	before ->
-		@checkSocket = (fn) ->
-			client = RealTimeClient.connect()
-			client.on 'connectionAccepted', fn
-			client.on 'connectionRejected', fn
-			return null
-
-	describe 'without cookies', ->
-		before ->
-			RealTimeClient.cookie = null
-
-		it 'should return a lookup error', (done) ->
-			@checkSocket (error) ->
-				expect(error).to.exist
-				expect(error.message).to.equal('invalid session')
-				done()
-
-	describe 'with a different cookie', ->
-		before ->
-			RealTimeClient.cookie = "some.key=someValue"
-
-		it 'should return a lookup error', (done) ->
-			@checkSocket (error) ->
-				expect(error).to.exist
-				expect(error.message).to.equal('invalid session')
-				done()
-
-	describe 'with an invalid cookie', ->
-		before (done) ->
-			RealTimeClient.setSession {}, (error) ->
-				return done(error) if error
-				RealTimeClient.cookie = "#{Settings.cookieName}=#{
-					RealTimeClient.cookie.slice(17, 49)
-				}"
-				done()
-			return null
-
-		it 'should return a lookup error', (done) ->
-			@checkSocket (error) ->
-				expect(error).to.exist
-				expect(error.message).to.equal('invalid session')
-				done()
-
-	describe 'with a valid cookie and no matching session', ->
-		before ->
-			RealTimeClient.cookie = "#{Settings.cookieName}=unknownId"
-
-		it 'should return a lookup error', (done) ->
-			@checkSocket (error) ->
-				expect(error).to.exist
-				expect(error.message).to.equal('invalid session')
-				done()
-
-	describe 'with a valid cookie and a matching session', ->
-		before (done) ->
-			RealTimeClient.setSession({}, done)
-			return null
-
-		it 'should not return an error', (done) ->
-			@checkSocket (error) ->
-				expect(error).to.not.exist
-				done()

services/real-time/test/acceptance/coffee/SessionTests.coffee

@@ -1,35 +0,0 @@
-chai = require("chai")
-expect = chai.expect
-
-RealTimeClient = require "./helpers/RealTimeClient"
-
-describe "Session", ->
-	describe "with an established session", ->
-		before (done) ->
-			@user_id = "mock-user-id"
-			RealTimeClient.setSession {
-				user: { _id: @user_id }
-			}, (error) =>
-				throw error if error?
-				@client = RealTimeClient.connect()
-				return done()
-			return null
-
-		it "should not get disconnected", (done) ->
-			disconnected = false
-			@client.on "disconnect", () ->
-				disconnected = true
-			setTimeout () =>
-				expect(disconnected).to.equal false
-				done()
-			, 500
-			
-		it "should appear in the list of connected clients", (done) ->
-			RealTimeClient.getConnectedClients (error, clients) =>
-				included = false
-				for client in clients
-					if client.client_id == @client.socket.sessionid
-						included = true
-						break
-				expect(included).to.equal true
-				done()

services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee

@@ -1,48 +0,0 @@
-RealTimeClient = require "./RealTimeClient"
-MockWebServer = require "./MockWebServer"
-MockDocUpdaterServer = require "./MockDocUpdaterServer"
-
-module.exports = FixturesManager =
-	setUpProject: (options = {}, callback = (error, data) ->) ->
-		options.user_id        ||= FixturesManager.getRandomId()
-		options.project_id     ||= FixturesManager.getRandomId()
-		options.project        ||= { name: "Test Project" }
-		{project_id, user_id, privilegeLevel, project, publicAccess} = options
-		
-		privileges = {}
-		privileges[user_id] = privilegeLevel
-		if publicAccess
-			privileges["anonymous-user"] = publicAccess
-		
-		MockWebServer.createMockProject(project_id, privileges, project)
-		MockWebServer.run (error) =>
-			throw error if error?
-			RealTimeClient.setSession {
-				user: {
-					_id: user_id
-					first_name: "Joe"
-					last_name: "Bloggs"
-				}
-			}, (error) =>
-				throw error if error?
-				callback null, {project_id, user_id, privilegeLevel, project}
-				
-	setUpDoc: (project_id, options = {}, callback = (error, data) ->) ->
-		options.doc_id  ||= FixturesManager.getRandomId()
-		options.lines   ||= ["doc", "lines"]
-		options.version ||= 42
-		options.ops     ||= ["mock", "ops"]
-		{doc_id, lines, version, ops, ranges} = options
-				
-		MockDocUpdaterServer.createMockDoc project_id, doc_id, {lines, version, ops, ranges}
-		MockDocUpdaterServer.run (error) =>
-			throw error if error?
-			callback null, {project_id, doc_id, lines, version, ops}
-		
-	getRandomId: () ->
-		return require("crypto")
-			.createHash("sha1")
-			.update(Math.random().toString())
-			.digest("hex")
-			.slice(0,24)
-		

services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee

@@ -1,46 +0,0 @@
-sinon = require "sinon"
-express = require "express"
-
-module.exports = MockDocUpdaterServer =
-	docs: {}
-	
-	createMockDoc: (project_id, doc_id, data) ->
-		MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] = data
-		
-	getDocument: (project_id, doc_id, fromVersion, callback = (error, data) ->) ->
-		callback(
-			null, MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"]
-		)
-		
-	deleteProject: sinon.stub().callsArg(1)
-		
-	getDocumentRequest: (req, res, next) ->
-		{project_id, doc_id} = req.params
-		{fromVersion} = req.query
-		fromVersion = parseInt(fromVersion, 10)
-		MockDocUpdaterServer.getDocument project_id, doc_id, fromVersion, (error, data) ->
-			return next(error) if error?
-			res.json data
-			
-	deleteProjectRequest: (req, res, next) ->
-		{project_id} = req.params
-		MockDocUpdaterServer.deleteProject project_id, (error) ->
-			return next(error) if error?
-			res.sendStatus 204
-	
-	running: false
-	run: (callback = (error) ->) ->
-		if MockDocUpdaterServer.running
-			return callback()
-		app = express()
-		app.get "/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest
-		app.delete "/project/:project_id", MockDocUpdaterServer.deleteProjectRequest
-		app.listen 3003, (error) ->
-			MockDocUpdaterServer.running = true
-			callback(error)
-		.on "error", (error) ->
-			console.error "error starting MockDocUpdaterServer:", error.message
-			process.exit(1)
-
-			
-sinon.spy MockDocUpdaterServer, "getDocument"

services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee

@@ -1,46 +0,0 @@
-sinon = require "sinon"
-express = require "express"
-
-module.exports = MockWebServer =
-	projects: {}
-	privileges: {}
-	
-	createMockProject: (project_id, privileges, project) ->
-		MockWebServer.privileges[project_id] = privileges
-		MockWebServer.projects[project_id] = project
-		
-	joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) ->
-		callback(
-			null,
-			MockWebServer.projects[project_id], 
-			MockWebServer.privileges[project_id][user_id]
-		)
-		
-	joinProjectRequest: (req, res, next) ->
-		{project_id} = req.params
-		{user_id} = req.query
-		if project_id == 'rate-limited'
-			res.status(429).send()
-		else
-			MockWebServer.joinProject project_id, user_id, (error, project, privilegeLevel) ->
-				return next(error) if error?
-				res.json {
-					project: project
-					privilegeLevel: privilegeLevel
-				}
-	
-	running: false
-	run: (callback = (error) ->) ->
-		if MockWebServer.running
-			return callback()
-		app = express()
-		app.post "/project/:project_id/join", MockWebServer.joinProjectRequest
-		app.listen 3000, (error) ->
-			MockWebServer.running = true
-			callback(error)
-		.on "error", (error) ->
-			console.error "error starting MockWebServer:", error.message
-			process.exit(1)
-
-			
-sinon.spy MockWebServer, "joinProject"

services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee

@@ -1,75 +0,0 @@
-XMLHttpRequest = require("../../libs/XMLHttpRequest").XMLHttpRequest
-io = require("socket.io-client")
-async = require("async")
-
-request = require "request"
-Settings = require "settings-sharelatex"
-redis = require "redis-sharelatex"
-rclient = redis.createClient(Settings.redis.websessions)
-
-uid = require('uid-safe').sync
-signature = require("cookie-signature")
-
-io.util.request = () ->
-	xhr = new XMLHttpRequest()
-	_open = xhr.open
-	xhr.open = () =>
-		_open.apply(xhr, arguments)
-		if Client.cookie?
-			xhr.setRequestHeader("Cookie", Client.cookie)
-	return xhr
-
-module.exports = Client =
-	cookie: null
-
-	setSession: (session, callback = (error) ->) ->
-		sessionId = uid(24)
-		session.cookie = {}
-		rclient.set "sess:" + sessionId, JSON.stringify(session), (error) ->
-			return callback(error) if error?
-			secret = Settings.security.sessionSecret
-			cookieKey = 's:' + signature.sign(sessionId, secret)
-			Client.cookie = "#{Settings.cookieName}=#{cookieKey}"
-			callback()
-			
-	unsetSession: (callback = (error) ->) ->
-		Client.cookie = null
-		callback()
-			
-	connect: (cookie) ->
-		client = io.connect("http://localhost:3026", 'force new connection': true)
-		client.on 'connectionAccepted', (_, publicId) ->
-			client.publicId = publicId
-		return client
-		
-	getConnectedClients: (callback = (error, clients) ->) ->
-		request.get {
-			url: "http://localhost:3026/clients"
-			json: true
-		}, (error, response, data) ->
-			callback error, data
-		
-	getConnectedClient: (client_id, callback = (error, clients) ->) ->
-		request.get {
-			url: "http://localhost:3026/clients/#{client_id}"
-			json: true
-		}, (error, response, data) ->
-			callback error, data
-
-
-	disconnectClient: (client_id, callback) ->
-		request.post {
-			url: "http://localhost:3026/client/#{client_id}/disconnect"
-			auth: {
-				user: Settings.internal.realTime.user,
-				pass: Settings.internal.realTime.pass
-			}
-		}, (error, response, data) ->
-			callback error, data
-		return null
-
-	disconnectAllClients: (callback) ->
-		Client.getConnectedClients (error, clients) ->
-			async.each clients, (clientView, cb) ->
-				Client.disconnectClient clientView.client_id, cb
-			, callback

services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee

@@ -1,23 +0,0 @@
-app = require('../../../../app')
-logger = require("logger-sharelatex")
-Settings = require("settings-sharelatex")
-
-module.exports =
-	running: false
-	initing: false
-	callbacks: []
-	ensureRunning: (callback = (error) ->) ->
-		if @running
-			return callback()
-		else if @initing
-			@callbacks.push callback
-		else
-			@initing = true
-			@callbacks.push callback
-			app.listen Settings.internal?.realtime?.port, "localhost", (error) => 
-				throw error if error?
-				@running = true
-				logger.log("clsi running in dev mode")
-
-				for callback in @callbacks
-					callback()

services/real-time/test/acceptance/js/ApplyUpdateTests.js

@@ -0,0 +1,447 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS201: Simplify complex destructure assignments
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const async = require('async')
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(settings.redis.documentupdater)
+
+const redisSettings = settings.redis
+
+describe('applyOtUpdate', function () {
+  before(function () {
+    return (this.update = {
+      op: [{ i: 'foo', p: 42 }]
+    })
+  })
+  describe('when authorized', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'applyOtUpdate',
+              this.doc_id,
+              this.update,
+              cb
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should push the doc into the pending updates list', function (done) {
+      rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => {
+        const [doc_id] = Array.from(rest[0])
+        doc_id.should.equal(`${this.project_id}:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+
+    it('should push the update into redis', function (done) {
+      rclient.lrange(
+        redisSettings.documentupdater.key_schema.pendingUpdates({
+          doc_id: this.doc_id
+        }),
+        0,
+        -1,
+        (error, ...rest) => {
+          let [update] = Array.from(rest[0])
+          update = JSON.parse(update)
+          update.op.should.deep.equal(this.update.op)
+          update.meta.should.deep.equal({
+            source: this.client.publicId,
+            user_id: this.user_id
+          })
+          return done()
+        }
+      )
+      return null
+    })
+
+    return after(function (done) {
+      return async.series(
+        [
+          (cb) => rclient.del('pending-updates-list', cb),
+          (cb) =>
+            rclient.del(
+              'DocsWithPendingUpdates',
+              `${this.project_id}:${this.doc_id}`,
+              cb
+            ),
+          (cb) =>
+            rclient.del(
+              redisSettings.documentupdater.key_schema.pendingUpdates(
+                this.doc_id
+              ),
+              cb
+            )
+        ],
+        done
+      )
+    })
+  })
+
+  describe('when authorized with a huge edit update', function () {
+    before(function (done) {
+      this.update = {
+        op: {
+          p: 12,
+          t: 'update is too large'.repeat(1024 * 400) // >7MB
+        }
+      }
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            this.client.on('connectionAccepted', cb)
+            return this.client.on('otUpdateError', (otUpdateError) => {
+              this.otUpdateError = otUpdateError
+            })
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'applyOtUpdate',
+              this.doc_id,
+              this.update,
+              (error) => {
+                this.error = error
+                return cb()
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should not return an error', function () {
+      return expect(this.error).to.not.exist
+    })
+
+    it('should send an otUpdateError to the client', function (done) {
+      return setTimeout(() => {
+        expect(this.otUpdateError).to.exist
+        return done()
+      }, 300)
+    })
+
+    it('should disconnect the client', function (done) {
+      return setTimeout(() => {
+        this.client.socket.connected.should.equal(false)
+        return done()
+      }, 300)
+    })
+
+    return it('should not put the update in redis', function (done) {
+      rclient.llen(
+        redisSettings.documentupdater.key_schema.pendingUpdates({
+          doc_id: this.doc_id
+        }),
+        (error, len) => {
+          len.should.equal(0)
+          return done()
+        }
+      )
+      return null
+    })
+  })
+
+  describe('when authorized to read-only with an edit update', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readOnly'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'applyOtUpdate',
+              this.doc_id,
+              this.update,
+              (error) => {
+                this.error = error
+                return cb()
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should return an error', function () {
+      return expect(this.error).to.exist
+    })
+
+    it('should disconnect the client', function (done) {
+      return setTimeout(() => {
+        this.client.socket.connected.should.equal(false)
+        return done()
+      }, 300)
+    })
+
+    return it('should not put the update in redis', function (done) {
+      rclient.llen(
+        redisSettings.documentupdater.key_schema.pendingUpdates({
+          doc_id: this.doc_id
+        }),
+        (error, len) => {
+          len.should.equal(0)
+          return done()
+        }
+      )
+      return null
+    })
+  })
+
+  return describe('when authorized to read-only with a comment update', function () {
+    before(function (done) {
+      this.comment_update = {
+        op: [{ c: 'foo', p: 42 }]
+      }
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readOnly'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'applyOtUpdate',
+              this.doc_id,
+              this.comment_update,
+              cb
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should push the doc into the pending updates list', function (done) {
+      rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => {
+        const [doc_id] = Array.from(rest[0])
+        doc_id.should.equal(`${this.project_id}:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+
+    it('should push the update into redis', function (done) {
+      rclient.lrange(
+        redisSettings.documentupdater.key_schema.pendingUpdates({
+          doc_id: this.doc_id
+        }),
+        0,
+        -1,
+        (error, ...rest) => {
+          let [update] = Array.from(rest[0])
+          update = JSON.parse(update)
+          update.op.should.deep.equal(this.comment_update.op)
+          update.meta.should.deep.equal({
+            source: this.client.publicId,
+            user_id: this.user_id
+          })
+          return done()
+        }
+      )
+      return null
+    })
+
+    return after(function (done) {
+      return async.series(
+        [
+          (cb) => rclient.del('pending-updates-list', cb),
+          (cb) =>
+            rclient.del(
+              'DocsWithPendingUpdates',
+              `${this.project_id}:${this.doc_id}`,
+              cb
+            ),
+          (cb) =>
+            rclient.del(
+              redisSettings.documentupdater.key_schema.pendingUpdates({
+                doc_id: this.doc_id
+              }),
+              cb
+            )
+        ],
+        done
+      )
+    })
+  })
+})

services/real-time/test/acceptance/js/ClientTrackingTests.js

@@ -0,0 +1,255 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockWebServer = require('./helpers/MockWebServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+describe('clientTracking', function () {
+  describe('when a client updates its cursor location', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: { name: 'Test Project' }
+              },
+              (error, { user_id, project_id }) => {
+                this.user_id = user_id
+                this.project_id = project_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            this.clientB = RealTimeClient.connect()
+            return this.clientB.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              {
+                project_id: this.project_id
+              },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.clientA.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            return this.clientB.emit(
+              'joinProject',
+              {
+                project_id: this.project_id
+              },
+              cb
+            )
+          },
+
+          (cb) => {
+            this.updates = []
+            this.clientB.on('clientTracking.clientUpdated', (data) => {
+              return this.updates.push(data)
+            })
+
+            return this.clientA.emit(
+              'clientTracking.updatePosition',
+              {
+                row: (this.row = 42),
+                column: (this.column = 36),
+                doc_id: this.doc_id
+              },
+              (error) => {
+                if (error != null) {
+                  throw error
+                }
+                return setTimeout(cb, 300)
+              }
+            )
+          } // Give the message a chance to reach client B.
+        ],
+        done
+      )
+    })
+
+    it('should tell other clients about the update', function () {
+      return this.updates.should.deep.equal([
+        {
+          row: this.row,
+          column: this.column,
+          doc_id: this.doc_id,
+          id: this.clientA.publicId,
+          user_id: this.user_id,
+          name: 'Joe Bloggs'
+        }
+      ])
+    })
+
+    return it('should record the update in getConnectedUsers', function (done) {
+      return this.clientB.emit(
+        'clientTracking.getConnectedUsers',
+        (error, users) => {
+          for (const user of Array.from(users)) {
+            if (user.client_id === this.clientA.publicId) {
+              expect(user.cursorData).to.deep.equal({
+                row: this.row,
+                column: this.column,
+                doc_id: this.doc_id
+              })
+              return done()
+            }
+          }
+          throw new Error('user was never found')
+        }
+      )
+    })
+  })
+
+  return describe('when an anonymous client updates its cursor location', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: { name: 'Test Project' },
+                publicAccess: 'readAndWrite'
+              },
+              (error, { user_id, project_id }) => {
+                this.user_id = user_id
+                this.project_id = project_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              {
+                project_id: this.project_id
+              },
+              cb
+            )
+          },
+
+          (cb) => {
+            return RealTimeClient.setSession({}, cb)
+          },
+
+          (cb) => {
+            this.anonymous = RealTimeClient.connect()
+            return this.anonymous.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.anonymous.emit(
+              'joinProject',
+              {
+                project_id: this.project_id
+              },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.anonymous.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            this.updates = []
+            this.clientA.on('clientTracking.clientUpdated', (data) => {
+              return this.updates.push(data)
+            })
+
+            return this.anonymous.emit(
+              'clientTracking.updatePosition',
+              {
+                row: (this.row = 42),
+                column: (this.column = 36),
+                doc_id: this.doc_id
+              },
+              (error) => {
+                if (error != null) {
+                  throw error
+                }
+                return setTimeout(cb, 300)
+              }
+            )
+          } // Give the message a chance to reach client B.
+        ],
+        done
+      )
+    })
+
+    return it('should tell other clients about the update', function () {
+      return this.updates.should.deep.equal([
+        {
+          row: this.row,
+          column: this.column,
+          doc_id: this.doc_id,
+          id: this.anonymous.publicId,
+          user_id: 'anonymous-user',
+          name: ''
+        }
+      ])
+    })
+  })
+})

services/real-time/test/acceptance/js/DrainManagerTests.js

@@ -0,0 +1,135 @@
+/* eslint-disable
+    camelcase,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const RealTimeClient = require('./helpers/RealTimeClient')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const { expect } = require('chai')
+
+const async = require('async')
+const request = require('request')
+
+const Settings = require('settings-sharelatex')
+
+const drain = function (rate, callback) {
+  request.post(
+    {
+      url: `http://localhost:3026/drain?rate=${rate}`,
+      auth: {
+        user: Settings.internal.realTime.user,
+        pass: Settings.internal.realTime.pass
+      }
+    },
+    (error, response, data) => callback(error, data)
+  )
+  return null
+}
+
+describe('DrainManagerTests', function () {
+  before(function (done) {
+    FixturesManager.setUpProject(
+      {
+        privilegeLevel: 'owner',
+        project: {
+          name: 'Test Project'
+        }
+      },
+      (e, { project_id, user_id }) => {
+        this.project_id = project_id
+        this.user_id = user_id
+        return done()
+      }
+    )
+    return null
+  })
+
+  before(function (done) {
+    // cleanup to speedup reconnecting
+    this.timeout(10000)
+    return RealTimeClient.disconnectAllClients(done)
+  })
+
+  // trigger and check cleanup
+  it('should have disconnected all previous clients', function (done) {
+    return RealTimeClient.getConnectedClients((error, data) => {
+      if (error) {
+        return done(error)
+      }
+      expect(data.length).to.equal(0)
+      return done()
+    })
+  })
+
+  return describe('with two clients in the project', function () {
+    beforeEach(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            this.clientB = RealTimeClient.connect()
+            return this.clientB.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.clientB.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    return describe('starting to drain', function () {
+      beforeEach(function (done) {
+        return async.parallel(
+          [
+            (cb) => {
+              return this.clientA.on('reconnectGracefully', cb)
+            },
+            (cb) => {
+              return this.clientB.on('reconnectGracefully', cb)
+            },
+
+            (cb) => drain(2, cb)
+          ],
+          done
+        )
+      })
+
+      afterEach(function (done) {
+        return drain(0, done)
+      }) // reset drain
+
+      it('should not timeout', function () {
+        return expect(true).to.equal(true)
+      })
+
+      return it('should not have disconnected', function () {
+        expect(this.clientA.socket.connected).to.equal(true)
+        return expect(this.clientB.socket.connected).to.equal(true)
+      })
+    })
+  })
+})

services/real-time/test/acceptance/js/EarlyDisconnect.js

@@ -0,0 +1,288 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const async = require('async')
+const { expect } = require('chai')
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer')
+const MockWebServer = require('./helpers/MockWebServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(settings.redis.pubsub)
+const rclientRT = redis.createClient(settings.redis.realtime)
+const KeysRT = settings.redis.realtime.key_schema
+
+describe('EarlyDisconnect', function () {
+  before(function (done) {
+    return MockDocUpdaterServer.run(done)
+  })
+
+  describe('when the client disconnects before joinProject completes', function () {
+    before(function () {
+      // slow down web-api requests to force the race condition
+      let joinProject
+      this.actualWebAPIjoinProject = joinProject = MockWebServer.joinProject
+      return (MockWebServer.joinProject = (project_id, user_id, cb) =>
+        setTimeout(() => joinProject(project_id, user_id, cb), 300))
+    })
+
+    after(function () {
+      return (MockWebServer.joinProject = this.actualWebAPIjoinProject)
+    })
+
+    beforeEach(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              () => {}
+            )
+            // disconnect before joinProject completes
+            this.clientA.on('disconnect', () => cb())
+            return this.clientA.disconnect()
+          },
+
+          (cb) => {
+            // wait for joinDoc and subscribe
+            return setTimeout(cb, 500)
+          }
+        ],
+        done
+      )
+    })
+
+    // we can force the race condition, there is no need to repeat too often
+    return Array.from(Array.from({ length: 5 }).map((_, i) => i + 1)).map(
+      (attempt) =>
+        it(`should not subscribe to the pub/sub channel anymore (race ${attempt})`, function (done) {
+          rclient.pubsub('CHANNELS', (err, resp) => {
+            if (err) {
+              return done(err)
+            }
+            expect(resp).to.not.include(`editor-events:${this.project_id}`)
+            return done()
+          })
+          return null
+        })
+    )
+  })
+
+  describe('when the client disconnects before joinDoc completes', function () {
+    beforeEach(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            // disconnect before joinDoc completes
+            this.clientA.on('disconnect', () => cb())
+            return this.clientA.disconnect()
+          },
+
+          (cb) => {
+            // wait for subscribe and unsubscribe
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    // we can not force the race condition, so we have to try many times
+    return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map(
+      (attempt) =>
+        it(`should not subscribe to the pub/sub channels anymore (race ${attempt})`, function (done) {
+          rclient.pubsub('CHANNELS', (err, resp) => {
+            if (err) {
+              return done(err)
+            }
+            expect(resp).to.not.include(`editor-events:${this.project_id}`)
+
+            return rclient.pubsub('CHANNELS', (err, resp) => {
+              if (err) {
+                return done(err)
+              }
+              expect(resp).to.not.include(`applied-ops:${this.doc_id}`)
+              return done()
+            })
+          })
+          return null
+        })
+    )
+  })
+
+  return describe('when the client disconnects before clientTracking.updatePosition starts', function () {
+    beforeEach(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return this.clientA.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            this.clientA.emit(
+              'clientTracking.updatePosition',
+              {
+                row: 42,
+                column: 36,
+                doc_id: this.doc_id
+              },
+              () => {}
+            )
+            // disconnect before updateClientPosition completes
+            this.clientA.on('disconnect', () => cb())
+            return this.clientA.disconnect()
+          },
+
+          (cb) => {
+            // wait for updateClientPosition
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    // we can not force the race condition, so we have to try many times
+    return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map(
+      (attempt) =>
+        it(`should not show the client as connected (race ${attempt})`, function (done) {
+          rclientRT.smembers(
+            KeysRT.clientsInProject({ project_id: this.project_id }),
+            (err, results) => {
+              if (err) {
+                return done(err)
+              }
+              expect(results).to.deep.equal([])
+              return done()
+            }
+          )
+          return null
+        })
+    )
+  })
+})

services/real-time/test/acceptance/js/HttpControllerTests.js

@@ -0,0 +1,117 @@
+/* eslint-disable
+    camelcase,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const async = require('async')
+const { expect } = require('chai')
+const request = require('request').defaults({
+  baseUrl: 'http://localhost:3026'
+})
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const FixturesManager = require('./helpers/FixturesManager')
+
+describe('HttpControllerTests', function () {
+  describe('without a user', function () {
+    return it('should return 404 for the client view', function (done) {
+      const client_id = 'not-existing'
+      return request.get(
+        {
+          url: `/clients/${client_id}`,
+          json: true
+        },
+        (error, response, data) => {
+          if (error) {
+            return done(error)
+          }
+          expect(response.statusCode).to.equal(404)
+          return done()
+        }
+      )
+    })
+  })
+
+  return describe('with a user and after joining a project', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner'
+              },
+              (error, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {},
+              (error, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit('joinDoc', this.doc_id, cb)
+          }
+        ],
+        done
+      )
+    })
+
+    return it('should send a client view', function (done) {
+      return request.get(
+        {
+          url: `/clients/${this.client.socket.sessionid}`,
+          json: true
+        },
+        (error, response, data) => {
+          if (error) {
+            return done(error)
+          }
+          expect(response.statusCode).to.equal(200)
+          expect(data.connected_time).to.exist
+          delete data.connected_time
+          // .email is not set in the session
+          delete data.email
+          expect(data).to.deep.equal({
+            client_id: this.client.socket.sessionid,
+            first_name: 'Joe',
+            last_name: 'Bloggs',
+            project_id: this.project_id,
+            user_id: this.user_id,
+            rooms: [this.project_id, this.doc_id]
+          })
+          return done()
+        }
+      )
+    })
+  })
+})

services/real-time/test/acceptance/js/JoinDocTests.js

@@ -0,0 +1,565 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+describe('joinDoc', function () {
+  before(function () {
+    this.lines = ['test', 'doc', 'lines']
+    this.version = 42
+    this.ops = ['mock', 'doc', 'ops']
+    return (this.ranges = { mock: 'ranges' })
+  })
+
+  describe('when authorised readAndWrite', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, -1)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when authorised readOnly', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readOnly'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, -1)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when authorised as owner', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, -1)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  // It is impossible to write an acceptance test to test joining an unauthorized
+  // project, since joinProject already catches that. If you can join a project,
+  // then you can join a doc in that project.
+
+  describe('with a fromVersion', function () {
+    before(function (done) {
+      this.fromVersion = 36
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              this.fromVersion,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater with the fromVersion', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, this.fromVersion)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('with options', function () {
+    before(function (done) {
+      this.options = { encodeRanges: true }
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              this.options,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater with the default fromVersion', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, -1)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  return describe('with fromVersion and options', function () {
+    before(function (done) {
+      this.fromVersion = 36
+      this.options = { encodeRanges: true }
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              {
+                lines: this.lines,
+                version: this.version,
+                ops: this.ops,
+                ranges: this.ranges
+              },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              this.fromVersion,
+              this.options,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the doc from the doc updater with the fromVersion', function () {
+      return MockDocUpdaterServer.getDocument
+        .calledWith(this.project_id, this.doc_id, this.fromVersion)
+        .should.equal(true)
+    })
+
+    it('should return the doc lines, version, ranges and ops', function () {
+      return this.returnedArgs.should.deep.equal([
+        this.lines,
+        this.version,
+        this.ops,
+        this.ranges
+      ])
+    })
+
+    return it('should have joined the doc room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+})

services/real-time/test/acceptance/js/JoinProjectTests.js

@@ -0,0 +1,208 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockWebServer = require('./helpers/MockWebServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+describe('joinProject', function () {
+  describe('when authorized', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should get the project from web', function () {
+      return MockWebServer.joinProject
+        .calledWith(this.project_id, this.user_id)
+        .should.equal(true)
+    })
+
+    it('should return the project', function () {
+      return this.project.should.deep.equal({
+        name: 'Test Project'
+      })
+    })
+
+    it('should return the privilege level', function () {
+      return this.privilegeLevel.should.equal('owner')
+    })
+
+    it('should return the protocolVersion', function () {
+      return this.protocolVersion.should.equal(2)
+    })
+
+    it('should have joined the project room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.project_id)).to.equal(
+            true
+          )
+          return done()
+        }
+      )
+    })
+
+    return it('should have marked the user as connected', function (done) {
+      return this.client.emit(
+        'clientTracking.getConnectedUsers',
+        (error, users) => {
+          let connected = false
+          for (const user of Array.from(users)) {
+            if (
+              user.client_id === this.client.publicId &&
+              user.user_id === this.user_id
+            ) {
+              connected = true
+              break
+            }
+          }
+          expect(connected).to.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+
+  describe('when not authorized', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: null,
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.error = error
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb()
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    it('should return an error', function () {
+      return this.error.message.should.equal('not authorized')
+    })
+
+    return it('should not have joined the project room', function (done) {
+      return RealTimeClient.getConnectedClient(
+        this.client.socket.sessionid,
+        (error, client) => {
+          expect(Array.from(client.rooms).includes(this.project_id)).to.equal(
+            false
+          )
+          return done()
+        }
+      )
+    })
+  })
+
+  return describe('when over rate limit', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: 'rate-limited' },
+              (error) => {
+                this.error = error
+                return cb()
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    return it('should return a TooManyRequests error code', function () {
+      this.error.message.should.equal('rate-limit hit when joining project')
+      return this.error.code.should.equal('TooManyRequests')
+    })
+  })
+})

services/real-time/test/acceptance/js/LeaveDocTests.js

@@ -0,0 +1,176 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+const sinon = require('sinon')
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer')
+const FixturesManager = require('./helpers/FixturesManager')
+const logger = require('logger-sharelatex')
+
+const async = require('async')
+
+describe('leaveDoc', function () {
+  before(function () {
+    this.lines = ['test', 'doc', 'lines']
+    this.version = 42
+    this.ops = ['mock', 'doc', 'ops']
+    sinon.spy(logger, 'error')
+    sinon.spy(logger, 'warn')
+    sinon.spy(logger, 'log')
+    return (this.other_doc_id = FixturesManager.getRandomId())
+  })
+
+  after(function () {
+    logger.error.restore() // remove the spy
+    logger.warn.restore()
+    return logger.log.restore()
+  })
+
+  return describe('when joined to a doc', function () {
+    beforeEach(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'readAndWrite'
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.client = RealTimeClient.connect()
+            return this.client.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              cb
+            )
+          },
+
+          (cb) => {
+            return this.client.emit(
+              'joinDoc',
+              this.doc_id,
+              (error, ...rest) => {
+                ;[...this.returnedArgs] = Array.from(rest)
+                return cb(error)
+              }
+            )
+          }
+        ],
+        done
+      )
+    })
+
+    describe('then leaving the doc', function () {
+      beforeEach(function (done) {
+        return this.client.emit('leaveDoc', this.doc_id, (error) => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        })
+      })
+
+      return it('should have left the doc room', function (done) {
+        return RealTimeClient.getConnectedClient(
+          this.client.socket.sessionid,
+          (error, client) => {
+            expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(
+              false
+            )
+            return done()
+          }
+        )
+      })
+    })
+
+    describe('when sending a leaveDoc request before the previous joinDoc request has completed', function () {
+      beforeEach(function (done) {
+        this.client.emit('leaveDoc', this.doc_id, () => {})
+        this.client.emit('joinDoc', this.doc_id, () => {})
+        return this.client.emit('leaveDoc', this.doc_id, (error) => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        })
+      })
+
+      it('should not trigger an error', function () {
+        return sinon.assert.neverCalledWith(
+          logger.error,
+          sinon.match.any,
+          "not subscribed - shouldn't happen"
+        )
+      })
+
+      return it('should have left the doc room', function (done) {
+        return RealTimeClient.getConnectedClient(
+          this.client.socket.sessionid,
+          (error, client) => {
+            expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(
+              false
+            )
+            return done()
+          }
+        )
+      })
+    })
+
+    return describe('when sending a leaveDoc for a room the client has not joined ', function () {
+      beforeEach(function (done) {
+        return this.client.emit('leaveDoc', this.other_doc_id, (error) => {
+          if (error != null) {
+            throw error
+          }
+          return done()
+        })
+      })
+
+      return it('should trigger a low level message only', function () {
+        return sinon.assert.calledWith(
+          logger.log,
+          sinon.match.any,
+          'ignoring request from client to leave room it is not in'
+        )
+      })
+    })
+  })
+})

services/real-time/test/acceptance/js/LeaveProjectTests.js

@@ -0,0 +1,270 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-throw-literal,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+const settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(settings.redis.pubsub)
+
+describe('leaveProject', function () {
+  before(function (done) {
+    return MockDocUpdaterServer.run(done)
+  })
+
+  describe('with other clients in the project', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connectionAccepted', cb)
+          },
+
+          (cb) => {
+            this.clientB = RealTimeClient.connect()
+            this.clientB.on('connectionAccepted', cb)
+
+            this.clientBDisconnectMessages = []
+            return this.clientB.on(
+              'clientTracking.clientDisconnected',
+              (data) => {
+                return this.clientBDisconnectMessages.push(data)
+              }
+            )
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return this.clientB.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            return this.clientA.emit('joinDoc', this.doc_id, cb)
+          },
+          (cb) => {
+            return this.clientB.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            // leaveProject is called when the client disconnects
+            this.clientA.on('disconnect', () => cb())
+            return this.clientA.disconnect()
+          },
+
+          (cb) => {
+            // The API waits a little while before flushing changes
+            return setTimeout(done, 1000)
+          }
+        ],
+        done
+      )
+    })
+
+    it('should emit a disconnect message to the room', function () {
+      return this.clientBDisconnectMessages.should.deep.equal([
+        this.clientA.publicId
+      ])
+    })
+
+    it('should no longer list the client in connected users', function (done) {
+      return this.clientB.emit(
+        'clientTracking.getConnectedUsers',
+        (error, users) => {
+          for (const user of Array.from(users)) {
+            if (user.client_id === this.clientA.publicId) {
+              throw 'Expected clientA to not be listed in connected users'
+            }
+          }
+          return done()
+        }
+      )
+    })
+
+    it('should not flush the project to the document updater', function () {
+      return MockDocUpdaterServer.deleteProject
+        .calledWith(this.project_id)
+        .should.equal(false)
+    })
+
+    it('should remain subscribed to the editor-events channels', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.include(`editor-events:${this.project_id}`)
+        return done()
+      })
+      return null
+    })
+
+    return it('should remain subscribed to the applied-ops channels', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+
+  return describe('with no other clients in the project', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connect', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+          (cb) => {
+            return this.clientA.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            // leaveProject is called when the client disconnects
+            this.clientA.on('disconnect', () => cb())
+            return this.clientA.disconnect()
+          },
+
+          (cb) => {
+            // The API waits a little while before flushing changes
+            return setTimeout(done, 1000)
+          }
+        ],
+        done
+      )
+    })
+
+    it('should flush the project to the document updater', function () {
+      return MockDocUpdaterServer.deleteProject
+        .calledWith(this.project_id)
+        .should.equal(true)
+    })
+
+    it('should not subscribe to the editor-events channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`editor-events:${this.project_id}`)
+        return done()
+      })
+      return null
+    })
+
+    return it('should not subscribe to the applied-ops channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+})

services/real-time/test/acceptance/js/PubSubRace.js

@@ -0,0 +1,373 @@
+/* eslint-disable
+    camelcase,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+const settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(settings.redis.pubsub)
+
+describe('PubSubRace', function () {
+  before(function (done) {
+    return MockDocUpdaterServer.run(done)
+  })
+
+  describe('when the client leaves a doc before joinDoc completes', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connect', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            // leave before joinDoc completes
+            return this.clientA.emit('leaveDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            // wait for subscribe and unsubscribe
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    return it('should not subscribe to the applied-ops channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+
+  describe('when the client emits joinDoc and leaveDoc requests frequently and leaves eventually', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connect', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            return this.clientA.emit('leaveDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            // wait for subscribe and unsubscribe
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    return it('should not subscribe to the applied-ops channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+
+  describe('when the client emits joinDoc and leaveDoc requests frequently and remains in the doc', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connect', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            this.clientA.emit('joinDoc', this.doc_id, () => {})
+            this.clientA.emit('leaveDoc', this.doc_id, () => {})
+            return this.clientA.emit('joinDoc', this.doc_id, cb)
+          },
+
+          (cb) => {
+            // wait for subscribe and unsubscribe
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    return it('should subscribe to the applied-ops channels', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+
+  return describe('when the client disconnects before joinDoc completes', function () {
+    before(function (done) {
+      return async.series(
+        [
+          (cb) => {
+            return FixturesManager.setUpProject(
+              {
+                privilegeLevel: 'owner',
+                project: {
+                  name: 'Test Project'
+                }
+              },
+              (e, { project_id, user_id }) => {
+                this.project_id = project_id
+                this.user_id = user_id
+                return cb()
+              }
+            )
+          },
+
+          (cb) => {
+            this.clientA = RealTimeClient.connect()
+            return this.clientA.on('connect', cb)
+          },
+
+          (cb) => {
+            return this.clientA.emit(
+              'joinProject',
+              { project_id: this.project_id },
+              (error, project, privilegeLevel, protocolVersion) => {
+                this.project = project
+                this.privilegeLevel = privilegeLevel
+                this.protocolVersion = protocolVersion
+                return cb(error)
+              }
+            )
+          },
+
+          (cb) => {
+            return FixturesManager.setUpDoc(
+              this.project_id,
+              { lines: this.lines, version: this.version, ops: this.ops },
+              (e, { doc_id }) => {
+                this.doc_id = doc_id
+                return cb(e)
+              }
+            )
+          },
+
+          (cb) => {
+            let joinDocCompleted = false
+            this.clientA.emit(
+              'joinDoc',
+              this.doc_id,
+              () => (joinDocCompleted = true)
+            )
+            // leave before joinDoc completes
+            return setTimeout(
+              () => {
+                if (joinDocCompleted) {
+                  return cb(new Error('joinDocCompleted -- lower timeout'))
+                }
+                this.clientA.on('disconnect', () => cb())
+                return this.clientA.disconnect()
+              },
+              // socket.io processes joinDoc and disconnect with different delays:
+              //  - joinDoc goes through two process.nextTick
+              //  - disconnect goes through one process.nextTick
+              // We have to inject the disconnect event into a different event loop
+              //  cycle.
+              3
+            )
+          },
+
+          (cb) => {
+            // wait for subscribe and unsubscribe
+            return setTimeout(cb, 100)
+          }
+        ],
+        done
+      )
+    })
+
+    it('should not subscribe to the editor-events channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`editor-events:${this.project_id}`)
+        return done()
+      })
+      return null
+    })
+
+    return it('should not subscribe to the applied-ops channels anymore', function (done) {
+      rclient.pubsub('CHANNELS', (err, resp) => {
+        if (err) {
+          return done(err)
+        }
+        resp.should.not.include(`applied-ops:${this.doc_id}`)
+        return done()
+      })
+      return null
+    })
+  })
+})

services/real-time/test/acceptance/js/ReceiveUpdateTests.js

@@ -0,0 +1,349 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const MockWebServer = require('./helpers/MockWebServer')
+const FixturesManager = require('./helpers/FixturesManager')
+
+const async = require('async')
+
+const settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(settings.redis.pubsub)
+
+describe('receiveUpdate', function () {
+  beforeEach(function (done) {
+    this.lines = ['test', 'doc', 'lines']
+    this.version = 42
+    this.ops = ['mock', 'doc', 'ops']
+
+    return async.series(
+      [
+        (cb) => {
+          return FixturesManager.setUpProject(
+            {
+              privilegeLevel: 'owner',
+              project: { name: 'Test Project' }
+            },
+            (error, { user_id, project_id }) => {
+              this.user_id = user_id
+              this.project_id = project_id
+              return cb()
+            }
+          )
+        },
+
+        (cb) => {
+          return FixturesManager.setUpDoc(
+            this.project_id,
+            { lines: this.lines, version: this.version, ops: this.ops },
+            (e, { doc_id }) => {
+              this.doc_id = doc_id
+              return cb(e)
+            }
+          )
+        },
+
+        (cb) => {
+          this.clientA = RealTimeClient.connect()
+          return this.clientA.on('connectionAccepted', cb)
+        },
+
+        (cb) => {
+          this.clientB = RealTimeClient.connect()
+          return this.clientB.on('connectionAccepted', cb)
+        },
+
+        (cb) => {
+          return this.clientA.emit(
+            'joinProject',
+            {
+              project_id: this.project_id
+            },
+            cb
+          )
+        },
+
+        (cb) => {
+          return this.clientA.emit('joinDoc', this.doc_id, cb)
+        },
+
+        (cb) => {
+          return this.clientB.emit(
+            'joinProject',
+            {
+              project_id: this.project_id
+            },
+            cb
+          )
+        },
+
+        (cb) => {
+          return this.clientB.emit('joinDoc', this.doc_id, cb)
+        },
+
+        (cb) => {
+          return FixturesManager.setUpProject(
+            {
+              privilegeLevel: 'owner',
+              project: { name: 'Test Project' }
+            },
+            (
+              error,
+              { user_id: user_id_second, project_id: project_id_second }
+            ) => {
+              this.user_id_second = user_id_second
+              this.project_id_second = project_id_second
+              return cb()
+            }
+          )
+        },
+
+        (cb) => {
+          return FixturesManager.setUpDoc(
+            this.project_id_second,
+            { lines: this.lines, version: this.version, ops: this.ops },
+            (e, { doc_id: doc_id_second }) => {
+              this.doc_id_second = doc_id_second
+              return cb(e)
+            }
+          )
+        },
+
+        (cb) => {
+          this.clientC = RealTimeClient.connect()
+          return this.clientC.on('connectionAccepted', cb)
+        },
+
+        (cb) => {
+          return this.clientC.emit(
+            'joinProject',
+            {
+              project_id: this.project_id_second
+            },
+            cb
+          )
+        },
+        (cb) => {
+          return this.clientC.emit('joinDoc', this.doc_id_second, cb)
+        },
+
+        (cb) => {
+          this.clientAUpdates = []
+          this.clientA.on('otUpdateApplied', (update) =>
+            this.clientAUpdates.push(update)
+          )
+          this.clientBUpdates = []
+          this.clientB.on('otUpdateApplied', (update) =>
+            this.clientBUpdates.push(update)
+          )
+          this.clientCUpdates = []
+          this.clientC.on('otUpdateApplied', (update) =>
+            this.clientCUpdates.push(update)
+          )
+
+          this.clientAErrors = []
+          this.clientA.on('otUpdateError', (error) =>
+            this.clientAErrors.push(error)
+          )
+          this.clientBErrors = []
+          this.clientB.on('otUpdateError', (error) =>
+            this.clientBErrors.push(error)
+          )
+          this.clientCErrors = []
+          this.clientC.on('otUpdateError', (error) =>
+            this.clientCErrors.push(error)
+          )
+          return cb()
+        }
+      ],
+      done
+    )
+  })
+
+  afterEach(function () {
+    if (this.clientA != null) {
+      this.clientA.disconnect()
+    }
+    if (this.clientB != null) {
+      this.clientB.disconnect()
+    }
+    return this.clientC != null ? this.clientC.disconnect() : undefined
+  })
+
+  describe('with an update from clientA', function () {
+    beforeEach(function (done) {
+      this.update = {
+        doc_id: this.doc_id,
+        op: {
+          meta: {
+            source: this.clientA.publicId
+          },
+          v: this.version,
+          doc: this.doc_id,
+          op: [{ i: 'foo', p: 50 }]
+        }
+      }
+      rclient.publish('applied-ops', JSON.stringify(this.update))
+      return setTimeout(done, 200)
+    }) // Give clients time to get message
+
+    it('should send the full op to clientB', function () {
+      return this.clientBUpdates.should.deep.equal([this.update.op])
+    })
+
+    it('should send an ack to clientA', function () {
+      return this.clientAUpdates.should.deep.equal([
+        {
+          v: this.version,
+          doc: this.doc_id
+        }
+      ])
+    })
+
+    return it('should send nothing to clientC', function () {
+      return this.clientCUpdates.should.deep.equal([])
+    })
+  })
+
+  describe('with an update from clientC', function () {
+    beforeEach(function (done) {
+      this.update = {
+        doc_id: this.doc_id_second,
+        op: {
+          meta: {
+            source: this.clientC.publicId
+          },
+          v: this.version,
+          doc: this.doc_id_second,
+          op: [{ i: 'update from clientC', p: 50 }]
+        }
+      }
+      rclient.publish('applied-ops', JSON.stringify(this.update))
+      return setTimeout(done, 200)
+    }) // Give clients time to get message
+
+    it('should send nothing to clientA', function () {
+      return this.clientAUpdates.should.deep.equal([])
+    })
+
+    it('should send nothing to clientB', function () {
+      return this.clientBUpdates.should.deep.equal([])
+    })
+
+    return it('should send an ack to clientC', function () {
+      return this.clientCUpdates.should.deep.equal([
+        {
+          v: this.version,
+          doc: this.doc_id_second
+        }
+      ])
+    })
+  })
+
+  describe('with an update from a remote client for project 1', function () {
+    beforeEach(function (done) {
+      this.update = {
+        doc_id: this.doc_id,
+        op: {
+          meta: {
+            source: 'this-is-a-remote-client-id'
+          },
+          v: this.version,
+          doc: this.doc_id,
+          op: [{ i: 'foo', p: 50 }]
+        }
+      }
+      rclient.publish('applied-ops', JSON.stringify(this.update))
+      return setTimeout(done, 200)
+    }) // Give clients time to get message
+
+    it('should send the full op to clientA', function () {
+      return this.clientAUpdates.should.deep.equal([this.update.op])
+    })
+
+    it('should send the full op to clientB', function () {
+      return this.clientBUpdates.should.deep.equal([this.update.op])
+    })
+
+    return it('should send nothing to clientC', function () {
+      return this.clientCUpdates.should.deep.equal([])
+    })
+  })
+
+  describe('with an error for the first project', function () {
+    beforeEach(function (done) {
+      rclient.publish(
+        'applied-ops',
+        JSON.stringify({
+          doc_id: this.doc_id,
+          error: (this.error = 'something went wrong')
+        })
+      )
+      return setTimeout(done, 200)
+    }) // Give clients time to get message
+
+    it('should send the error to the clients in the first project', function () {
+      this.clientAErrors.should.deep.equal([this.error])
+      return this.clientBErrors.should.deep.equal([this.error])
+    })
+
+    it('should not send any errors to the client in the second project', function () {
+      return this.clientCErrors.should.deep.equal([])
+    })
+
+    it('should disconnect the clients of the first project', function () {
+      this.clientA.socket.connected.should.equal(false)
+      return this.clientB.socket.connected.should.equal(false)
+    })
+
+    return it('should not disconnect the client in the second project', function () {
+      return this.clientC.socket.connected.should.equal(true)
+    })
+  })
+
+  return describe('with an error for the second project', function () {
+    beforeEach(function (done) {
+      rclient.publish(
+        'applied-ops',
+        JSON.stringify({
+          doc_id: this.doc_id_second,
+          error: (this.error = 'something went wrong')
+        })
+      )
+      return setTimeout(done, 200)
+    }) // Give clients time to get message
+
+    it('should not send any errors to the clients in the first project', function () {
+      this.clientAErrors.should.deep.equal([])
+      return this.clientBErrors.should.deep.equal([])
+    })
+
+    it('should send the error to the client in the second project', function () {
+      return this.clientCErrors.should.deep.equal([this.error])
+    })
+
+    it('should not disconnect the clients of the first project', function () {
+      this.clientA.socket.connected.should.equal(true)
+      return this.clientB.socket.connected.should.equal(true)
+    })
+
+    return it('should disconnect the client in the second project', function () {
+      return this.clientC.socket.connected.should.equal(false)
+    })
+  })
+})

services/real-time/test/acceptance/js/RouterTests.js

@@ -0,0 +1,121 @@
+/* eslint-disable
+    camelcase,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const async = require('async')
+const { expect } = require('chai')
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+const FixturesManager = require('./helpers/FixturesManager')
+
+describe('Router', function () {
+  return describe('joinProject', function () {
+    describe('when there is no callback provided', function () {
+      after(function () {
+        return process.removeListener('unhandledRejection', this.onUnhandled)
+      })
+
+      before(function (done) {
+        this.onUnhandled = (error) => done(error)
+        process.on('unhandledRejection', this.onUnhandled)
+        return async.series(
+          [
+            (cb) => {
+              return FixturesManager.setUpProject(
+                {
+                  privilegeLevel: 'owner',
+                  project: {
+                    name: 'Test Project'
+                  }
+                },
+                (e, { project_id, user_id }) => {
+                  this.project_id = project_id
+                  this.user_id = user_id
+                  return cb(e)
+                }
+              )
+            },
+
+            (cb) => {
+              this.client = RealTimeClient.connect()
+              return this.client.on('connectionAccepted', cb)
+            },
+
+            (cb) => {
+              this.client = RealTimeClient.connect()
+              return this.client.on('connectionAccepted', cb)
+            },
+
+            (cb) => {
+              this.client.emit('joinProject', { project_id: this.project_id })
+              return setTimeout(cb, 100)
+            }
+          ],
+          done
+        )
+      })
+
+      return it('should keep on going', function () {
+        return expect('still running').to.exist
+      })
+    })
+
+    return describe('when there are too many arguments', function () {
+      after(function () {
+        return process.removeListener('unhandledRejection', this.onUnhandled)
+      })
+
+      before(function (done) {
+        this.onUnhandled = (error) => done(error)
+        process.on('unhandledRejection', this.onUnhandled)
+        return async.series(
+          [
+            (cb) => {
+              return FixturesManager.setUpProject(
+                {
+                  privilegeLevel: 'owner',
+                  project: {
+                    name: 'Test Project'
+                  }
+                },
+                (e, { project_id, user_id }) => {
+                  this.project_id = project_id
+                  this.user_id = user_id
+                  return cb(e)
+                }
+              )
+            },
+
+            (cb) => {
+              this.client = RealTimeClient.connect()
+              return this.client.on('connectionAccepted', cb)
+            },
+
+            (cb) => {
+              this.client = RealTimeClient.connect()
+              return this.client.on('connectionAccepted', cb)
+            },
+
+            (cb) => {
+              return this.client.emit('joinProject', 1, 2, 3, 4, 5, (error) => {
+                this.error = error
+                return cb()
+              })
+            }
+          ],
+          done
+        )
+      })
+
+      return it('should return an error message', function () {
+        return expect(this.error.message).to.equal('unexpected arguments')
+      })
+    })
+  })
+})

services/real-time/test/acceptance/js/SessionSocketsTests.js

@@ -0,0 +1,103 @@
+/* eslint-disable
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const RealTimeClient = require('./helpers/RealTimeClient')
+const Settings = require('settings-sharelatex')
+const { expect } = require('chai')
+
+describe('SessionSockets', function () {
+  before(function () {
+    return (this.checkSocket = function (fn) {
+      const client = RealTimeClient.connect()
+      client.on('connectionAccepted', fn)
+      client.on('connectionRejected', fn)
+      return null
+    })
+  })
+
+  describe('without cookies', function () {
+    before(function () {
+      return (RealTimeClient.cookie = null)
+    })
+
+    return it('should return a lookup error', function (done) {
+      return this.checkSocket((error) => {
+        expect(error).to.exist
+        expect(error.message).to.equal('invalid session')
+        return done()
+      })
+    })
+  })
+
+  describe('with a different cookie', function () {
+    before(function () {
+      return (RealTimeClient.cookie = 'some.key=someValue')
+    })
+
+    return it('should return a lookup error', function (done) {
+      return this.checkSocket((error) => {
+        expect(error).to.exist
+        expect(error.message).to.equal('invalid session')
+        return done()
+      })
+    })
+  })
+
+  describe('with an invalid cookie', function () {
+    before(function (done) {
+      RealTimeClient.setSession({}, (error) => {
+        if (error) {
+          return done(error)
+        }
+        RealTimeClient.cookie = `${
+          Settings.cookieName
+        }=${RealTimeClient.cookie.slice(17, 49)}`
+        return done()
+      })
+      return null
+    })
+
+    return it('should return a lookup error', function (done) {
+      return this.checkSocket((error) => {
+        expect(error).to.exist
+        expect(error.message).to.equal('invalid session')
+        return done()
+      })
+    })
+  })
+
+  describe('with a valid cookie and no matching session', function () {
+    before(function () {
+      return (RealTimeClient.cookie = `${Settings.cookieName}=unknownId`)
+    })
+
+    return it('should return a lookup error', function (done) {
+      return this.checkSocket((error) => {
+        expect(error).to.exist
+        expect(error.message).to.equal('invalid session')
+        return done()
+      })
+    })
+  })
+
+  return describe('with a valid cookie and a matching session', function () {
+    before(function (done) {
+      RealTimeClient.setSession({}, done)
+      return null
+    })
+
+    return it('should not return an error', function (done) {
+      return this.checkSocket((error) => {
+        expect(error).to.not.exist
+        return done()
+      })
+    })
+  })
+})

services/real-time/test/acceptance/js/SessionTests.js

@@ -0,0 +1,61 @@
+/* eslint-disable
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+
+const RealTimeClient = require('./helpers/RealTimeClient')
+
+describe('Session', function () {
+  return describe('with an established session', function () {
+    before(function (done) {
+      this.user_id = 'mock-user-id'
+      RealTimeClient.setSession(
+        {
+          user: { _id: this.user_id }
+        },
+        (error) => {
+          if (error != null) {
+            throw error
+          }
+          this.client = RealTimeClient.connect()
+          return done()
+        }
+      )
+      return null
+    })
+
+    it('should not get disconnected', function (done) {
+      let disconnected = false
+      this.client.on('disconnect', () => (disconnected = true))
+      return setTimeout(() => {
+        expect(disconnected).to.equal(false)
+        return done()
+      }, 500)
+    })
+
+    return it('should appear in the list of connected clients', function (done) {
+      return RealTimeClient.getConnectedClients((error, clients) => {
+        let included = false
+        for (const client of Array.from(clients)) {
+          if (client.client_id === this.client.socket.sessionid) {
+            included = true
+            break
+          }
+        }
+        expect(included).to.equal(true)
+        return done()
+      })
+    })
+  })
+})

services/real-time/test/acceptance/js/helpers/FixturesManager.js

@@ -0,0 +1,119 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let FixturesManager
+const RealTimeClient = require('./RealTimeClient')
+const MockWebServer = require('./MockWebServer')
+const MockDocUpdaterServer = require('./MockDocUpdaterServer')
+
+module.exports = FixturesManager = {
+  setUpProject(options, callback) {
+    if (options == null) {
+      options = {}
+    }
+    if (callback == null) {
+      callback = function (error, data) {}
+    }
+    if (!options.user_id) {
+      options.user_id = FixturesManager.getRandomId()
+    }
+    if (!options.project_id) {
+      options.project_id = FixturesManager.getRandomId()
+    }
+    if (!options.project) {
+      options.project = { name: 'Test Project' }
+    }
+    const {
+      project_id,
+      user_id,
+      privilegeLevel,
+      project,
+      publicAccess
+    } = options
+
+    const privileges = {}
+    privileges[user_id] = privilegeLevel
+    if (publicAccess) {
+      privileges['anonymous-user'] = publicAccess
+    }
+
+    MockWebServer.createMockProject(project_id, privileges, project)
+    return MockWebServer.run((error) => {
+      if (error != null) {
+        throw error
+      }
+      return RealTimeClient.setSession(
+        {
+          user: {
+            _id: user_id,
+            first_name: 'Joe',
+            last_name: 'Bloggs'
+          }
+        },
+        (error) => {
+          if (error != null) {
+            throw error
+          }
+          return callback(null, {
+            project_id,
+            user_id,
+            privilegeLevel,
+            project
+          })
+        }
+      )
+    })
+  },
+
+  setUpDoc(project_id, options, callback) {
+    if (options == null) {
+      options = {}
+    }
+    if (callback == null) {
+      callback = function (error, data) {}
+    }
+    if (!options.doc_id) {
+      options.doc_id = FixturesManager.getRandomId()
+    }
+    if (!options.lines) {
+      options.lines = ['doc', 'lines']
+    }
+    if (!options.version) {
+      options.version = 42
+    }
+    if (!options.ops) {
+      options.ops = ['mock', 'ops']
+    }
+    const { doc_id, lines, version, ops, ranges } = options
+
+    MockDocUpdaterServer.createMockDoc(project_id, doc_id, {
+      lines,
+      version,
+      ops,
+      ranges
+    })
+    return MockDocUpdaterServer.run((error) => {
+      if (error != null) {
+        throw error
+      }
+      return callback(null, { project_id, doc_id, lines, version, ops })
+    })
+  },
+
+  getRandomId() {
+    return require('crypto')
+      .createHash('sha1')
+      .update(Math.random().toString())
+      .digest('hex')
+      .slice(0, 24)
+  }
+}

services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js

@@ -0,0 +1,90 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockDocUpdaterServer
+const sinon = require('sinon')
+const express = require('express')
+
+module.exports = MockDocUpdaterServer = {
+  docs: {},
+
+  createMockDoc(project_id, doc_id, data) {
+    return (MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] = data)
+  },
+
+  getDocument(project_id, doc_id, fromVersion, callback) {
+    if (callback == null) {
+      callback = function (error, data) {}
+    }
+    return callback(null, MockDocUpdaterServer.docs[`${project_id}:${doc_id}`])
+  },
+
+  deleteProject: sinon.stub().callsArg(1),
+
+  getDocumentRequest(req, res, next) {
+    const { project_id, doc_id } = req.params
+    let { fromVersion } = req.query
+    fromVersion = parseInt(fromVersion, 10)
+    return MockDocUpdaterServer.getDocument(
+      project_id,
+      doc_id,
+      fromVersion,
+      (error, data) => {
+        if (error != null) {
+          return next(error)
+        }
+        return res.json(data)
+      }
+    )
+  },
+
+  deleteProjectRequest(req, res, next) {
+    const { project_id } = req.params
+    return MockDocUpdaterServer.deleteProject(project_id, (error) => {
+      if (error != null) {
+        return next(error)
+      }
+      return res.sendStatus(204)
+    })
+  },
+
+  running: false,
+  run(callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (MockDocUpdaterServer.running) {
+      return callback()
+    }
+    const app = express()
+    app.get(
+      '/project/:project_id/doc/:doc_id',
+      MockDocUpdaterServer.getDocumentRequest
+    )
+    app.delete(
+      '/project/:project_id',
+      MockDocUpdaterServer.deleteProjectRequest
+    )
+    return app
+      .listen(3003, (error) => {
+        MockDocUpdaterServer.running = true
+        return callback(error)
+      })
+      .on('error', (error) => {
+        console.error('error starting MockDocUpdaterServer:', error.message)
+        return process.exit(1)
+      })
+  }
+}
+
+sinon.spy(MockDocUpdaterServer, 'getDocument')

services/real-time/test/acceptance/js/helpers/MockWebServer.js

@@ -0,0 +1,82 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockWebServer
+const sinon = require('sinon')
+const express = require('express')
+
+module.exports = MockWebServer = {
+  projects: {},
+  privileges: {},
+
+  createMockProject(project_id, privileges, project) {
+    MockWebServer.privileges[project_id] = privileges
+    return (MockWebServer.projects[project_id] = project)
+  },
+
+  joinProject(project_id, user_id, callback) {
+    if (callback == null) {
+      callback = function (error, project, privilegeLevel) {}
+    }
+    return callback(
+      null,
+      MockWebServer.projects[project_id],
+      MockWebServer.privileges[project_id][user_id]
+    )
+  },
+
+  joinProjectRequest(req, res, next) {
+    const { project_id } = req.params
+    const { user_id } = req.query
+    if (project_id === 'rate-limited') {
+      return res.status(429).send()
+    } else {
+      return MockWebServer.joinProject(
+        project_id,
+        user_id,
+        (error, project, privilegeLevel) => {
+          if (error != null) {
+            return next(error)
+          }
+          return res.json({
+            project,
+            privilegeLevel
+          })
+        }
+      )
+    }
+  },
+
+  running: false,
+  run(callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (MockWebServer.running) {
+      return callback()
+    }
+    const app = express()
+    app.post('/project/:project_id/join', MockWebServer.joinProjectRequest)
+    return app
+      .listen(3000, (error) => {
+        MockWebServer.running = true
+        return callback(error)
+      })
+      .on('error', (error) => {
+        console.error('error starting MockWebServer:', error.message)
+        return process.exit(1)
+      })
+  }
+}
+
+sinon.spy(MockWebServer, 'joinProject')

services/real-time/test/acceptance/js/helpers/RealTimeClient.js

@@ -0,0 +1,131 @@
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let Client
+const { XMLHttpRequest } = require('../../libs/XMLHttpRequest')
+const io = require('socket.io-client')
+const async = require('async')
+
+const request = require('request')
+const Settings = require('settings-sharelatex')
+const redis = require('redis-sharelatex')
+const rclient = redis.createClient(Settings.redis.websessions)
+
+const uid = require('uid-safe').sync
+const signature = require('cookie-signature')
+
+io.util.request = function () {
+  const xhr = new XMLHttpRequest()
+  const _open = xhr.open
+  xhr.open = function () {
+    _open.apply(xhr, arguments)
+    if (Client.cookie != null) {
+      return xhr.setRequestHeader('Cookie', Client.cookie)
+    }
+  }
+  return xhr
+}
+
+module.exports = Client = {
+  cookie: null,
+
+  setSession(session, callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    const sessionId = uid(24)
+    session.cookie = {}
+    return rclient.set(
+      'sess:' + sessionId,
+      JSON.stringify(session),
+      (error) => {
+        if (error != null) {
+          return callback(error)
+        }
+        const secret = Settings.security.sessionSecret
+        const cookieKey = 's:' + signature.sign(sessionId, secret)
+        Client.cookie = `${Settings.cookieName}=${cookieKey}`
+        return callback()
+      }
+    )
+  },
+
+  unsetSession(callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    Client.cookie = null
+    return callback()
+  },
+
+  connect(cookie) {
+    const client = io.connect('http://localhost:3026', {
+      'force new connection': true
+    })
+    client.on(
+      'connectionAccepted',
+      (_, publicId) => (client.publicId = publicId)
+    )
+    return client
+  },
+
+  getConnectedClients(callback) {
+    if (callback == null) {
+      callback = function (error, clients) {}
+    }
+    return request.get(
+      {
+        url: 'http://localhost:3026/clients',
+        json: true
+      },
+      (error, response, data) => callback(error, data)
+    )
+  },
+
+  getConnectedClient(client_id, callback) {
+    if (callback == null) {
+      callback = function (error, clients) {}
+    }
+    return request.get(
+      {
+        url: `http://localhost:3026/clients/${client_id}`,
+        json: true
+      },
+      (error, response, data) => callback(error, data)
+    )
+  },
+
+  disconnectClient(client_id, callback) {
+    request.post(
+      {
+        url: `http://localhost:3026/client/${client_id}/disconnect`,
+        auth: {
+          user: Settings.internal.realTime.user,
+          pass: Settings.internal.realTime.pass
+        }
+      },
+      (error, response, data) => callback(error, data)
+    )
+    return null
+  },
+
+  disconnectAllClients(callback) {
+    return Client.getConnectedClients((error, clients) =>
+      async.each(
+        clients,
+        (clientView, cb) => Client.disconnectClient(clientView.client_id, cb),
+        callback
+      )
+    )
+  }
+}

services/real-time/test/acceptance/js/helpers/RealtimeServer.js

@@ -0,0 +1,64 @@
+/* eslint-disable
+    handle-callback-err,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const app = require('../../../../app')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+
+module.exports = {
+  running: false,
+  initing: false,
+  callbacks: [],
+  ensureRunning(callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    if (this.running) {
+      return callback()
+    } else if (this.initing) {
+      return this.callbacks.push(callback)
+    } else {
+      this.initing = true
+      this.callbacks.push(callback)
+      return app.listen(
+        __guard__(
+          Settings.internal != null ? Settings.internal.realtime : undefined,
+          (x) => x.port
+        ),
+        'localhost',
+        (error) => {
+          if (error != null) {
+            throw error
+          }
+          this.running = true
+          logger.log('clsi running in dev mode')
+
+          return (() => {
+            const result = []
+            for (callback of Array.from(this.callbacks)) {
+              result.push(callback())
+            }
+            return result
+          })()
+        }
+      )
+    }
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/real-time/test/acceptance/libs/XMLHttpRequest.js

@@ -11,100 +11,95 @@
  * @license MIT
  */
 
-var Url = require("url")
-  , spawn = require("child_process").spawn
-  , fs = require('fs');
+const { URL } = require('url')
+var spawn = require('child_process').spawn
+var fs = require('fs')
 
-exports.XMLHttpRequest = function() {
+exports.XMLHttpRequest = function () {
   /**
    * Private variables
    */
-  var self = this;
-  var http = require('http');
-  var https = require('https');
+  var self = this
+  var http = require('http')
+  var https = require('https')
 
   // Holds http.js objects
-  var client;
-  var request;
-  var response;
+  var request
+  var response
 
   // Request settings
-  var settings = {};
+  var settings = {}
 
   // Set some default headers
   var defaultHeaders = {
-    "User-Agent": "node-XMLHttpRequest",
-    "Accept": "*/*",
-  };
+    'User-Agent': 'node-XMLHttpRequest',
+    Accept: '*/*'
+  }
 
-  var headers = defaultHeaders;
+  var headers = defaultHeaders
 
   // These headers are not user setable.
   // The following are allowed but banned in the spec:
   // * user-agent
   var forbiddenRequestHeaders = [
-    "accept-charset",
-    "accept-encoding",
-    "access-control-request-headers",
-    "access-control-request-method",
-    "connection",
-    "content-length",
-    "content-transfer-encoding",
-    //"cookie",
-    "cookie2",
-    "date",
-    "expect",
-    "host",
-    "keep-alive",
-    "origin",
-    "referer",
-    "te",
-    "trailer",
-    "transfer-encoding",
-    "upgrade",
-    "via"
-  ];
+    'accept-charset',
+    'accept-encoding',
+    'access-control-request-headers',
+    'access-control-request-method',
+    'connection',
+    'content-length',
+    'content-transfer-encoding',
+    // "cookie",
+    'cookie2',
+    'date',
+    'expect',
+    'host',
+    'keep-alive',
+    'origin',
+    'referer',
+    'te',
+    'trailer',
+    'transfer-encoding',
+    'upgrade',
+    'via'
+  ]
 
   // These request methods are not allowed
-  var forbiddenRequestMethods = [
-    "TRACE",
-    "TRACK",
-    "CONNECT"
-  ];
+  var forbiddenRequestMethods = ['TRACE', 'TRACK', 'CONNECT']
 
   // Send flag
-  var sendFlag = false;
+  var sendFlag = false
   // Error flag, used when errors occur or abort is called
-  var errorFlag = false;
+  var errorFlag = false
 
   // Event listeners
-  var listeners = {};
+  var listeners = {}
 
   /**
    * Constants
    */
 
-  this.UNSENT = 0;
-  this.OPENED = 1;
-  this.HEADERS_RECEIVED = 2;
-  this.LOADING = 3;
-  this.DONE = 4;
+  this.UNSENT = 0
+  this.OPENED = 1
+  this.HEADERS_RECEIVED = 2
+  this.LOADING = 3
+  this.DONE = 4
 
   /**
    * Public vars
    */
 
   // Current state
-  this.readyState = this.UNSENT;
+  this.readyState = this.UNSENT
 
   // default ready state change handler in case one is not set or is set late
-  this.onreadystatechange = null;
+  this.onreadystatechange = null
 
   // Result & response
-  this.responseText = "";
-  this.responseXML = "";
-  this.status = null;
-  this.statusText = null;
+  this.responseText = ''
+  this.responseXML = ''
+  this.status = null
+  this.statusText = null
 
   /**
    * Private methods
@@ -116,9 +111,11 @@ exports.XMLHttpRequest = function() {
    * @param string header Header to validate
    * @return boolean False if not allowed, otherwise true
    */
-  var isAllowedHttpHeader = function(header) {
-    return (header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1);
-  };
+  var isAllowedHttpHeader = function (header) {
+    return (
+      header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1
+    )
+  }
 
   /**
    * Check if the specified method is allowed.
@@ -126,9 +123,9 @@ exports.XMLHttpRequest = function() {
    * @param string method Request method to validate
    * @return boolean False if not allowed, otherwise true
    */
-  var isAllowedHttpMethod = function(method) {
-    return (method && forbiddenRequestMethods.indexOf(method) === -1);
-  };
+  var isAllowedHttpMethod = function (method) {
+    return method && forbiddenRequestMethods.indexOf(method) === -1
+  }
 
   /**
    * Public methods
@@ -143,26 +140,25 @@ exports.XMLHttpRequest = function() {
    * @param string user Username for basic authentication (optional)
    * @param string password Password for basic authentication (optional)
    */
-  this.open = function(method, url, async, user, password) {
-    this.abort();
-    errorFlag = false;
+  this.open = function (method, url, async, user, password) {
+    this.abort()
+    errorFlag = false
 
     // Check for valid request method
     if (!isAllowedHttpMethod(method)) {
-      throw "SecurityError: Request method not allowed";
-      return;
+      throw new Error('SecurityError: Request method not allowed')
     }
 
     settings = {
-      "method": method,
-      "url": url.toString(),
-      "async": (typeof async !== "boolean" ? true : async),
-      "user": user || null,
-      "password": password || null
-    };
+      method: method,
+      url: url.toString(),
+      async: typeof async !== 'boolean' ? true : async,
+      user: user || null,
+      password: password || null
+    }
 
-    setState(this.OPENED);
-  };
+    setState(this.OPENED)
+  }
 
   /**
    * Sets a header for the request.
@@ -170,19 +166,21 @@ exports.XMLHttpRequest = function() {
    * @param string header Header name
    * @param string value Header value
    */
-  this.setRequestHeader = function(header, value) {
-    if (this.readyState != this.OPENED) {
-      throw "INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN";
+  this.setRequestHeader = function (header, value) {
+    if (this.readyState !== this.OPENED) {
+      throw new Error(
+        'INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN'
+      )
     }
     if (!isAllowedHttpHeader(header)) {
-      console.warn('Refused to set unsafe header "' + header + '"');
-      return;
+      console.warn('Refused to set unsafe header "' + header + '"')
+      return
     }
     if (sendFlag) {
-      throw "INVALID_STATE_ERR: send flag is true";
+      throw new Error('INVALID_STATE_ERR: send flag is true')
     }
-    headers[header] = value;
-  };
+    headers[header] = value
+  }
 
   /**
    * Gets a header from the server response.
@@ -190,37 +188,38 @@ exports.XMLHttpRequest = function() {
    * @param string header Name of header to get.
    * @return string Text of the header or null if it doesn't exist.
    */
-  this.getResponseHeader = function(header) {
-    if (typeof header === "string"
-      && this.readyState > this.OPENED
-      && response.headers[header.toLowerCase()]
-      && !errorFlag
+  this.getResponseHeader = function (header) {
+    if (
+      typeof header === 'string' &&
+      this.readyState > this.OPENED &&
+      response.headers[header.toLowerCase()] &&
+      !errorFlag
     ) {
-      return response.headers[header.toLowerCase()];
+      return response.headers[header.toLowerCase()]
     }
 
-    return null;
-  };
+    return null
+  }
 
   /**
    * Gets all the response headers.
    *
    * @return string A string with all response headers separated by CR+LF
    */
-  this.getAllResponseHeaders = function() {
+  this.getAllResponseHeaders = function () {
     if (this.readyState < this.HEADERS_RECEIVED || errorFlag) {
-      return "";
+      return ''
     }
-    var result = "";
+    var result = ''
 
     for (var i in response.headers) {
       // Cookie headers are excluded
-      if (i !== "set-cookie" && i !== "set-cookie2") {
-        result += i + ": " + response.headers[i] + "\r\n";
+      if (i !== 'set-cookie' && i !== 'set-cookie2') {
+        result += i + ': ' + response.headers[i] + '\r\n'
       }
     }
-    return result.substr(0, result.length - 2);
-  };
+    return result.substr(0, result.length - 2)
+  }
 
   /**
    * Gets a request header
@@ -228,13 +227,13 @@ exports.XMLHttpRequest = function() {
    * @param string name Name of header to get
    * @return string Returns the request header or empty string if not set
    */
-  this.getRequestHeader = function(name) {
+  this.getRequestHeader = function (name) {
     // @TODO Make this case insensitive
-    if (typeof name === "string" && headers[name]) {
-      return headers[name];
+    if (typeof name === 'string' && headers[name]) {
+      return headers[name]
     }
 
-    return "";
+    return ''
   }
 
   /**
@@ -242,103 +241,108 @@ exports.XMLHttpRequest = function() {
    *
    * @param string data Optional data to send as request body.
    */
-  this.send = function(data) {
-    if (this.readyState != this.OPENED) {
-      throw "INVALID_STATE_ERR: connection must be opened before send() is called";
+  this.send = function (data) {
+    if (this.readyState !== this.OPENED) {
+      throw new Error(
+        'INVALID_STATE_ERR: connection must be opened before send() is called'
+      )
     }
 
     if (sendFlag) {
-      throw "INVALID_STATE_ERR: send has already been called";
+      throw new Error('INVALID_STATE_ERR: send has already been called')
     }
 
-    var ssl = false, local = false;
-    var url = Url.parse(settings.url);
+    var host
+    var ssl = false
+    var local = false
+    var url = new URL(settings.url)
 
     // Determine the server
     switch (url.protocol) {
       case 'https:':
-        ssl = true;
-        // SSL & non-SSL both need host, no break here.
+        ssl = true
+        host = url.hostname
+        break
       case 'http:':
-        var host = url.hostname;
-        break;
+        host = url.hostname
+        break
 
       case 'file:':
-        local = true;
-        break;
+        local = true
+        break
 
       case undefined:
       case '':
-        var host = "localhost";
-        break;
+        host = 'localhost'
+        break
 
       default:
-        throw "Protocol not supported.";
+        throw new Error('Protocol not supported.')
     }
 
     // Load files off the local filesystem (file://)
     if (local) {
-      if (settings.method !== "GET") {
-        throw "XMLHttpRequest: Only GET method is supported";
+      if (settings.method !== 'GET') {
+        throw new Error('XMLHttpRequest: Only GET method is supported')
       }
 
       if (settings.async) {
-        fs.readFile(url.pathname, 'utf8', function(error, data) {
+        fs.readFile(url.pathname, 'utf8', (error, data) => {
           if (error) {
-            self.handleError(error);
+            self.handleError(error)
           } else {
-            self.status = 200;
-            self.responseText = data;
-            setState(self.DONE);
+            self.status = 200
+            self.responseText = data
+            setState(self.DONE)
           }
-        });
+        })
       } else {
         try {
-          this.responseText = fs.readFileSync(url.pathname, 'utf8');
-          this.status = 200;
-          setState(self.DONE);
-        } catch(e) {
-          this.handleError(e);
+          this.responseText = fs.readFileSync(url.pathname, 'utf8')
+          this.status = 200
+          setState(self.DONE)
+        } catch (e) {
+          this.handleError(e)
         }
       }
 
-      return;
+      return
     }
 
     // Default to port 80. If accessing localhost on another port be sure
     // to use http://localhost:port/path
-    var port = url.port || (ssl ? 443 : 80);
+    var port = url.port || (ssl ? 443 : 80)
     // Add query string if one is used
-    var uri = url.pathname + (url.search ? url.search : '');
+    var uri = url.pathname + (url.search ? url.search : '')
 
     // Set the Host header or the server may reject the request
-    headers["Host"] = host;
+    headers.Host = host
     if (!((ssl && port === 443) || port === 80)) {
-      headers["Host"] += ':' + url.port;
+      headers.Host += ':' + url.port
     }
 
     // Set Basic Auth if necessary
     if (settings.user) {
-      if (typeof settings.password == "undefined") {
-        settings.password = "";
+      if (typeof settings.password === 'undefined') {
+        settings.password = ''
       }
-      var authBuf = new Buffer(settings.user + ":" + settings.password);
-      headers["Authorization"] = "Basic " + authBuf.toString("base64");
+      var authBuf = Buffer.from(settings.user + ':' + settings.password)
+      headers.Authorization = 'Basic ' + authBuf.toString('base64')
     }
 
     // Set content length header
-    if (settings.method === "GET" || settings.method === "HEAD") {
-      data = null;
+    if (settings.method === 'GET' || settings.method === 'HEAD') {
+      data = null
     } else if (data) {
-      headers["Content-Length"] = Buffer.byteLength(data);
+      headers['Content-Length'] = Buffer.byteLength(data)
 
-      if (!headers["Content-Type"]) {
-        headers["Content-Type"] = "text/plain;charset=UTF-8";
+      if (!headers['Content-Type']) {
+        headers['Content-Type'] = 'text/plain;charset=UTF-8'
       }
-    } else if (settings.method === "POST") {
+    } else if (settings.method === 'POST') {
       // For a post with no data set Content-Length: 0.
       // This is required by buggy servers that don't meet the specs.
-      headers["Content-Length"] = 0;
+      headers['Content-Length'] = 0
     }
 
     var options = {
@@ -347,202 +351,229 @@ exports.XMLHttpRequest = function() {
       path: uri,
       method: settings.method,
       headers: headers
-    };
+    }
 
     // Reset error flag
-    errorFlag = false;
+    errorFlag = false
 
     // Handle async requests
     if (settings.async) {
       // Use the proper protocol
-      var doRequest = ssl ? https.request : http.request;
+      var doRequest = ssl ? https.request : http.request
 
       // Request is being sent, set send flag
-      sendFlag = true;
+      sendFlag = true
 
       // As per spec, this is called here for historical reasons.
-      self.dispatchEvent("readystatechange");
+      self.dispatchEvent('readystatechange')
 
       // Create the request
-      request = doRequest(options, function(resp) {
-        response = resp;
-        response.setEncoding("utf8");
+      request = doRequest(options, (resp) => {
+        response = resp
+        response.setEncoding('utf8')
 
-        setState(self.HEADERS_RECEIVED);
-        self.status = response.statusCode;
+        setState(self.HEADERS_RECEIVED)
+        self.status = response.statusCode
 
-        response.on('data', function(chunk) {
+        response.on('data', (chunk) => {
           // Make sure there's some data
           if (chunk) {
-            self.responseText += chunk;
+            self.responseText += chunk
           }
           // Don't emit state changes if the connection has been aborted.
           if (sendFlag) {
-            setState(self.LOADING);
+            setState(self.LOADING)
           }
-        });
+        })
 
-        response.on('end', function() {
+        response.on('end', () => {
           if (sendFlag) {
             // Discard the 'end' event if the connection has been aborted
-            setState(self.DONE);
-            sendFlag = false;
+            setState(self.DONE)
+            sendFlag = false
           }
-        });
+        })
 
-        response.on('error', function(error) {
-          self.handleError(error);
-        });
-      }).on('error', function(error) {
-        self.handleError(error);
-      });
+        response.on('error', (error) => {
+          self.handleError(error)
+        })
+      }).on('error', (error) => {
+        self.handleError(error)
+      })
 
       // Node 0.4 and later won't accept empty data. Make sure it's needed.
       if (data) {
-        request.write(data);
+        request.write(data)
       }
 
-      request.end();
+      request.end()
 
-      self.dispatchEvent("loadstart");
-    } else { // Synchronous
+      self.dispatchEvent('loadstart')
+    } else {
+      // Synchronous
       // Create a temporary file for communication with the other Node process
-      var syncFile = ".node-xmlhttprequest-sync-" + process.pid;
-      fs.writeFileSync(syncFile, "", "utf8");
+      var syncFile = '.node-xmlhttprequest-sync-' + process.pid
+      fs.writeFileSync(syncFile, '', 'utf8')
       // The async request the other Node process executes
-      var execString = "var http = require('http'), https = require('https'), fs = require('fs');"
-        + "var doRequest = http" + (ssl ? "s" : "") + ".request;"
-        + "var options = " + JSON.stringify(options) + ";"
-        + "var responseText = '';"
-        + "var req = doRequest(options, function(response) {"
-        + "response.setEncoding('utf8');"
-        + "response.on('data', function(chunk) {"
-        + "responseText += chunk;"
-        + "});"
-        + "response.on('end', function() {"
-        + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-STATUS:' + response.statusCode + ',' + responseText, 'utf8');"
-        + "});"
-        + "response.on('error', function(error) {"
-        + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
-        + "});"
-        + "}).on('error', function(error) {"
-        + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
-        + "});"
-        + (data ? "req.write('" + data.replace(/'/g, "\\'") + "');":"")
-        + "req.end();";
+      var execString =
+        "var http = require('http'), https = require('https'), fs = require('fs');" +
+        'var doRequest = http' +
+        (ssl ? 's' : '') +
+        '.request;' +
+        'var options = ' +
+        JSON.stringify(options) +
+        ';' +
+        "var responseText = '';" +
+        'var req = doRequest(options, function(response) {' +
+        "response.setEncoding('utf8');" +
+        "response.on('data', function(chunk) {" +
+        'responseText += chunk;' +
+        '});' +
+        "response.on('end', function() {" +
+        "fs.writeFileSync('" +
+        syncFile +
+        "', 'NODE-XMLHTTPREQUEST-STATUS:' + response.statusCode + ',' + responseText, 'utf8');" +
+        '});' +
+        "response.on('error', function(error) {" +
+        "fs.writeFileSync('" +
+        syncFile +
+        "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" +
+        '});' +
+        "}).on('error', function(error) {" +
+        "fs.writeFileSync('" +
+        syncFile +
+        "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" +
+        '});' +
+        (data ? "req.write('" + data.replace(/'/g, "\\'") + "');" : '') +
+        'req.end();'
       // Start the other Node Process, executing this string
-      syncProc = spawn(process.argv[0], ["-e", execString]);
-      while((self.responseText = fs.readFileSync(syncFile, 'utf8')) == "") {
+      const syncProc = spawn(process.argv[0], ['-e', execString])
+      while ((self.responseText = fs.readFileSync(syncFile, 'utf8')) === '') {
         // Wait while the file is empty
       }
       // Kill the child process once the file has data
-      syncProc.stdin.end();
+      syncProc.stdin.end()
       // Remove the temporary file
-      fs.unlinkSync(syncFile);
+      fs.unlinkSync(syncFile)
       if (self.responseText.match(/^NODE-XMLHTTPREQUEST-ERROR:/)) {
         // If the file returned an error, handle it
-        var errorObj = self.responseText.replace(/^NODE-XMLHTTPREQUEST-ERROR:/, "");
-        self.handleError(errorObj);
+        var errorObj = self.responseText.replace(
+          /^NODE-XMLHTTPREQUEST-ERROR:/,
+          ''
+        )
+        self.handleError(errorObj)
       } else {
         // If the file returned okay, parse its data and move to the DONE state
-        self.status = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, "$1");
-        self.responseText = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, "$1");
-        setState(self.DONE);
+        self.status = self.responseText.replace(
+          /^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/,
+          '$1'
+        )
+        self.responseText = self.responseText.replace(
+          /^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/,
+          '$1'
+        )
+        setState(self.DONE)
       }
     }
-  };
+  }
 
   /**
    * Called when an error is encountered to deal with it.
    */
-  this.handleError = function(error) {
-    this.status = 503;
-    this.statusText = error;
-    this.responseText = error.stack;
-    errorFlag = true;
-    setState(this.DONE);
-  };
+  this.handleError = function (error) {
+    this.status = 503
+    this.statusText = error
+    this.responseText = error.stack
+    errorFlag = true
+    setState(this.DONE)
+  }
 
   /**
    * Aborts a request.
    */
-  this.abort = function() {
+  this.abort = function () {
     if (request) {
-      request.abort();
-      request = null;
+      request.abort()
+      request = null
     }
 
-    headers = defaultHeaders;
-    this.responseText = "";
-    this.responseXML = "";
+    headers = defaultHeaders
+    this.responseText = ''
+    this.responseXML = ''
 
-    errorFlag = true;
+    errorFlag = true
 
-    if (this.readyState !== this.UNSENT
-        && (this.readyState !== this.OPENED || sendFlag)
-        && this.readyState !== this.DONE) {
-      sendFlag = false;
-      setState(this.DONE);
+    if (
+      this.readyState !== this.UNSENT &&
+      (this.readyState !== this.OPENED || sendFlag) &&
+      this.readyState !== this.DONE
+    ) {
+      sendFlag = false
+      setState(this.DONE)
     }
-    this.readyState = this.UNSENT;
-  };
+    this.readyState = this.UNSENT
+  }
 
   /**
    * Adds an event listener. Preferred method of binding to events.
    */
-  this.addEventListener = function(event, callback) {
+  this.addEventListener = function (event, callback) {
     if (!(event in listeners)) {
-      listeners[event] = [];
+      listeners[event] = []
     }
     // Currently allows duplicate callbacks. Should it?
-    listeners[event].push(callback);
-  };
+    listeners[event].push(callback)
+  }
 
   /**
    * Remove an event callback that has already been bound.
    * Only works on the matching funciton, cannot be a copy.
    */
-  this.removeEventListener = function(event, callback) {
+  this.removeEventListener = function (event, callback) {
     if (event in listeners) {
       // Filter will return a new array with the callback removed
-      listeners[event] = listeners[event].filter(function(ev) {
-        return ev !== callback;
-      });
+      listeners[event] = listeners[event].filter((ev) => {
+        return ev !== callback
+      })
     }
-  };
+  }
 
   /**
    * Dispatch any events, including both "on" methods and events attached using addEventListener.
    */
-  this.dispatchEvent = function(event) {
-    if (typeof self["on" + event] === "function") {
-      self["on" + event]();
+  this.dispatchEvent = function (event) {
+    if (typeof self['on' + event] === 'function') {
+      self['on' + event]()
     }
     if (event in listeners) {
       for (var i = 0, len = listeners[event].length; i < len; i++) {
-        listeners[event][i].call(self);
+        listeners[event][i].call(self)
       }
     }
-  };
+  }
 
   /**
    * Changes readyState and calls onreadystatechange.
    *
    * @param int state New state
    */
-  var setState = function(state) {
+  var setState = function (state) {
     if (self.readyState !== state) {
-      self.readyState = state;
+      self.readyState = state
 
-      if (settings.async || self.readyState < self.OPENED || self.readyState === self.DONE) {
-        self.dispatchEvent("readystatechange");
+      if (
+        settings.async ||
+        self.readyState < self.OPENED ||
+        self.readyState === self.DONE
+      ) {
+        self.dispatchEvent('readystatechange')
       }
 
       if (self.readyState === self.DONE && !errorFlag) {
-        self.dispatchEvent("load");
+        self.dispatchEvent('load')
         // @TODO figure out InspectorInstrumentation::didLoadXHR(cookie)
-        self.dispatchEvent("loadend");
+        self.dispatchEvent('loadend')
       }
     }
-  };
-};
+  }
+}

services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee

@@ -1,166 +0,0 @@
-chai = require "chai"
-chai.should()
-expect = chai.expect
-sinon = require("sinon")
-SandboxedModule = require('sandboxed-module')
-path = require "path"
-modulePath = '../../../app/js/AuthorizationManager'
-
-describe 'AuthorizationManager', ->
-	beforeEach ->
-		@client =
-			ol_context: {}
-
-		@AuthorizationManager = SandboxedModule.require modulePath, requires: {}
-
-	describe "assertClientCanViewProject", ->
-		it "should allow the readOnly privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "readOnly"
-			@AuthorizationManager.assertClientCanViewProject @client, (error) ->
-				expect(error).to.be.null
-				done()
-
-		it "should allow the readAndWrite privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "readAndWrite"
-			@AuthorizationManager.assertClientCanViewProject @client, (error) ->
-				expect(error).to.be.null
-				done()
-
-		it "should allow the owner privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "owner"
-			@AuthorizationManager.assertClientCanViewProject @client, (error) ->
-				expect(error).to.be.null
-				done()
-
-		it "should return an error with any other privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "unknown"
-			@AuthorizationManager.assertClientCanViewProject @client, (error) ->
-				error.message.should.equal "not authorized"
-				done()
-
-	describe "assertClientCanEditProject", ->
-		it "should not allow the readOnly privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "readOnly"
-			@AuthorizationManager.assertClientCanEditProject @client, (error) ->
-				error.message.should.equal "not authorized"
-				done()
-
-		it "should allow the readAndWrite privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "readAndWrite"
-			@AuthorizationManager.assertClientCanEditProject @client, (error) ->
-				expect(error).to.be.null
-				done()
-
-		it "should allow the owner privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "owner"
-			@AuthorizationManager.assertClientCanEditProject @client, (error) ->
-				expect(error).to.be.null
-				done()
-
-		it "should return an error with any other privilegeLevel", (done) ->
-			@client.ol_context.privilege_level = "unknown"
-			@AuthorizationManager.assertClientCanEditProject @client, (error) ->
-				error.message.should.equal "not authorized"
-				done()
-
-	# check doc access for project
-
-	describe "assertClientCanViewProjectAndDoc", ->
-		beforeEach () ->
-			@doc_id = "12345"
-			@callback = sinon.stub()
-			@client.ol_context = {}
-
-		describe "when not authorised at the project level", ->
-			beforeEach () ->
-				@client.ol_context.privilege_level = "unknown"
-
-			it "should not allow access", () ->
-				@AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) ->
-					err.message.should.equal "not authorized"
-
-			describe "even when authorised at the doc level", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, done
-
-				it "should not allow access", () ->
-					@AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"
-
-		describe "when authorised at the project level", ->
-			beforeEach () ->
-				@client.ol_context.privilege_level = "readOnly"
-
-			describe "and not authorised at the document level", ->
-				it "should not allow access", () ->
-					@AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"
-
-			describe "and authorised at the document level", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, done
-
-				it "should allow access", () ->
-					@AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback
-					@callback
-						.calledWith(null)
-						.should.equal true
-
-			describe "when document authorisation is added and then removed", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, () =>
-						@AuthorizationManager.removeAccessToDoc @client, @doc_id, done
-
-				it "should deny access", () ->
-					@AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"
-
-	describe "assertClientCanEditProjectAndDoc", ->
-		beforeEach () ->
-			@doc_id = "12345"
-			@callback = sinon.stub()
-			@client.ol_context = {}
-
-		describe "when not authorised at the project level", ->
-			beforeEach () ->
-				@client.ol_context.privilege_level = "readOnly"
-
-			it "should not allow access", () ->
-				@AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) ->
-					err.message.should.equal "not authorized"
-
-			describe "even when authorised at the doc level", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, done
-
-				it "should not allow access", () ->
-					@AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"
-
-		describe "when authorised at the project level", ->
-			beforeEach () ->
-				@client.ol_context.privilege_level = "readAndWrite"
-
-			describe "and not authorised at the document level", ->
-				it "should not allow access", () ->
-					@AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"
-
-			describe "and authorised at the document level", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, done
-
-				it "should allow access", () ->
-					@AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback
-					@callback
-						.calledWith(null)
-						.should.equal true
-
-			describe "when document authorisation is added and then removed", ->
-				beforeEach (done) ->
-					@AuthorizationManager.addAccessToDoc @client, @doc_id, () =>
-						@AuthorizationManager.removeAccessToDoc @client, @doc_id, done
-
-				it "should deny access", () ->
-					@AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) ->
-						err.message.should.equal "not authorized"

services/real-time/test/unit/coffee/ChannelManagerTests.coffee

@@ -1,220 +0,0 @@
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-sinon = require("sinon")
-modulePath = "../../../app/js/ChannelManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe 'ChannelManager', ->
-	beforeEach ->
-		@rclient = {}
-		@other_rclient = {}
-		@ChannelManager = SandboxedModule.require modulePath, requires:
-			"settings-sharelatex": @settings = {}
-			"metrics-sharelatex": @metrics = {inc: sinon.stub(), summary: sinon.stub()}
-			"logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }
-
-	describe "subscribe", ->
-
-		describe "when there is no existing subscription for this redis client", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should subscribe to the redis channel", ->
-				@rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true
-
-		describe "when there is an existing subscription for this redis client", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should subscribe to the redis channel again", ->
-				@rclient.subscribe.callCount.should.equal 2
-
-		describe "when subscribe errors", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub()
-					.onFirstCall().rejects(new Error("some redis error"))
-					.onSecondCall().resolves()
-				p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				p.then () ->
-					done(new Error('should not subscribe but fail'))
-				.catch (err) =>
-					err.message.should.equal "some redis error"
-					@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false
-					@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-					# subscribe is wrapped in Promise, delay other assertions
-					setTimeout done
-				return null
-
-			it "should have recorded the error", ->
-				expect(@metrics.inc.calledWithExactly("subscribe.failed.applied-ops")).to.equal(true)
-
-			it "should subscribe again", ->
-				@rclient.subscribe.callCount.should.equal 2
-
-			it "should cleanup", ->
-				@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false
-
-		describe "when subscribe errors and the clientChannelMap entry was replaced", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub()
-					.onFirstCall().rejects(new Error("some redis error"))
-					.onSecondCall().resolves()
-				@first = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				# ignore error
-				@first.catch((()->))
-				expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @first
-
-				@rclient.unsubscribe = sinon.stub().resolves()
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-				@second = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				# should get replaced immediately
-				expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @second
-
-				# let the first subscribe error -> unsubscribe -> subscribe
-				setTimeout done
-
-			it "should cleanup the second subscribePromise", ->
-				expect(@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef")).to.equal false
-
-		describe "when there is an existing subscription for another redis client but not this one", ->
-			beforeEach (done) ->
-				@other_rclient.subscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef"
-				@rclient.subscribe = sinon.stub().resolves()  # discard the original stub
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should subscribe to the redis channel on this redis client", ->
-				@rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true
-
-	describe "unsubscribe", ->
-
-		describe "when there is no existing subscription for this redis client", ->
-			beforeEach (done) ->
-				@rclient.unsubscribe = sinon.stub().resolves()
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should unsubscribe from the redis channel", ->
-				@rclient.unsubscribe.called.should.equal true
-
-
-		describe "when there is an existing subscription for this another redis client but not this one", ->
-			beforeEach (done) ->
-				@other_rclient.subscribe = sinon.stub().resolves()
-				@rclient.unsubscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef"
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should still unsubscribe from the redis channel on this client", ->
-				@rclient.unsubscribe.called.should.equal true
-
-		describe "when unsubscribe errors and completes", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				@rclient.unsubscribe = sinon.stub().rejects(new Error("some redis error"))
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-				return null
-
-			it "should have cleaned up", ->
-				@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false
-
-			it "should not error out when subscribing again", (done) ->
-				p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				p.then () ->
-					done()
-				.catch done
-				return null
-
-		describe "when unsubscribe errors and another client subscribes at the same time", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				rejectSubscribe = undefined
-				@rclient.unsubscribe = () ->
-					return new Promise (resolve, reject) ->
-						rejectSubscribe = reject
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-
-				setTimeout () =>
-					# delay, actualUnsubscribe should not see the new subscribe request
-					@ChannelManager.subscribe(@rclient, "applied-ops", "1234567890abcdef")
-					.then () ->
-						setTimeout done
-					.catch done
-					setTimeout ->
-						# delay, rejectSubscribe is not defined immediately
-						rejectSubscribe(new Error("redis error"))
-				return null
-
-			it "should have recorded the error", ->
-				expect(@metrics.inc.calledWithExactly("unsubscribe.failed.applied-ops")).to.equal(true)
-
-			it "should have subscribed", ->
-				@rclient.subscribe.called.should.equal true
-
-			it "should have discarded the finished Promise", ->
-				@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false
-
-		describe "when there is an existing subscription for this redis client", ->
-			beforeEach (done) ->
-				@rclient.subscribe = sinon.stub().resolves()
-				@rclient.unsubscribe = sinon.stub().resolves()
-				@ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef"
-				@ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef"
-				setTimeout done
-
-			it "should unsubscribe from the redis channel", ->
-				@rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true
-
-	describe "publish", ->
-
-		describe "when the channel is 'all'", ->
-			beforeEach ->
-				@rclient.publish = sinon.stub()
-				@ChannelManager.publish @rclient, "applied-ops", "all", "random-message"
-
-			it "should publish on the base channel", ->
-				@rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true
-
-		describe "when the channel has an specific id", ->
-
-			describe "when the individual channel setting is false", ->
-				beforeEach ->
-					@rclient.publish = sinon.stub()
-					@settings.publishOnIndividualChannels = false
-					@ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message"
-
-				it "should publish on the per-id channel", ->
-					@rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true
-					@rclient.publish.calledOnce.should.equal true
-
-			describe "when the individual channel setting is true", ->
-				beforeEach ->
-					@rclient.publish = sinon.stub()
-					@settings.publishOnIndividualChannels = true
-					@ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message"
-
-				it "should publish on the per-id channel", ->
-					@rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal true
-					@rclient.publish.calledOnce.should.equal true
-
-		describe "metrics", ->
-			beforeEach ->
-				@rclient.publish = sinon.stub()
-				@ChannelManager.publish @rclient, "applied-ops", "all", "random-message"
-
-			it "should track the payload size", ->
-				@metrics.summary.calledWithExactly(
-					"redis.publish.applied-ops",
-					"random-message".length
-				).should.equal true

services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee

@@ -1,164 +0,0 @@
-
-should = require('chai').should()
-SandboxedModule = require('sandboxed-module')
-assert = require('assert')
-path = require('path')
-sinon = require('sinon')
-modulePath = path.join __dirname, "../../../app/js/ConnectedUsersManager"
-expect = require("chai").expect
-tk = require("timekeeper")
-
-
-describe "ConnectedUsersManager", ->
-
-	beforeEach ->
-
-		@settings =
-			redis:
-				realtime:
-					key_schema:
-						clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}"
-						connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}"
-		@rClient =
-			auth:->
-			setex:sinon.stub()
-			sadd:sinon.stub()
-			get: sinon.stub()
-			srem:sinon.stub()
-			del:sinon.stub()
-			smembers:sinon.stub()
-			expire:sinon.stub()
-			hset:sinon.stub()
-			hgetall:sinon.stub()
-			exec:sinon.stub()
-			multi: => return @rClient
-		tk.freeze(new Date())
-
-		@ConnectedUsersManager = SandboxedModule.require modulePath, requires:
-			"settings-sharelatex":@settings
-			"logger-sharelatex": log:->
-			"redis-sharelatex": createClient:=> 
-				return @rClient
-		@client_id = "32132132"
-		@project_id = "dskjh2u21321"
-		@user = {
-			_id: "user-id-123"
-			first_name: "Joe"
-			last_name: "Bloggs"
-			email: "joe@example.com"
-		}
-		@cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' }
-
-	afterEach -> 
-		tk.reset()
-
-	describe "updateUserPosition", ->
-		beforeEach ->
-			@rClient.exec.callsArgWith(0)
-
-		it "should set a key with the date and give it a ttl", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_updated_at", Date.now()).should.equal true
-				done()
-
-		it "should set a key with the user_id", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "user_id", @user._id).should.equal true
-				done()
-
-		it "should set a key with the first_name", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "first_name", @user.first_name).should.equal true
-				done()
-
-		it "should set a key with the last_name", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_name", @user.last_name).should.equal true
-				done()
-
-		it "should set a key with the email", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "email", @user.email).should.equal true
-				done()
-
-		it "should push the client_id on to the project list", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.sadd.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true
-				done()
-
-		it "should add a ttl to the project set so it stays clean", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true
-				done()
-
-		it "should add a ttl to the connected user so it stays clean", (done) ->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=>
-				@rClient.expire.calledWith("connected_user:#{@project_id}:#{@client_id}", 60 * 15).should.equal true
-				done()
-
-		it "should set the cursor position when provided", (done)->
-			@ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, @cursorData, (err)=>
-				@rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "cursorData", JSON.stringify(@cursorData)).should.equal true
-				done()
-
-	describe "markUserAsDisconnected", ->
-		beforeEach ->
-			@rClient.exec.callsArgWith(0)
-
-		it "should remove the user from the set", (done)->
-			@ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=>
-				@rClient.srem.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true
-				done()
-
-		it "should delete the connected_user string", (done)->
-			@ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=>
-				@rClient.del.calledWith("connected_user:#{@project_id}:#{@client_id}").should.equal true
-				done()
-
-		it "should add a ttl to the connected user set so it stays clean", (done)->
-			@ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=>
-				@rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true
-				done()
-
-	describe "_getConnectedUser", ->
-
-		it "should return a connected user if there is a user object", (done)->
-			cursorData = JSON.stringify(cursorData:{row:1})
-			@rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), user_id: @user._id, last_updated_at: "#{Date.now()}", cursorData})
-			@ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=>
-				result.connected.should.equal true
-				result.client_id.should.equal @client_id
-				done()
-
-		it "should return a not connected user if there is no object", (done)->
-			@rClient.hgetall.callsArgWith(1, null, null)
-			@ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=>
-				result.connected.should.equal false
-				result.client_id.should.equal @client_id
-				done()
-
-		it "should return a not connected user if there is an empty object", (done)->
-			@rClient.hgetall.callsArgWith(1, null, {})
-			@ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=>
-				result.connected.should.equal false
-				result.client_id.should.equal @client_id
-				done()
-
-	describe "getConnectedUsers", ->
-
-		beforeEach ->
-			@users = ["1234", "5678", "9123", "8234"]
-			@rClient.smembers.callsArgWith(1, null, @users)
-			@ConnectedUsersManager._getConnectedUser = sinon.stub()
-			@ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[0]).callsArgWith(2, null, {connected:true, client_age: 2, client_id:@users[0]})
-			@ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[1]).callsArgWith(2, null, {connected:false, client_age: 1, client_id:@users[1]})
-			@ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[2]).callsArgWith(2, null, {connected:true, client_age: 3, client_id:@users[2]})
-			@ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[3]).callsArgWith(2, null, {connected:true, client_age: 11, client_id:@users[3]}) # connected but old
-
-		it "should only return the users in the list which are still in redis and recently updated", (done)->
-			@ConnectedUsersManager.getConnectedUsers @project_id, (err, users)=>
-				users.length.should.equal 2
-				users[0].should.deep.equal {client_id:@users[0], client_age: 2, connected:true}
-				users[1].should.deep.equal {client_id:@users[2], client_age: 3, connected:true}
-				done()
-

services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee

@@ -1,153 +0,0 @@
-SandboxedModule = require('sandboxed-module')
-sinon = require('sinon')
-require('chai').should()
-modulePath = require('path').join __dirname, '../../../app/js/DocumentUpdaterController'
-MockClient = require "./helpers/MockClient"
-
-describe "DocumentUpdaterController", ->
-	beforeEach ->
-		@project_id = "project-id-123"
-		@doc_id = "doc-id-123"
-		@callback = sinon.stub()
-		@io = { "mock": "socket.io" }
-		@rclient = []
-		@RoomEvents = { on: sinon.stub() }
-		@EditorUpdatesController = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() }
-			"settings-sharelatex": @settings =
-				redis:
-					documentupdater:
-						key_schema:
-							pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
-					pubsub: null
-			"redis-sharelatex" : @redis =
-				createClient: (name) =>
-					@rclient.push(rclientStub = {name:name})
-					return rclientStub
-			"./SafeJsonParse": @SafeJsonParse =
-				parse: (data, cb) => cb null, JSON.parse(data)
-			"./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()}
-			"./HealthCheckManager": {check: sinon.stub()}
-			"metrics-sharelatex": @metrics = {inc: sinon.stub()}
-			"./RoomManager" : @RoomManager = { eventSource: sinon.stub().returns @RoomEvents}
-			"./ChannelManager": @ChannelManager = {}
-
-	describe "listenForUpdatesFromDocumentUpdater", ->
-		beforeEach ->
-			@rclient.length = 0  # clear any existing clients
-			@EditorUpdatesController.rclientList = [@redis.createClient("first"), @redis.createClient("second")]
-			@rclient[0].subscribe = sinon.stub()
-			@rclient[0].on = sinon.stub()
-			@rclient[1].subscribe = sinon.stub()
-			@rclient[1].on = sinon.stub()
-			@EditorUpdatesController.listenForUpdatesFromDocumentUpdater()
-		
-		it "should subscribe to the doc-updater stream", ->
-			@rclient[0].subscribe.calledWith("applied-ops").should.equal true
-
-		it "should register a callback to handle updates", ->
-			@rclient[0].on.calledWith("message").should.equal true
-
-		it "should subscribe to any additional doc-updater stream", ->
-			@rclient[1].subscribe.calledWith("applied-ops").should.equal true
-			@rclient[1].on.calledWith("message").should.equal true
-
-	describe "_processMessageFromDocumentUpdater", ->
-		describe "with bad JSON", ->
-			beforeEach ->
-				@SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops")
-				@EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", "blah"
-			
-			it "should log an error", ->
-				@logger.error.called.should.equal true
-
-		describe "with update", ->
-			beforeEach ->
-				@message =
-					doc_id: @doc_id
-					op: {t: "foo", p: 12}
-				@EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub()
-				@EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message)
-
-			it "should apply the update", ->
-				@EditorUpdatesController._applyUpdateFromDocumentUpdater
-					.calledWith(@io, @doc_id, @message.op)
-					.should.equal true
-
-		describe "with error", ->
-			beforeEach ->
-				@message =
-					doc_id: @doc_id
-					error: "Something went wrong"
-				@EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub()
-				@EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message)
-
-			it "should process the error", ->
-				@EditorUpdatesController._processErrorFromDocumentUpdater
-					.calledWith(@io, @doc_id, @message.error)
-					.should.equal true
-
-	describe "_applyUpdateFromDocumentUpdater", ->
-		beforeEach ->
-			@sourceClient = new MockClient()
-			@otherClients = [new MockClient(), new MockClient()]
-			@update =
-				op: [ t: "foo", p: 12 ]
-				meta: source: @sourceClient.publicId
-				v: @version = 42
-				doc: @doc_id
-			@io.sockets =
-				clients: sinon.stub().returns([@sourceClient, @otherClients..., @sourceClient]) # include a duplicate client
-		
-		describe "normally", ->
-			beforeEach ->
-				@EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update
-
-			it "should send a version bump to the source client", ->
-				@sourceClient.emit
-					.calledWith("otUpdateApplied", v: @version, doc: @doc_id)
-					.should.equal true
-				@sourceClient.emit.calledOnce.should.equal true
-
-			it "should get the clients connected to the document", ->
-				@io.sockets.clients
-					.calledWith(@doc_id)
-					.should.equal true
-
-			it "should send the full update to the other clients", ->
-				for client in @otherClients
-					client.emit
-						.calledWith("otUpdateApplied", @update)
-						.should.equal true
-		
-		describe "with a duplicate op", ->
-			beforeEach ->
-				@update.dup = true
-				@EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update
-			
-			it "should send a version bump to the source client as usual", ->
-				@sourceClient.emit
-					.calledWith("otUpdateApplied", v: @version, doc: @doc_id)
-					.should.equal true
-
-			it "should not send anything to the other clients (they've already had the op)", ->
-				for client in @otherClients
-					client.emit
-						.calledWith("otUpdateApplied")
-						.should.equal false
-
-	describe "_processErrorFromDocumentUpdater", ->
-		beforeEach ->
-			@clients = [new MockClient(), new MockClient()]
-			@io.sockets =
-				clients: sinon.stub().returns(@clients)
-			@EditorUpdatesController._processErrorFromDocumentUpdater @io, @doc_id, "Something went wrong"
-
-		it "should log a warning", ->
-			@logger.warn.called.should.equal true
-
-		it "should disconnect all clients in that document", ->
-			@io.sockets.clients.calledWith(@doc_id).should.equal true
-			for client in @clients
-				client.disconnect.called.should.equal true
-

services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee

@@ -1,193 +0,0 @@
-require('chai').should()
-sinon = require("sinon")
-SandboxedModule = require('sandboxed-module')
-path = require "path"
-modulePath = '../../../app/js/DocumentUpdaterManager'
-
-describe 'DocumentUpdaterManager', ->
-	beforeEach ->
-		@project_id = "project-id-923"
-		@doc_id = "doc-id-394"
-		@lines = ["one", "two", "three"]
-		@version = 42
-		@settings =
-			apis: documentupdater: url: "http://doc-updater.example.com"
-			redis: documentupdater:
-				key_schema:
-					pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
-			maxUpdateSize: 7 * 1024 * 1024
-		@rclient = {auth:->}
-
-		@DocumentUpdaterManager = SandboxedModule.require modulePath,
-			requires:
-				'settings-sharelatex':@settings
-				'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()}
-				'request': @request = {}
-				'redis-sharelatex' : createClient: () => @rclient
-				'metrics-sharelatex': @Metrics =
-					summary: sinon.stub()
-					Timer: class Timer
-						done: () ->
-			globals:
-				JSON: @JSON = Object.create(JSON) # avoid modifying JSON object directly
-
-	describe "getDocument", ->
-		beforeEach ->
-			@callback = sinon.stub()
-
-		describe "successfully", ->
-			beforeEach ->
-				@body = JSON.stringify
-					lines: @lines
-					version: @version
-					ops: @ops = ["mock-op-1", "mock-op-2"]
-					ranges: @ranges = {"mock": "ranges"}
-				@fromVersion = 2
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback
-
-			it 'should get the document from the document updater', ->
-				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}?fromVersion=#{@fromVersion}"
-				@request.get.calledWith(url).should.equal true
-
-			it "should call the callback with the lines, version, ranges and ops", ->
-				@callback.calledWith(null, @lines, @version, @ranges, @ops).should.equal true
-
-		describe "when the document updater API returns an error", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		[404, 422].forEach (statusCode) ->
-			describe "when the document updater returns a #{statusCode} status code", ->
-				beforeEach ->
-					@request.get = sinon.stub().callsArgWith(1, null, { statusCode }, "")
-					@DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback
-
-				it "should return the callback with an error", ->
-					@callback.called.should.equal(true)
-					err = @callback.getCall(0).args[0]
-					err.should.have.property('statusCode', statusCode)
-					err.should.have.property('message', "doc updater could not load requested ops")
-					@logger.error.called.should.equal(false)
-					@logger.warn.called.should.equal(true)
-
-		describe "when the document updater returns a failure error code", ->
-			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "")
-				@DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback
-
-			it "should return the callback with an error", ->
-				@callback.called.should.equal(true)
-				err = @callback.getCall(0).args[0]
-				err.should.have.property('statusCode', 500)
-				err.should.have.property('message', "doc updater returned a non-success status code: 500")
-				@logger.error.called.should.equal(true)
-
-	describe 'flushProjectToMongoAndDelete', ->
-		beforeEach ->
-			@callback = sinon.stub()
-
-		describe "successfully", ->
-			beforeEach ->
-				@request.del = sinon.stub().callsArgWith(1, null, {statusCode: 204}, "")
-				@DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback
-
-			it 'should delete the project from the document updater', ->
-				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}?background=true"
-				@request.del.calledWith(url).should.equal true
-
-			it "should call the callback with no error", ->
-				@callback.calledWith(null).should.equal true
-
-		describe "when the document updater API returns an error", ->
-			beforeEach ->
-				@request.del = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
-				@DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback
-
-			it "should return an error to the callback", ->
-				@callback.calledWith(@error).should.equal true
-
-		describe "when the document updater returns a failure error code", ->
-			beforeEach ->
-				@request.del = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "")
-				@DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback
-
-			it "should return the callback with an error", ->
-				@callback.called.should.equal(true)
-				err = @callback.getCall(0).args[0]
-				err.should.have.property('statusCode', 500)
-				err.should.have.property('message', "document updater returned a failure status code: 500")
-
-	describe 'queueChange', ->
-		beforeEach ->
-			@change = {
-				"doc":"1234567890",
-				"op":["d":"test", "p":345]
-				"v": 789
-			}
-			@rclient.rpush = sinon.stub().yields()
-			@callback = sinon.stub()
-
-		describe "successfully", ->
-			beforeEach ->
-				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should push the change", ->
-				@rclient.rpush
-					.calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change))
-					.should.equal true
-
-			it "should notify the doc updater of the change via the pending-updates-list queue", ->
-				@rclient.rpush
-					.calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}")
-					.should.equal true
-
-		describe "with error talking to redis during rpush", ->
-			beforeEach ->
-				@rclient.rpush = sinon.stub().yields(new Error("something went wrong"))
-				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should return an error", ->
-				@callback.calledWithExactly(sinon.match(Error)).should.equal true
-
-		describe "with null byte corruption", ->
-			beforeEach ->
-				@JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]'
-				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should return an error", ->
-				@callback.calledWithExactly(sinon.match(Error)).should.equal true
-
-			it "should not push the change onto the pending-updates-list queue", ->
-				@rclient.rpush.called.should.equal false
-
-		describe "when the update is too large", ->
-			beforeEach ->
-				@change = {op: {p: 12,t: "update is too large".repeat(1024 * 400)}}
-				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should return an error", ->
-				@callback.calledWithExactly(sinon.match(Error)).should.equal true
-
-			it "should add the size to the error", ->
-				@callback.args[0][0].updateSize.should.equal 7782422
-
-			it "should not push the change onto the pending-updates-list queue", ->
-				@rclient.rpush.called.should.equal false
-
-		describe "with invalid keys", ->
-			beforeEach ->
-				@change = {
-					"op":["d":"test", "p":345]
-					"version": 789 # not a valid key
-				}
-				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should remove the invalid keys from the change", ->
-				@rclient.rpush
-					.calledWith("PendingUpdates:#{@doc_id}", JSON.stringify({op:@change.op}))
-					.should.equal true