From d0c5eb569807ff14315c688fac5275742b9a0163 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 13 Sep 2019 15:06:42 +0100 Subject: [PATCH 01/36] support migration of project history keys to separate redis instance --- .../coffee/ProjectHistoryRedisManager.coffee | 3 +- .../app/coffee/RedisMigrationManager.coffee | 199 ++++++++++++++++++ .../config/settings.defaults.coffee | 12 ++ services/document-updater/docker-compose.yml | 9 +- .../coffee/helpers/DocUpdaterApp.coffee | 4 +- .../ProjectHistoryRedisManagerTests.coffee | 2 + 6 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 services/document-updater/app/coffee/RedisMigrationManager.coffee diff --git a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee index 1cc80ea722..c6362c4fbd 100644 --- a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee +++ b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee @@ -1,6 +1,7 @@ Settings = require('settings-sharelatex') projectHistoryKeys = Settings.redis?.project_history?.key_schema -rclient = require("redis-sharelatex").createClient(Settings.redis.documentupdater) +#rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) +rclient = require("./RedisMigrationManager").createClient(Settings.redis.project_history, Settings.redis.new_project_history) logger = require('logger-sharelatex') module.exports = ProjectHistoryRedisManager = diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee new file mode 100644 index 0000000000..c00d7727af --- /dev/null +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -0,0 +1,199 @@ +logger = require "logger-sharelatex" +Settings = require "settings-sharelatex" +redis = require("redis-sharelatex") +LockManager = require("./LockManager") +async = require("async") + +# The aim is to migrate the project history queues +# ProjectHistory:Ops:{project_id} from the existing redis to a new redis. +# +# This has to work in conjunction with changes in project history. +# +# The basic principles are: +# +# - project history is modified to read from an 'old' and 'new' queue. It reads +# from the 'old' queue first, and when that queue is empty it reads from the +# 'new' queue. +# - docupdater will migrate to writing to the 'new' queue when the 'old' queue +# is empty. +# +# Some facts about the update process: +# +# - project history has a lock on the project-id, so each queue is processed in +# isolation +# - docupdaters take a lock on the doc_id but not the project_id, therefore +# multiple docupdaters can be appending to the queue for a project at the same +# time (provided they updates for individual docs are in order this is +# acceptable) +# - as we want to do this without shutting down the site, we have to take into +# account that different versions of the code will be running while deploys +# are in progress. +# +# The migration has to be carried out with the following constraint: +# +# - a docupdater should never write to the "old" queue when there are updates in +# the "new" queue (there is a strict ordering on the versions, new > old) +# +# The deployment process for docupdater will be +# +# - add a project-level lock to the queuing in docupdater +# - use a per-project migration flag to determine when to write to the new redis +# - set the migration flag for projects with an empty queue in the old redis +# - when all docupdaters respect the flag, make a new deploy which starts to set +# the flag +# - when all docupdaters are setting the flag (and writing to the new redis), +# finish the migration by writing all data to the new redis +# +# Rollback +# +# Under the scheme above a project should only ever have data in the old redis +# or the new redis, but never both at the same time. +# +# Two scenarios: +# +# Hard rollback +# +# If we want to roll back to the old redis immediately, we need to get the data +# out of the new queues and back into the old queues, before appending to the +# old queues again. The actions to do this are: +# +# - close the site +# - revert docupdater so it only writes to the original redis (there will now +# be some data in the new redis for some projects which we need to recover) +# - run a script to move the new queues back into the old redis +# - revert project history to only read from the original redis +# +# Graceful rollback +# +# If we are prepared to keep the new redis running, but not add new projects to +# it we can do the following: +# +# - deploy all docupdaters to update from the "switch" phase into the +# "rollback" phase (projects in the new redis will continue to send data +# there, project not yet migrated will continue to go to the old redis) +# - deploy project history with the "old queue" pointing to the new redis and +# the "new queue" to the old redis to clear the new queue before processing +# the new queue (i.e. add a rollback:true property in new_project_history in +# the project-history settings) +# - projects will now clear gradually from the new redis back to the old redis +# - get a list of all the projects in the new redis and flush them, which will +# cause the new queues to be cleared and the old redis to be used for those +# projects. + +getProjectId = (key) -> + key.match(/\{([0-9a-f]{24})\}/)[1] + +class Multi + constructor: (@migrationClient) -> + @command_list = [] + @queueKey = null + rpush: (args...) -> + @queueKey = args[0] + @command_list.push { command:'rpush', args: args} + setnx: (args...) -> + @command_list.push { command: 'setnx', args: args} + exec: (callback) -> + # decide which client to use + project_id = getProjectId(@queueKey) + LockManager.getLock project_id, (error, lockValue) => + return callback(error) if error? + releaseLock = (args...) => + LockManager.releaseLock project_id, lockValue, (lockError) -> + return callback(lockError) if lockError? + callback(args...) + @migrationClient.findQueue @queueKey, (err, rclient) => + return releaseLock(err) if err? + multi = rclient.multi() + for entry in @command_list + multi[entry.command](entry.args...) + multi.exec releaseLock + +class MigrationClient + constructor: (old_settings, new_settings) -> + @rclient_old = redis.createClient(old_settings) + @rclient_new = redis.createClient(new_settings) + @new_key_schema = new_settings.key_schema + @migration_phase = new_settings.migration_phase + throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'start', 'switch', 'complete'] + + getMigrationStatus: (key, migrationKey, callback) -> + async.series [ + (cb) => @rclient_new.exists migrationKey, cb + (cb) => @rclient_new.exists key, cb + (cb) => @rclient_old.exists key, cb + ], (err, result) -> + return callback(err) if err? + migrationKeyExists = result[0] > 0 + newQueueExists = result[1] > 0 + oldQueueExists = result[2] > 0 + callback(null, migrationKeyExists, newQueueExists, oldQueueExists) + + findQueue: (key, callback) -> + project_id = getProjectId(key) + migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) + + @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) -> + return callback(err) if err? + # In all cases, if the migration key exists we must always write to the + # new redis, unless we are rolling back. + if @migration_phase is "prepare" + # in this phase we prepare for the switch, when some docupdaters will + # start setting the migration flag. We monitor the migration key and + # write to the new redis if the key is present, but we do not set the + # migration key. At this point no writes will be going into the new + # redis. When all the docupdaters are in the "prepare" phase we can + # begin deploying the "switch" phase. + if migrationKeyExists + logger.debug {project_id}, "using new client because migration key exists" + return callback(null, @rclient_new) + else + logger.debug {project_id}, "using old client because migration key does not exist" + return callback(null, @rclient_old) + else if @migration_phase is "switch" + # As we deploy the "switch" phase new docupdaters will set the migration + # flag for projects which have an empty queue in the old redis, and + # write updates into the new redis. Existing docupdaters still in the + # "prepare" phase will pick up the migration flag and write new updates + # into the new redis when appropriate. When this deploy is complete + # writes will be going into the new redis for projects with an empty + # queue in the old redis. We have to remain in the switch phase until + # all projects are flushed from the old redis. + if migrationKeyExists + logger.debug {project_id}, "using new client because migration key exists" + return callback(null, @rclient_new) + else + if oldQueueExists + logger.debug {project_id}, "using old client because old queue exists" + return callback(null, @rclient_old) + else + @rclient_new.setnx migrationKey, "NEW", (err) => + return callback(err) if err? + logger.debug {key: key}, "switching to new redis because old queue is empty" + return callback(null, @rclient_new) + else if @migration_phase is "rollback" + # If we need to roll back gracefully we do the opposite of the "switch" + # phase. We use the new redis when the migration key is set and the + # queue exists in the new redis, but if the queue in the new redis is + # empty we delete the migration key and send further updates to the old + # redis. + if migrationKeyExists + if newQueueExists + logger.debug {project_id}, "using new client because migration key exists and new queue is present" + return callback(null, @rclient_new) + else + @rclient_new.del migrationKey, (err) => + return callback(err) if err? + logger.debug {key: key}, "switching to old redis in rollback phase because new queue is empty" + return callback(null, @rclient_old) + else + logger.debug {project_id}, "using old client because migration key does not exist" + return callback(null, @rclient_old) + else + logger.error {key: key}, "unknown migration phase" + callback(new Error('invalid migration phase')) + multi: () -> + new Multi(@) + +module.exports = RedisMigrationManager = + createClient: (args...) -> + new MigrationClient(args...) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 9eebe86005..f890eb0f4a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -45,6 +45,18 @@ module.exports = projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + new_project_history: + port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" + host: process.env["NEW_HISTORY_REDIS_HOST"] + password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + key_schema: + projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" + projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" + migration_phase: "prepare" + redisOptions: + keepAlive: 100 + lock: port: process.env["LOCK_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" host: process.env["LOCK_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 6dc90009ca..31869acb50 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + NEW_HISTORY_REDIS_HOST: new_redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -34,10 +35,9 @@ services: depends_on: - mongo - redis + - new_redis command: npm run test:acceptance - - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -49,7 +49,8 @@ services: redis: image: redis + new_redis: + image: redis + mongo: image: mongo:3.4 - - diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee index 9819f9f99e..0f77199e73 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee @@ -13,8 +13,8 @@ module.exports = else @initing = true @callbacks.push callback - app.listen 3003, "localhost", (error) => + app.listen 3003, "localhost", (error) => throw error if error? @running = true for callback in @callbacks - callback() \ No newline at end of file + callback() diff --git a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee index a93545b250..6748f87af9 100644 --- a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee @@ -24,6 +24,8 @@ describe "ProjectHistoryRedisManager", -> } "redis-sharelatex": createClient: () => @rclient + "./RedisMigrationManager": + createClient: () => @rclient "logger-sharelatex": log:-> globals: From a85dffbcefad8d52e8508541eda4ee76a39471a7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 09:27:00 +0000 Subject: [PATCH 02/36] fix acceptance tests --- services/document-updater/docker-compose.ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index c78d90e8ed..d65f97b913 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -20,6 +20,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + NEW_HISTORY_REDIS_HOST: new_redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -27,6 +28,7 @@ services: depends_on: - mongo - redis + - new_redis user: node command: npm run test:acceptance:_run @@ -43,5 +45,8 @@ services: redis: image: redis + new_redis: + image: redis + mongo: image: mongo:3.4 From a2e63d009ee16a9064c0cb5ce3fd2ad7c9411812 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 09:55:26 +0000 Subject: [PATCH 03/36] fix migration phase check --- .../app/coffee/RedisMigrationManager.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index c00d7727af..f1ec00cf5c 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -114,7 +114,7 @@ class MigrationClient @rclient_new = redis.createClient(new_settings) @new_key_schema = new_settings.key_schema @migration_phase = new_settings.migration_phase - throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'start', 'switch', 'complete'] + throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] getMigrationStatus: (key, migrationKey, callback) -> async.series [ @@ -132,7 +132,7 @@ class MigrationClient project_id = getProjectId(key) migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) -> + @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => return callback(err) if err? # In all cases, if the migration key exists we must always write to the # new redis, unless we are rolling back. @@ -189,7 +189,7 @@ class MigrationClient logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) else - logger.error {key: key}, "unknown migration phase" + logger.error {key: key, migration_phase: @migration_phase}, "unknown migration phase" callback(new Error('invalid migration phase')) multi: () -> new Multi(@) From 97cbf461601add8ac9c024b9cbc189afe3fdc26b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 11:46:35 +0000 Subject: [PATCH 04/36] add metrics for migration --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 5 +++++ services/document-updater/package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index f1ec00cf5c..553e707753 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -2,6 +2,7 @@ logger = require "logger-sharelatex" Settings = require "settings-sharelatex" redis = require("redis-sharelatex") LockManager = require("./LockManager") +metrics = require "./Metrics" async = require("async") # The aim is to migrate the project history queues @@ -89,6 +90,7 @@ class Multi @queueKey = null rpush: (args...) -> @queueKey = args[0] + @updates_count = args.length - 1 @command_list.push { command:'rpush', args: args} setnx: (args...) -> @command_list.push { command: 'setnx', args: args} @@ -103,6 +105,9 @@ class Multi callback(args...) @migrationClient.findQueue @queueKey, (err, rclient) => return releaseLock(err) if err? + # add metric for updates + dest = (if rclient == @rclient_new then "new" else "old") + metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} multi = rclient.multi() for entry in @command_list multi[entry.command](entry.args...) diff --git a/services/document-updater/package.json b/services/document-updater/package.json index f80251248b..ca79ec51ac 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -26,7 +26,7 @@ "lodash": "^4.17.4", "logger-sharelatex": "^1.7.0", "lynx": "0.0.11", - "metrics-sharelatex": "^2.2.0", + "metrics-sharelatex": "^2.4.0", "mongojs": "^2.6.0", "redis-sharelatex": "^1.0.11", "request": "2.47.0", From 8ae95ebf604734a4c3d2ddb96d7f3c29134fcc49 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:45:36 +0000 Subject: [PATCH 05/36] fix rclient check in migration metrics --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 553e707753..4978bf7e7b 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -106,7 +106,7 @@ class Multi @migrationClient.findQueue @queueKey, (err, rclient) => return releaseLock(err) if err? # add metric for updates - dest = (if rclient == @rclient_new then "new" else "old") + dest = (if rclient == @migrationClient.rclient_new then "new" else "old") metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} multi = rclient.multi() for entry in @command_list From 27044c2d02c5cd380dc9a29bf412d5e1d5c19d9c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:46:35 +0000 Subject: [PATCH 06/36] allow migration phase to be modified at runtime for testing --- .../app/coffee/RedisMigrationManager.coffee | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 4978bf7e7b..96c8ee049c 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -114,12 +114,17 @@ class Multi multi.exec releaseLock class MigrationClient - constructor: (old_settings, new_settings) -> - @rclient_old = redis.createClient(old_settings) - @rclient_new = redis.createClient(new_settings) + constructor: (@old_settings, @new_settings) -> + @rclient_old = redis.createClient(@old_settings) + @rclient_new = redis.createClient(@new_settings) @new_key_schema = new_settings.key_schema - @migration_phase = new_settings.migration_phase + # check that migration phase is valid on startup + @getMigrationPhase() + + getMigrationPhase: () -> + @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] + return @migration_phase getMigrationStatus: (key, migrationKey, callback) -> async.series [ @@ -136,12 +141,12 @@ class MigrationClient findQueue: (key, callback) -> project_id = getProjectId(key) migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - + migration_phase = @getMigrationPhase() # allow setting migration phase while running for testing @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => return callback(err) if err? # In all cases, if the migration key exists we must always write to the # new redis, unless we are rolling back. - if @migration_phase is "prepare" + if migration_phase is "prepare" # in this phase we prepare for the switch, when some docupdaters will # start setting the migration flag. We monitor the migration key and # write to the new redis if the key is present, but we do not set the @@ -154,7 +159,7 @@ class MigrationClient else logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) - else if @migration_phase is "switch" + else if migration_phase is "switch" # As we deploy the "switch" phase new docupdaters will set the migration # flag for projects which have an empty queue in the old redis, and # write updates into the new redis. Existing docupdaters still in the @@ -175,7 +180,7 @@ class MigrationClient return callback(err) if err? logger.debug {key: key}, "switching to new redis because old queue is empty" return callback(null, @rclient_new) - else if @migration_phase is "rollback" + else if migration_phase is "rollback" # If we need to roll back gracefully we do the opposite of the "switch" # phase. We use the new redis when the migration key is set and the # queue exists in the new redis, but if the queue in the new redis is @@ -194,7 +199,7 @@ class MigrationClient logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) else - logger.error {key: key, migration_phase: @migration_phase}, "unknown migration phase" + logger.error {key: key, migration_phase: migration_phase}, "unknown migration phase" callback(new Error('invalid migration phase')) multi: () -> new Multi(@) From c2714f9ae9c863890fc299915af4134d49676b4c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:50:06 +0000 Subject: [PATCH 07/36] add acceptance tests for RedisMigrationManager --- .../coffee/RedisMigrationManagerTests.coffee | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee diff --git a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee new file mode 100644 index 0000000000..2684a4a3d8 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee @@ -0,0 +1,320 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +expect = chai.expect +async = require "async" +Settings = require('settings-sharelatex') +rclient_old = require("redis-sharelatex").createClient(Settings.redis.project_history) +rclient_new = require("redis-sharelatex").createClient(Settings.redis.new_project_history) +rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) +Keys = Settings.redis.documentupdater.key_schema +HistoryKeys = Settings.redis.history.key_schema +ProjectHistoryKeys = Settings.redis.project_history.key_schema +NewProjectHistoryKeys = Settings.redis.new_project_history.key_schema + +MockTrackChangesApi = require "./helpers/MockTrackChangesApi" +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" +DocUpdaterApp = require "./helpers/DocUpdaterApp" + +describe "RedisMigrationManager", -> + before (done) -> + @lines = ["one", "two", "three"] + @version = 42 + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: @version + DocUpdaterApp.ensureRunning(done) + + describe "when the migration phase is 'prepare' (default)", -> + + describe "when there is no migration flag", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should not set the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should keep the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 1 + done() + return null + + describe "when the migration phase is 'switch'", -> + before -> + Settings.redis.new_project_history.migration_phase = 'switch' + + describe "when the old queue is empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should set the migration flag for the project", (done) -> + rclient_new.get NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal "NEW" + done() + return null + + describe "when the old queue is not empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + rclient_old.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal "dummy-op" + JSON.parse(updates[1]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should not set the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should keep the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 1 + done() + return null + + describe "when the migration phase is 'rollback'", -> + before -> + Settings.redis.new_project_history.migration_phase = 'rollback' + + describe "when the old queue is empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the new queue is not empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + rclient_new.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal "dummy-op" + updates.length.should.equal 1 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should delete the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + From a638ef425146bb31e5ebfcd2b7d32e297cc75ab2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 13 Jan 2020 15:56:28 +0000 Subject: [PATCH 08/36] add comment about locking in redis migration --- .../app/coffee/RedisMigrationManager.coffee | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 96c8ee049c..a12593d03f 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -97,6 +97,14 @@ class Multi exec: (callback) -> # decide which client to use project_id = getProjectId(@queueKey) + # Put a lock around finding and updating the queue to avoid time-of-check to + # time-of-use problems. When running in the "switch" phase we need a lock to + # guarantee the order of operations. (Example: docupdater A sees an old + # queue at t=t0 and pushes onto it at t=t1, project history clears the queue + # between t0 and t1, and docupdater B sees the empty queue, sets the + # migration flag and pushes onto the new queue at t2. Without a lock it's + # possible to have t2 < t1 if docupdater A is slower than B - then there + # would be entries in the old and new queues, which we want to avoid.) LockManager.getLock project_id, (error, lockValue) => return callback(error) if error? releaseLock = (args...) => From 531d9b77b988e2991517c077979056369f3da402 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Jan 2020 09:49:41 +0000 Subject: [PATCH 09/36] add redislabs ca cert to repository --- services/document-updater/Dockerfile | 1 + services/document-updater/install_deps.sh | 2 + services/document-updater/redislabs_ca.pem | 77 ++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 services/document-updater/install_deps.sh create mode 100644 services/document-updater/redislabs_ca.pem diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 59f5e61889..2845544ae6 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -17,6 +17,7 @@ FROM node:6.9.5 COPY --from=app /app /app WORKDIR /app +RUN chmod 0755 ./install_deps.sh && ./install_deps.sh USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/install_deps.sh b/services/document-updater/install_deps.sh new file mode 100644 index 0000000000..8016ec6c85 --- /dev/null +++ b/services/document-updater/install_deps.sh @@ -0,0 +1,2 @@ +cp redislabs_ca.pem /usr/local/share/ca-certificates/redislabs_ca.crt +update-ca-certificates diff --git a/services/document-updater/redislabs_ca.pem b/services/document-updater/redislabs_ca.pem new file mode 100644 index 0000000000..a4af612d25 --- /dev/null +++ b/services/document-updater/redislabs_ca.pem @@ -0,0 +1,77 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 11859567854091286320 (0xa495a620ecc0b730) + Signature Algorithm: sha1WithRSAEncryption + Issuer: O=Garantia Data, CN=SSL Certification Authority + Validity + Not Before: Oct 1 12:14:55 2013 GMT + Not After : Sep 29 12:14:55 2023 GMT + Subject: O=Garantia Data, CN=SSL Certification Authority + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b6:6a:92:1f:c3:73:35:8f:26:7c:67:1c:b4:3b: + 40:bd:13:e0:1e:02:0c:a5:81:28:27:22:b2:b8:86: + 6c:0e:99:78:f5:95:36:8e:21:7c:a4:02:e8:9a:f3: + 7d:1f:b4:f3:53:5e:0f:a5:5c:59:48:b3:ae:67:7e: + 8e:d3:e1:21:8e:1c:f9:65:50:62:6e:4f:29:a3:7a: + 0d:3d:62:99:87:71:43:0e:da:a8:ee:63:d8:a5:02: + 12:1f:dc:ce:7a:4b:c5:e4:87:a1:3c:65:47:7e:04: + 43:01:76:f1:69:77:7a:0d:af:73:97:2d:f0:b8:d4: + dd:ea:33:59:59:37:81:be:da:97:1f:66:48:0d:92: + 82:6b:97:e6:51:10:6b:09:7e:fa:b4:a3:b0:14:ad: + 7a:66:36:04:3c:0e:a4:03:17:22:b7:44:c8:ff:dc: + 56:7f:26:92:f8:bf:04:3b:39:33:91:be:d3:d8:f4: + 81:f8:72:0b:34:56:31:0e:c7:9f:bd:6e:d5:ea:25: + 47:1c:15:c6:08:b7:4c:c9:fe:fe:f4:da:15:2a:b1: + 2a:38:1c:93:ac:ee:01:88:c1:44:f6:87:7b:ba:8b: + c4:73:6b:d5:2a:3f:31:cf:67:3f:2f:b7:c0:77:9b: + 17:06:c8:72:75:28:8f:06:e9:e2:77:2d:91:66:e3: + 6f:67 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 + X509v3 Authority Key Identifier: + keyid:FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 6d:9e:ad:78:70:44:06:bb:f9:93:81:b3:40:7a:5f:9e:c7:c3: + 27:75:47:89:1f:99:77:2c:d2:bb:5a:95:b3:e9:be:05:0b:4a: + 20:7e:4c:26:df:dc:46:e1:26:71:c6:ca:f7:42:63:5b:6f:95: + f7:cb:8d:d0:3b:1c:9d:0f:08:e9:fe:61:82:c1:03:4a:53:53: + f7:72:be:b3:7a:4a:ef:0d:b9:2e:72:b9:b9:ed:f6:66:f5:de: + 70:c6:62:8d:6b:9e:dd:18:45:fc:4d:fb:c0:cc:dd:f5:c8:56: + bd:37:f0:0d:f4:52:53:d7:d8:eb:b5:13:11:49:4f:43:19:b8: + 52:98:e9:9b:cb:74:8e:bf:d5:c6:e0:9a:0b:8c:94:08:4c:f8: + 38:4a:c9:5e:92:af:9e:bd:f4:b3:37:ce:a7:88:f3:5e:a9:66: + 69:51:10:44:d8:90:6a:fd:d6:ae:e4:06:95:c9:bb:f7:6d:1d: + a1:b1:83:56:46:bb:ac:3f:3c:2b:18:19:47:04:09:61:0d:60: + 3e:15:40:f7:7c:37:7d:89:8c:e7:ee:ea:f1:20:a0:40:30:7c: + f3:fe:de:81:a9:67:89:b7:7b:00:02:71:63:80:7a:7a:9f:95: + bf:9c:41:80:b8:3e:c1:7b:a9:b5:c3:99:16:96:ad:b2:a7:b4: + e9:59:de:7d +-----BEGIN CERTIFICATE----- +MIIDTzCCAjegAwIBAgIJAKSVpiDswLcwMA0GCSqGSIb3DQEBBQUAMD4xFjAUBgNV +BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTAeFw0xMzEwMDExMjE0NTVaFw0yMzA5MjkxMjE0NTVaMD4xFjAUBgNV +BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALZqkh/DczWP +JnxnHLQ7QL0T4B4CDKWBKCcisriGbA6ZePWVNo4hfKQC6JrzfR+081NeD6VcWUiz +rmd+jtPhIY4c+WVQYm5PKaN6DT1imYdxQw7aqO5j2KUCEh/cznpLxeSHoTxlR34E +QwF28Wl3eg2vc5ct8LjU3eozWVk3gb7alx9mSA2SgmuX5lEQawl++rSjsBStemY2 +BDwOpAMXIrdEyP/cVn8mkvi/BDs5M5G+09j0gfhyCzRWMQ7Hn71u1eolRxwVxgi3 +TMn+/vTaFSqxKjgck6zuAYjBRPaHe7qLxHNr1So/Mc9nPy+3wHebFwbIcnUojwbp +4nctkWbjb2cCAwEAAaNQME4wHQYDVR0OBBYEFP1whtcrydmW3ZJeuSoKZIKjze3w +MB8GA1UdIwQYMBaAFP1whtcrydmW3ZJeuSoKZIKjze3wMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAG2erXhwRAa7+ZOBs0B6X57Hwyd1R4kfmXcs0rta +lbPpvgULSiB+TCbf3EbhJnHGyvdCY1tvlffLjdA7HJ0PCOn+YYLBA0pTU/dyvrN6 +Su8NuS5yubnt9mb13nDGYo1rnt0YRfxN+8DM3fXIVr038A30UlPX2Ou1ExFJT0MZ +uFKY6ZvLdI6/1cbgmguMlAhM+DhKyV6Sr5699LM3zqeI816pZmlREETYkGr91q7k +BpXJu/dtHaGxg1ZGu6w/PCsYGUcECWENYD4VQPd8N32JjOfu6vEgoEAwfPP+3oGp +Z4m3ewACcWOAenqflb+cQYC4PsF7qbXDmRaWrbKntOlZ3n0= +-----END CERTIFICATE----- From ad58fe76b27ccfe3e49502407356f13edd09cc3f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Jan 2020 15:35:37 +0000 Subject: [PATCH 10/36] add tls settings --- .../config/settings.defaults.coffee | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index f890eb0f4a..0965c5bd3d 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -1,6 +1,7 @@ Path = require('path') http = require('http') http.globalAgent.maxSockets = 300 +fs = require('fs') module.exports = internal: @@ -44,6 +45,12 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + tls: if process.env['REDIS_CA_CERT'] && process.env['REDIS_CLIENT_CERT'] && process.env['REDIS_CLIENT_KEY'] + ca: fs.readFileSync(process.env['REDIS_CA_CERT']), + cert: fs.readFileSync( + process.env['REDIS_CLIENT_CERT'] + ), + key: fs.readFileSync(process.env['REDIS_CLIENT_KEY']) new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" @@ -54,6 +61,12 @@ module.exports = projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" migration_phase: "prepare" + tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] + ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), + cert: fs.readFileSync( + process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] + ), + key: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CLIENT_KEY']) redisOptions: keepAlive: 100 From 31324fb65a585767fe61a80f9287f5dcc0c900c6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Jan 2020 15:28:34 +0000 Subject: [PATCH 11/36] add environment variable for migration_phase setting --- services/document-updater/config/settings.defaults.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 0965c5bd3d..a14a6d29fb 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -60,7 +60,7 @@ module.exports = projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" - migration_phase: "prepare" + migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), cert: fs.readFileSync( From d5a2b96df9c8c0f0ece5852a7931d7b0abd920f4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 14:36:59 +0000 Subject: [PATCH 12/36] add note about deleting the migration key entries --- .../app/coffee/RedisMigrationManager.coffee | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index a12593d03f..31ffa492c0 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -45,6 +45,12 @@ async = require("async") # - when all docupdaters are setting the flag (and writing to the new redis), # finish the migration by writing all data to the new redis # +# Final stage +# +# When all the queues are migrated, remove the migration code and return to a +# single client pointing at the new redis. Delete the +# ProjectHistory:MigrationKey:* entries in the new redis. +# # Rollback # # Under the scheme above a project should only ever have data in the old redis From 626e19ed1adc0d393c9395e3e0c38395eeef0f78 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 15:46:54 +0000 Subject: [PATCH 13/36] add logging of migration phase at startup --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 31ffa492c0..af2eb7ad33 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -133,7 +133,7 @@ class MigrationClient @rclient_new = redis.createClient(@new_settings) @new_key_schema = new_settings.key_schema # check that migration phase is valid on startup - @getMigrationPhase() + logger.warn {migration_phase: @getMigrationPhase()}, "running with RedisMigrationManager" getMigrationPhase: () -> @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing From 544ae05212103925a7b55b160ac324b69057d9ba Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 16:22:26 +0000 Subject: [PATCH 14/36] added note about rollback --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index af2eb7ad33..d11024fc94 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -81,7 +81,8 @@ async = require("async") # - deploy project history with the "old queue" pointing to the new redis and # the "new queue" to the old redis to clear the new queue before processing # the new queue (i.e. add a rollback:true property in new_project_history in -# the project-history settings) +# the project-history settings via the environment variable +# MIGRATION_PHASE="rollback"). # - projects will now clear gradually from the new redis back to the old redis # - get a list of all the projects in the new redis and flush them, which will # cause the new queues to be cleared and the old redis to be used for those From 412eabc3063d37457c1f79e49e52f8848db1d9bc Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 Feb 2020 09:26:42 +0000 Subject: [PATCH 15/36] Revert "add tls settings" This reverts commit 72a4994cebab2731f99f0ada7a094c8a0acb3293. --- .../config/settings.defaults.coffee | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index a14a6d29fb..6724aa6a9a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -1,7 +1,6 @@ Path = require('path') http = require('http') http.globalAgent.maxSockets = 300 -fs = require('fs') module.exports = internal: @@ -45,12 +44,6 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - tls: if process.env['REDIS_CA_CERT'] && process.env['REDIS_CLIENT_CERT'] && process.env['REDIS_CLIENT_KEY'] - ca: fs.readFileSync(process.env['REDIS_CA_CERT']), - cert: fs.readFileSync( - process.env['REDIS_CLIENT_CERT'] - ), - key: fs.readFileSync(process.env['REDIS_CLIENT_KEY']) new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" @@ -61,12 +54,6 @@ module.exports = projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" - tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] - ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), - cert: fs.readFileSync( - process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] - ), - key: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CLIENT_KEY']) redisOptions: keepAlive: 100 From 922f237c393e441b0124c44e4beabcc31d3bc9fe Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 Feb 2020 09:26:59 +0000 Subject: [PATCH 16/36] Revert "add redislabs ca cert to repository" This reverts commit eb7419b0f45992228393086caf7ed6f66296801f. --- services/document-updater/Dockerfile | 1 - services/document-updater/install_deps.sh | 2 - services/document-updater/redislabs_ca.pem | 77 ---------------------- 3 files changed, 80 deletions(-) delete mode 100644 services/document-updater/install_deps.sh delete mode 100644 services/document-updater/redislabs_ca.pem diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 2845544ae6..59f5e61889 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -17,7 +17,6 @@ FROM node:6.9.5 COPY --from=app /app /app WORKDIR /app -RUN chmod 0755 ./install_deps.sh && ./install_deps.sh USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/install_deps.sh b/services/document-updater/install_deps.sh deleted file mode 100644 index 8016ec6c85..0000000000 --- a/services/document-updater/install_deps.sh +++ /dev/null @@ -1,2 +0,0 @@ -cp redislabs_ca.pem /usr/local/share/ca-certificates/redislabs_ca.crt -update-ca-certificates diff --git a/services/document-updater/redislabs_ca.pem b/services/document-updater/redislabs_ca.pem deleted file mode 100644 index a4af612d25..0000000000 --- a/services/document-updater/redislabs_ca.pem +++ /dev/null @@ -1,77 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 11859567854091286320 (0xa495a620ecc0b730) - Signature Algorithm: sha1WithRSAEncryption - Issuer: O=Garantia Data, CN=SSL Certification Authority - Validity - Not Before: Oct 1 12:14:55 2013 GMT - Not After : Sep 29 12:14:55 2023 GMT - Subject: O=Garantia Data, CN=SSL Certification Authority - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:b6:6a:92:1f:c3:73:35:8f:26:7c:67:1c:b4:3b: - 40:bd:13:e0:1e:02:0c:a5:81:28:27:22:b2:b8:86: - 6c:0e:99:78:f5:95:36:8e:21:7c:a4:02:e8:9a:f3: - 7d:1f:b4:f3:53:5e:0f:a5:5c:59:48:b3:ae:67:7e: - 8e:d3:e1:21:8e:1c:f9:65:50:62:6e:4f:29:a3:7a: - 0d:3d:62:99:87:71:43:0e:da:a8:ee:63:d8:a5:02: - 12:1f:dc:ce:7a:4b:c5:e4:87:a1:3c:65:47:7e:04: - 43:01:76:f1:69:77:7a:0d:af:73:97:2d:f0:b8:d4: - dd:ea:33:59:59:37:81:be:da:97:1f:66:48:0d:92: - 82:6b:97:e6:51:10:6b:09:7e:fa:b4:a3:b0:14:ad: - 7a:66:36:04:3c:0e:a4:03:17:22:b7:44:c8:ff:dc: - 56:7f:26:92:f8:bf:04:3b:39:33:91:be:d3:d8:f4: - 81:f8:72:0b:34:56:31:0e:c7:9f:bd:6e:d5:ea:25: - 47:1c:15:c6:08:b7:4c:c9:fe:fe:f4:da:15:2a:b1: - 2a:38:1c:93:ac:ee:01:88:c1:44:f6:87:7b:ba:8b: - c4:73:6b:d5:2a:3f:31:cf:67:3f:2f:b7:c0:77:9b: - 17:06:c8:72:75:28:8f:06:e9:e2:77:2d:91:66:e3: - 6f:67 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 - X509v3 Authority Key Identifier: - keyid:FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 6d:9e:ad:78:70:44:06:bb:f9:93:81:b3:40:7a:5f:9e:c7:c3: - 27:75:47:89:1f:99:77:2c:d2:bb:5a:95:b3:e9:be:05:0b:4a: - 20:7e:4c:26:df:dc:46:e1:26:71:c6:ca:f7:42:63:5b:6f:95: - f7:cb:8d:d0:3b:1c:9d:0f:08:e9:fe:61:82:c1:03:4a:53:53: - f7:72:be:b3:7a:4a:ef:0d:b9:2e:72:b9:b9:ed:f6:66:f5:de: - 70:c6:62:8d:6b:9e:dd:18:45:fc:4d:fb:c0:cc:dd:f5:c8:56: - bd:37:f0:0d:f4:52:53:d7:d8:eb:b5:13:11:49:4f:43:19:b8: - 52:98:e9:9b:cb:74:8e:bf:d5:c6:e0:9a:0b:8c:94:08:4c:f8: - 38:4a:c9:5e:92:af:9e:bd:f4:b3:37:ce:a7:88:f3:5e:a9:66: - 69:51:10:44:d8:90:6a:fd:d6:ae:e4:06:95:c9:bb:f7:6d:1d: - a1:b1:83:56:46:bb:ac:3f:3c:2b:18:19:47:04:09:61:0d:60: - 3e:15:40:f7:7c:37:7d:89:8c:e7:ee:ea:f1:20:a0:40:30:7c: - f3:fe:de:81:a9:67:89:b7:7b:00:02:71:63:80:7a:7a:9f:95: - bf:9c:41:80:b8:3e:c1:7b:a9:b5:c3:99:16:96:ad:b2:a7:b4: - e9:59:de:7d ------BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIJAKSVpiDswLcwMA0GCSqGSIb3DQEBBQUAMD4xFjAUBgNV -BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 -dGhvcml0eTAeFw0xMzEwMDExMjE0NTVaFw0yMzA5MjkxMjE0NTVaMD4xFjAUBgNV -BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 -dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALZqkh/DczWP -JnxnHLQ7QL0T4B4CDKWBKCcisriGbA6ZePWVNo4hfKQC6JrzfR+081NeD6VcWUiz -rmd+jtPhIY4c+WVQYm5PKaN6DT1imYdxQw7aqO5j2KUCEh/cznpLxeSHoTxlR34E -QwF28Wl3eg2vc5ct8LjU3eozWVk3gb7alx9mSA2SgmuX5lEQawl++rSjsBStemY2 -BDwOpAMXIrdEyP/cVn8mkvi/BDs5M5G+09j0gfhyCzRWMQ7Hn71u1eolRxwVxgi3 -TMn+/vTaFSqxKjgck6zuAYjBRPaHe7qLxHNr1So/Mc9nPy+3wHebFwbIcnUojwbp -4nctkWbjb2cCAwEAAaNQME4wHQYDVR0OBBYEFP1whtcrydmW3ZJeuSoKZIKjze3w -MB8GA1UdIwQYMBaAFP1whtcrydmW3ZJeuSoKZIKjze3wMAwGA1UdEwQFMAMBAf8w -DQYJKoZIhvcNAQEFBQADggEBAG2erXhwRAa7+ZOBs0B6X57Hwyd1R4kfmXcs0rta -lbPpvgULSiB+TCbf3EbhJnHGyvdCY1tvlffLjdA7HJ0PCOn+YYLBA0pTU/dyvrN6 -Su8NuS5yubnt9mb13nDGYo1rnt0YRfxN+8DM3fXIVr038A30UlPX2Ou1ExFJT0MZ -uFKY6ZvLdI6/1cbgmguMlAhM+DhKyV6Sr5699LM3zqeI816pZmlREETYkGr91q7k -BpXJu/dtHaGxg1ZGu6w/PCsYGUcECWENYD4VQPd8N32JjOfu6vEgoEAwfPP+3oGp -Z4m3ewACcWOAenqflb+cQYC4PsF7qbXDmRaWrbKntOlZ3n0= ------END CERTIFICATE----- From 2e178b0e2d48e26935af69848baa3c4c80d72c3f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 21 Feb 2020 14:06:58 +0000 Subject: [PATCH 17/36] resolve merge conflicts --- services/document-updater/Dockerfile | 13 ++++++--- .../document-updater/docker-compose.ci.yml | 19 +++++++------ services/document-updater/docker-compose.yml | 27 ++++++++----------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 59f5e61889..e538fb48d9 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -1,7 +1,14 @@ -FROM node:6.9.5 as app +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3.5 + +FROM node:10.19.0 as base WORKDIR /app +FROM base as app + #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ @@ -12,11 +19,9 @@ COPY . /app RUN npm run compile:all -FROM node:6.9.5 +FROM base COPY --from=app /app /app - -WORKDIR /app USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index d65f97b913..56f3e1c42e 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: @@ -13,7 +13,6 @@ services: environment: NODE_ENV: test - test_acceptance: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -26,14 +25,15 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test depends_on: - - mongo - - redis - - new_redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + new_redis: + condition: service_healthy user: node command: npm run test:acceptance:_run - - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -41,7 +41,6 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: image: redis @@ -49,4 +48,4 @@ services: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 31869acb50..805e3b0d06 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -1,13 +1,13 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: - image: node:6.9.5 + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - build: . + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -33,19 +33,14 @@ services: NODE_ENV: test user: node depends_on: - - mongo - - redis - - new_redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + new_redis: + condition: service_healthy command: npm run test:acceptance - tar: - build: . - image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - volumes: - - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . - user: root - redis: image: redis @@ -53,4 +48,4 @@ services: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 From b2d1718a2ecef4adb59a4b9b056446eecac49f10 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 23 Mar 2020 16:18:05 +0100 Subject: [PATCH 18/36] [misc] bump logger-sharelatex to 1.9.1 --- services/document-updater/package-lock.json | 6 +++--- services/document-updater/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 852c1670c1..410fa44a09 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -2055,9 +2055,9 @@ "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==" }, "logger-sharelatex": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.9.0.tgz", - "integrity": "sha512-yVTuha82047IiMOQLgQHCZGKkJo6I2+2KtiFKpgkIooR2yZaoTEvAeoMwBesSDSpGUpvUJ/+9UI+PmRyc+PQKQ==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.9.1.tgz", + "integrity": "sha512-9s6JQnH/PN+Js2CmI8+J3MQCTNlRzP2Dh4pcekXrV6Jm5J4HzyPi+6d3zfBskZ4NBmaUVw9hC4p5dmdaRmh4mQ==", "requires": { "@google-cloud/logging-bunyan": "^2.0.0", "@overleaf/o-error": "^2.0.0", diff --git a/services/document-updater/package.json b/services/document-updater/package.json index e01ebbbb8d..b0277ecea1 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -25,7 +25,7 @@ "coffee-script": "~1.7.0", "express": "3.11.0", "lodash": "^4.17.13", - "logger-sharelatex": "^1.7.0", + "logger-sharelatex": "^1.9.1", "metrics-sharelatex": "^2.5.1", "mongojs": "^2.6.0", "redis-sharelatex": "^1.0.11", From e293d86c148a93b039ef3a084d84e3a9a0983229 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 25 Mar 2020 12:15:16 +0000 Subject: [PATCH 19/36] add metric for project history queue --- .../app/coffee/ProjectHistoryRedisManager.coffee | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee index 1cc80ea722..5763aa7ed3 100644 --- a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee +++ b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee @@ -2,9 +2,13 @@ Settings = require('settings-sharelatex') projectHistoryKeys = Settings.redis?.project_history?.key_schema rclient = require("redis-sharelatex").createClient(Settings.redis.documentupdater) logger = require('logger-sharelatex') +metrics = require('./Metrics') module.exports = ProjectHistoryRedisManager = queueOps: (project_id, ops..., callback = (error, projectUpdateCount) ->) -> + # Record metric for ops pushed onto queue + for op in ops + metrics.summary "redis.projectHistoryOps", op.length, {status: "push"} multi = rclient.multi() # Push the ops onto the project history queue multi.rpush projectHistoryKeys.projectHistoryOps({project_id}), ops... From 891fcc696bcef5d5606dd4086d041eb55c3ccff0 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 25 Mar 2020 12:15:35 +0000 Subject: [PATCH 20/36] add metric for pending updates queue --- .../document-updater/app/coffee/RealTimeRedisManager.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/document-updater/app/coffee/RealTimeRedisManager.coffee b/services/document-updater/app/coffee/RealTimeRedisManager.coffee index d04f2304d3..6fd48033da 100644 --- a/services/document-updater/app/coffee/RealTimeRedisManager.coffee +++ b/services/document-updater/app/coffee/RealTimeRedisManager.coffee @@ -5,6 +5,7 @@ Keys = Settings.redis.documentupdater.key_schema logger = require('logger-sharelatex') os = require "os" crypto = require "crypto" +metrics = require('./Metrics') HOST = os.hostname() RND = crypto.randomBytes(4).toString('hex') # generate a random key for this process @@ -27,6 +28,8 @@ module.exports = RealTimeRedisManager = catch e return callback e updates.push update + # record metric for updates removed from queue + metrics.summary "redis.pendingUpdates", jsonUpdate.length, {status: "pop"} callback error, updates getUpdatesLength: (doc_id, callback)-> From 1a0550364dc0fa88dd5d18004632926da14cd66f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 25 Mar 2020 13:39:37 +0000 Subject: [PATCH 21/36] add metric for getdoc bytes --- services/document-updater/app/coffee/RedisManager.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index ca4151d299..6b4dc20257 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -41,6 +41,7 @@ module.exports = RedisManager = logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message return callback(error) docHash = RedisManager._computeHash(docLines) + metrics.summary "redis.setDoc", docLines.length, {status: "set"} logger.log {project_id, doc_id, version, docHash, pathname, projectHistoryId}, "putting doc in redis" RedisManager._serializeRanges ranges, (error, ranges) -> if error? From fcb72b9bf7a6a0974c7f971c9ba2864f75b287a3 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 25 Mar 2020 14:27:04 +0000 Subject: [PATCH 22/36] update tests --- services/document-updater/package-lock.json | 12 ++++++------ services/document-updater/package.json | 2 +- .../ProjectHistoryRedisManagerTests.coffee | 1 + .../RealTimeRedisManagerTests.coffee | 15 ++++++++------- .../coffee/RedisManager/RedisManagerTests.coffee | 1 + 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 852c1670c1..83448c1c77 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -853,9 +853,9 @@ } }, "acorn": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.0.tgz", - "integrity": "sha512-gac8OEcQ2Li1dxIEWGZzsp2BitJxwkwcOm0zHAJLcPJaVvm58FRnk6RkuLRpU1EujipU2ZFODv2P9DLMfnV8mw==" + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" }, "agent-base": { "version": "6.0.0", @@ -2183,9 +2183,9 @@ "integrity": "sha512-2403MfnVypWSNIEpmQ26/ObZ5kSUx37E8NHRvriw0+I8Sne7k0HGuLGCk0OrCqURh4UIygD0cSsYq+Ll+kzNqA==" }, "metrics-sharelatex": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.5.1.tgz", - "integrity": "sha512-C2gmkl/tUnq3IlSX/x3dixGhdvfD6H9FR9mBf9lnkeyy2arafxhCU6u+1IQj6byjBM7vGpYHyjwWnmoi3Vb+ZQ==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.6.2.tgz", + "integrity": "sha512-bOLfkSCexiPgB96hdXhoOWyvvrwscgjeZPEqdcJ7BTGxY59anzvymNf5hTGJ1RtS4sblDKxITw3L5a+gYKhRYQ==", "requires": { "@google-cloud/debug-agent": "^3.0.0", "@google-cloud/profiler": "^0.2.3", diff --git a/services/document-updater/package.json b/services/document-updater/package.json index e01ebbbb8d..72bb0cd868 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -26,7 +26,7 @@ "express": "3.11.0", "lodash": "^4.17.13", "logger-sharelatex": "^1.7.0", - "metrics-sharelatex": "^2.5.1", + "metrics-sharelatex": "^2.6.2", "mongojs": "^2.6.0", "redis-sharelatex": "^1.0.11", "request": "^2.47.0", diff --git a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee index a93545b250..9810b77d5f 100644 --- a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee @@ -26,6 +26,7 @@ describe "ProjectHistoryRedisManager", -> createClient: () => @rclient "logger-sharelatex": log:-> + "./Metrics": @metrics = { summary: sinon.stub()} globals: JSON: @JSON = JSON diff --git a/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee index 1d97779bfa..13e532736e 100644 --- a/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee @@ -11,7 +11,7 @@ describe "RealTimeRedisManager", -> auth: () -> exec: sinon.stub() @rclient.multi = () => @rclient - @pubsubClient = + @pubsubClient = publish: sinon.stub() @RealTimeRedisManager = SandboxedModule.require modulePath, requires: "redis-sharelatex": createClient: (config) => if (config.name is 'pubsub') then @pubsubClient else @rclient @@ -25,11 +25,12 @@ describe "RealTimeRedisManager", -> "logger-sharelatex": { log: () -> } "crypto": @crypto = { randomBytes: sinon.stub().withArgs(4).returns(Buffer.from([0x1, 0x2, 0x3, 0x4])) } "os": @os = {hostname: sinon.stub().returns("somehost")} + "./Metrics": @metrics = { summary: sinon.stub()} @doc_id = "doc-id-123" @project_id = "project-id-123" @callback = sinon.stub() - + describe "getPendingUpdatesForDoc", -> beforeEach -> @rclient.lrange = sinon.stub() @@ -44,7 +45,7 @@ describe "RealTimeRedisManager", -> @jsonUpdates = @updates.map (update) -> JSON.stringify update @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates]) @RealTimeRedisManager.getPendingUpdatesForDoc @doc_id, @callback - + it "should get the pending updates", -> @rclient.lrange .calledWith("PendingUpdates:#{@doc_id}", 0, 7) @@ -75,10 +76,10 @@ describe "RealTimeRedisManager", -> beforeEach -> @rclient.llen = sinon.stub().yields(null, @length = 3) @RealTimeRedisManager.getUpdatesLength @doc_id, @callback - + it "should look up the length", -> @rclient.llen.calledWith("PendingUpdates:#{@doc_id}").should.equal true - + it "should return the length", -> @callback.calledWith(null, @length).should.equal true @@ -86,6 +87,6 @@ describe "RealTimeRedisManager", -> beforeEach -> @message_id = "doc:somehost:01020304-0" @RealTimeRedisManager.sendData({op: "thisop"}) - + it "should send the op with a message id", -> - @pubsubClient.publish.calledWith("applied-ops", JSON.stringify({op:"thisop",_id:@message_id})).should.equal true \ No newline at end of file + @pubsubClient.publish.calledWith("applied-ops", JSON.stringify({op:"thisop",_id:@message_id})).should.equal true diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index b666163762..2ac8ac9c16 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -48,6 +48,7 @@ describe "RedisManager", -> createClient: () => @rclient "./Metrics": @metrics = inc: sinon.stub() + summary: sinon.stub() Timer: class Timer constructor: () -> this.start = new Date() From 17c2add0cf55648aae24dd85643ef8c97eb44be2 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 30 Mar 2020 11:31:43 +0200 Subject: [PATCH 23/36] [misc] track redis pub/sub payload sizes on publish --- .../app/coffee/RealTimeRedisManager.coffee | 8 ++++++-- .../RealTimeRedisManager/RealTimeRedisManagerTests.coffee | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/RealTimeRedisManager.coffee b/services/document-updater/app/coffee/RealTimeRedisManager.coffee index 6fd48033da..775132f1b6 100644 --- a/services/document-updater/app/coffee/RealTimeRedisManager.coffee +++ b/services/document-updater/app/coffee/RealTimeRedisManager.coffee @@ -39,9 +39,13 @@ module.exports = RealTimeRedisManager = # create a unique message id using a counter message_id = "doc:#{HOST}:#{RND}-#{COUNT++}" data?._id = message_id + + blob = JSON.stringify(data) + metrics.summary "redis.publish.applied-ops", blob.length + # publish on separate channels for individual projects and docs when # configured (needs realtime to be configured for this too). if Settings.publishOnIndividualChannels - pubsubClient.publish "applied-ops:#{data.doc_id}", JSON.stringify(data) + pubsubClient.publish "applied-ops:#{data.doc_id}", blob else - pubsubClient.publish "applied-ops", JSON.stringify(data) + pubsubClient.publish "applied-ops", blob diff --git a/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee index 13e532736e..429a03b971 100644 --- a/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RealTimeRedisManager/RealTimeRedisManagerTests.coffee @@ -90,3 +90,6 @@ describe "RealTimeRedisManager", -> it "should send the op with a message id", -> @pubsubClient.publish.calledWith("applied-ops", JSON.stringify({op:"thisop",_id:@message_id})).should.equal true + + it "should track the payload size", -> + @metrics.summary.calledWith("redis.publish.applied-ops", JSON.stringify({op:"thisop",_id:@message_id}).length).should.equal true From c2b050e2869aa039f888dc66a69c2c3d096e175d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 31 Mar 2020 10:21:50 +0100 Subject: [PATCH 24/36] bump redis to 1.0.12 --- services/document-updater/package-lock.json | 14 +++++++------- services/document-updater/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 410fa44a09..09c2554bac 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -1876,9 +1876,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ioredis": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz", - "integrity": "sha512-94W+X//GHM+1GJvDk6JPc+8qlM7Dul+9K+lg3/aHixPN7ZGkW6qlvX0DG6At9hWtH2v3B32myfZqWoANUJYGJA==", + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.16.1.tgz", + "integrity": "sha512-g76Mm9dE7BLuewncu1MimGZw5gDDjDwjoRony/VoSxSJEKAhuYncDEwYKYjtHi2NWsTNIB6XXRjE64uVa/wpKQ==", "requires": { "cluster-key-slot": "^1.1.0", "debug": "^4.1.1", @@ -2792,13 +2792,13 @@ } }, "redis-sharelatex": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.11.tgz", - "integrity": "sha512-rKXPVLmFC9ycpRc5e4rULOwi9DB0LqRcWEiUxQuJNSVgcqCxpGqVw+zwivo+grk3G2tGpduh3/8y+4KVHWOntw==", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.12.tgz", + "integrity": "sha512-Z+LDGaRNgZ+NiDaCC/R0N3Uy6SCtbKXqiXlvCwAbIQRSZUc69OVx/cQ3i5qDF7zeERhh+pnTd+zGs8nVfa5p+Q==", "requires": { "async": "^2.5.0", "coffee-script": "1.8.0", - "ioredis": "~4.14.1", + "ioredis": "~4.16.1", "redis-sentinel": "0.1.1", "underscore": "1.7.0" }, diff --git a/services/document-updater/package.json b/services/document-updater/package.json index b0277ecea1..c27bc47799 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -28,7 +28,7 @@ "logger-sharelatex": "^1.9.1", "metrics-sharelatex": "^2.5.1", "mongojs": "^2.6.0", - "redis-sharelatex": "^1.0.11", + "redis-sharelatex": "^1.0.12", "request": "^2.47.0", "requestretry": "^4.1.0", "settings-sharelatex": "^1.1.0" From 00b11bda96c6c1d2d4ec49e68741f5ed038d5913 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 1 Apr 2020 14:50:55 +0100 Subject: [PATCH 25/36] use separate loop for pendingUpdates metric --- .../document-updater/app/coffee/RealTimeRedisManager.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/RealTimeRedisManager.coffee b/services/document-updater/app/coffee/RealTimeRedisManager.coffee index 6fd48033da..1bfa509078 100644 --- a/services/document-updater/app/coffee/RealTimeRedisManager.coffee +++ b/services/document-updater/app/coffee/RealTimeRedisManager.coffee @@ -21,6 +21,9 @@ module.exports = RealTimeRedisManager = multi.exec (error, replys) -> return callback(error) if error? jsonUpdates = replys[0] + for jsonUpdate in jsonUpdates + # record metric for each update removed from queue + metrics.summary "redis.pendingUpdates", jsonUpdate.length, {status: "pop"} updates = [] for jsonUpdate in jsonUpdates try @@ -28,8 +31,6 @@ module.exports = RealTimeRedisManager = catch e return callback e updates.push update - # record metric for updates removed from queue - metrics.summary "redis.pendingUpdates", jsonUpdate.length, {status: "pop"} callback error, updates getUpdatesLength: (doc_id, callback)-> From 3a8c362fbaa9f2e875db6322006767a61d39b5e9 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 1 Apr 2020 15:59:25 +0100 Subject: [PATCH 26/36] add doclines set/del metric --- services/document-updater/app/coffee/RedisManager.coffee | 8 ++++++-- .../unit/coffee/RedisManager/RedisManagerTests.coffee | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 6b4dc20257..d784be8495 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -41,7 +41,7 @@ module.exports = RedisManager = logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message return callback(error) docHash = RedisManager._computeHash(docLines) - metrics.summary "redis.setDoc", docLines.length, {status: "set"} + metrics.summary "redis.docLines", docLines.length, {status: "set"} logger.log {project_id, doc_id, version, docHash, pathname, projectHistoryId}, "putting doc in redis" RedisManager._serializeRanges ranges, (error, ranges) -> if error? @@ -74,6 +74,7 @@ module.exports = RedisManager = _callback() multi = rclient.multi() + multi.strlen keys.docLines(doc_id:doc_id) multi.del keys.docLines(doc_id:doc_id) multi.del keys.projectKey(doc_id:doc_id) multi.del keys.docVersion(doc_id:doc_id) @@ -85,8 +86,11 @@ module.exports = RedisManager = multi.del keys.unflushedTime(doc_id:doc_id) multi.del keys.lastUpdatedAt(doc_id: doc_id) multi.del keys.lastUpdatedBy(doc_id: doc_id) - multi.exec (error) -> + multi.exec (error, response) -> return callback(error) if error? + length = response?[0] + if length > 0 + metrics.summary "redis.docLines", length, {status: "del"} multi = rclient.multi() multi.srem keys.docsInProject(project_id:project_id), doc_id multi.del keys.projectState(project_id:project_id) diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index 2ac8ac9c16..254de8d0a7 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -671,11 +671,17 @@ describe "RedisManager", -> describe "removeDocFromMemory", -> beforeEach (done) -> + @multi.strlen = sinon.stub() @multi.del = sinon.stub() @multi.srem = sinon.stub() @multi.exec.yields() @RedisManager.removeDocFromMemory @project_id, @doc_id, done + it "should check the length of the current doclines", -> + @multi.strlen + .calledWith("doclines:#{@doc_id}") + .should.equal true + it "should delete the lines", -> @multi.del .calledWith("doclines:#{@doc_id}") From 2b72ec49a10e4d599858da32122cc0817f1bebac Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 2 Apr 2020 11:33:52 +0100 Subject: [PATCH 27/36] add comments for redis metrics --- services/document-updater/app/coffee/RedisManager.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index d784be8495..fa2e312f33 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -41,6 +41,7 @@ module.exports = RedisManager = logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message return callback(error) docHash = RedisManager._computeHash(docLines) + # record bytes sent to redis metrics.summary "redis.docLines", docLines.length, {status: "set"} logger.log {project_id, doc_id, version, docHash, pathname, projectHistoryId}, "putting doc in redis" RedisManager._serializeRanges ranges, (error, ranges) -> @@ -90,6 +91,7 @@ module.exports = RedisManager = return callback(error) if error? length = response?[0] if length > 0 + # record bytes freed in redis metrics.summary "redis.docLines", length, {status: "del"} multi = rclient.multi() multi.srem keys.docsInProject(project_id:project_id), doc_id From beb3691795e3f7432620d4a1114a8392b0cfe94f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 2 Apr 2020 11:34:19 +0100 Subject: [PATCH 28/36] add metrics for redis get/update --- services/document-updater/app/coffee/RedisManager.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index fa2e312f33..f212cfbadc 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -132,6 +132,9 @@ module.exports = RedisManager = if timeSpan > MAX_REDIS_REQUEST_LENGTH error = new Error("redis getDoc exceeded timeout") return callback(error) + # record bytes loaded from redis + if docLines? + metrics.summary "redis.docLines", docLines.length, {status: "get"} # check sha1 hash value if present if docLines? and storedHash? computedHash = RedisManager._computeHash(docLines) @@ -247,7 +250,8 @@ module.exports = RedisManager = opVersions = appliedOps.map (op) -> op?.v logger.log doc_id: doc_id, version: newVersion, hash: newHash, op_versions: opVersions, "updating doc in redis" - + # record bytes sent to redis in update + metrics.summary "redis.docLines", newDocLines.length, {status: "update"} RedisManager._serializeRanges ranges, (error, ranges) -> if error? logger.error {err: error, doc_id}, error.message From c095feaa06a6ac9b89114f96279063549e172e6d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Apr 2020 10:43:53 +0100 Subject: [PATCH 29/36] upgrade logger-sharelatex --- services/document-updater/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/package.json b/services/document-updater/package.json index 72bb0cd868..0fb47d191f 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -25,7 +25,7 @@ "coffee-script": "~1.7.0", "express": "3.11.0", "lodash": "^4.17.13", - "logger-sharelatex": "^1.7.0", + "logger-sharelatex": "^1.9.1", "metrics-sharelatex": "^2.6.2", "mongojs": "^2.6.0", "redis-sharelatex": "^1.0.11", From 8e210fe44140ca5fee302410924b2282068f72e2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:41:30 +0100 Subject: [PATCH 30/36] update unit tests --- .../coffee/ApplyingUpdatesToADocTests.coffee | 13 +++++++------ .../ApplyingUpdatesToProjectStructureTests.coffee | 14 +++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee index 0b28dea7a7..489f8d98eb 100644 --- a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee @@ -4,7 +4,8 @@ chai.should() expect = chai.expect async = require "async" Settings = require('settings-sharelatex') -rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) +rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) # note: this is track changes, not project-history +rclient_project_history = require("redis-sharelatex").createClient(Settings.redis.project_history) rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) Keys = Settings.redis.documentupdater.key_schema HistoryKeys = Settings.redis.history.key_schema @@ -65,14 +66,14 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null it "should set the first op timestamp", (done) -> - rclient_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => + rclient_project_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => throw error if error? result.should.be.within(@startTime, Date.now()) @firstOpTimestamp = result @@ -90,7 +91,7 @@ describe "Applying updates to a doc", -> return null it "should not change the first op timestamp", (done) -> - rclient_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => + rclient_project_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => throw error if error? result.should.equal @firstOpTimestamp done() @@ -130,7 +131,7 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null @@ -164,7 +165,7 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee index cbb9fd9ea5..e18aa2e6a1 100644 --- a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee @@ -2,7 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() Settings = require('settings-sharelatex') -rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) +rclient_project_history = require("redis-sharelatex").createClient(Settings.redis.project_history) ProjectHistoryKeys = Settings.redis.project_history.key_schema MockProjectHistoryApi = require "./helpers/MockProjectHistoryApi" @@ -30,7 +30,7 @@ describe "Applying updates to a project's structure", -> setTimeout done, 200 it "should push the applied file renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -61,7 +61,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -97,7 +97,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -141,7 +141,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -194,7 +194,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the file addition to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -222,7 +222,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the doc addition to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) From af93193d6e05fca73b5f3f658ac2e6e93078240f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:43:48 +0100 Subject: [PATCH 31/36] remove new_project_history and use project_history instead --- .../app/coffee/ProjectHistoryRedisManager.coffee | 3 +-- .../document-updater/config/settings.defaults.coffee | 11 ----------- .../ProjectHistoryRedisManagerTests.coffee | 2 -- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee index 31842a1c8f..af75487a90 100644 --- a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee +++ b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee @@ -1,7 +1,6 @@ Settings = require('settings-sharelatex') projectHistoryKeys = Settings.redis?.project_history?.key_schema -#rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) -rclient = require("./RedisMigrationManager").createClient(Settings.redis.project_history, Settings.redis.new_project_history) +rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) logger = require('logger-sharelatex') metrics = require('./Metrics') diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 6724aa6a9a..2fb398251a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -37,23 +37,12 @@ module.exports = docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}" project_history: - port: process.env["HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" - host: process.env["HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" - password: process.env["HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") - key_schema: - projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" - projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - - new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" host: process.env["NEW_HISTORY_REDIS_HOST"] password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" - migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" redisOptions: keepAlive: 100 diff --git a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee index 7199002162..9810b77d5f 100644 --- a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee @@ -24,8 +24,6 @@ describe "ProjectHistoryRedisManager", -> } "redis-sharelatex": createClient: () => @rclient - "./RedisMigrationManager": - createClient: () => @rclient "logger-sharelatex": log:-> "./Metrics": @metrics = { summary: sinon.stub()} From 248edc03faaa36e6464e28b79d1fcc464ef6fe8e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:44:19 +0100 Subject: [PATCH 32/36] add comment about the two history clients --- services/document-updater/app/coffee/RedisManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index f212cfbadc..3eeed78ffb 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -23,7 +23,7 @@ MEGABYTES = 1024 * 1024 MAX_RANGES_SIZE = 3 * MEGABYTES keys = Settings.redis.documentupdater.key_schema -historyKeys = Settings.redis.history.key_schema +historyKeys = Settings.redis.history.key_schema # note: this is track changes, not project-history module.exports = RedisManager = rclient: rclient From a51f61a5558a276585b1503168d7f8cf5b538c4a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:48:47 +0100 Subject: [PATCH 33/36] remove redis migration code --- .../app/coffee/RedisMigrationManager.coffee | 224 ------------ .../coffee/RedisMigrationManagerTests.coffee | 320 ------------------ 2 files changed, 544 deletions(-) delete mode 100644 services/document-updater/app/coffee/RedisMigrationManager.coffee delete mode 100644 services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee deleted file mode 100644 index d11024fc94..0000000000 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ /dev/null @@ -1,224 +0,0 @@ -logger = require "logger-sharelatex" -Settings = require "settings-sharelatex" -redis = require("redis-sharelatex") -LockManager = require("./LockManager") -metrics = require "./Metrics" -async = require("async") - -# The aim is to migrate the project history queues -# ProjectHistory:Ops:{project_id} from the existing redis to a new redis. -# -# This has to work in conjunction with changes in project history. -# -# The basic principles are: -# -# - project history is modified to read from an 'old' and 'new' queue. It reads -# from the 'old' queue first, and when that queue is empty it reads from the -# 'new' queue. -# - docupdater will migrate to writing to the 'new' queue when the 'old' queue -# is empty. -# -# Some facts about the update process: -# -# - project history has a lock on the project-id, so each queue is processed in -# isolation -# - docupdaters take a lock on the doc_id but not the project_id, therefore -# multiple docupdaters can be appending to the queue for a project at the same -# time (provided they updates for individual docs are in order this is -# acceptable) -# - as we want to do this without shutting down the site, we have to take into -# account that different versions of the code will be running while deploys -# are in progress. -# -# The migration has to be carried out with the following constraint: -# -# - a docupdater should never write to the "old" queue when there are updates in -# the "new" queue (there is a strict ordering on the versions, new > old) -# -# The deployment process for docupdater will be -# -# - add a project-level lock to the queuing in docupdater -# - use a per-project migration flag to determine when to write to the new redis -# - set the migration flag for projects with an empty queue in the old redis -# - when all docupdaters respect the flag, make a new deploy which starts to set -# the flag -# - when all docupdaters are setting the flag (and writing to the new redis), -# finish the migration by writing all data to the new redis -# -# Final stage -# -# When all the queues are migrated, remove the migration code and return to a -# single client pointing at the new redis. Delete the -# ProjectHistory:MigrationKey:* entries in the new redis. -# -# Rollback -# -# Under the scheme above a project should only ever have data in the old redis -# or the new redis, but never both at the same time. -# -# Two scenarios: -# -# Hard rollback -# -# If we want to roll back to the old redis immediately, we need to get the data -# out of the new queues and back into the old queues, before appending to the -# old queues again. The actions to do this are: -# -# - close the site -# - revert docupdater so it only writes to the original redis (there will now -# be some data in the new redis for some projects which we need to recover) -# - run a script to move the new queues back into the old redis -# - revert project history to only read from the original redis -# -# Graceful rollback -# -# If we are prepared to keep the new redis running, but not add new projects to -# it we can do the following: -# -# - deploy all docupdaters to update from the "switch" phase into the -# "rollback" phase (projects in the new redis will continue to send data -# there, project not yet migrated will continue to go to the old redis) -# - deploy project history with the "old queue" pointing to the new redis and -# the "new queue" to the old redis to clear the new queue before processing -# the new queue (i.e. add a rollback:true property in new_project_history in -# the project-history settings via the environment variable -# MIGRATION_PHASE="rollback"). -# - projects will now clear gradually from the new redis back to the old redis -# - get a list of all the projects in the new redis and flush them, which will -# cause the new queues to be cleared and the old redis to be used for those -# projects. - -getProjectId = (key) -> - key.match(/\{([0-9a-f]{24})\}/)[1] - -class Multi - constructor: (@migrationClient) -> - @command_list = [] - @queueKey = null - rpush: (args...) -> - @queueKey = args[0] - @updates_count = args.length - 1 - @command_list.push { command:'rpush', args: args} - setnx: (args...) -> - @command_list.push { command: 'setnx', args: args} - exec: (callback) -> - # decide which client to use - project_id = getProjectId(@queueKey) - # Put a lock around finding and updating the queue to avoid time-of-check to - # time-of-use problems. When running in the "switch" phase we need a lock to - # guarantee the order of operations. (Example: docupdater A sees an old - # queue at t=t0 and pushes onto it at t=t1, project history clears the queue - # between t0 and t1, and docupdater B sees the empty queue, sets the - # migration flag and pushes onto the new queue at t2. Without a lock it's - # possible to have t2 < t1 if docupdater A is slower than B - then there - # would be entries in the old and new queues, which we want to avoid.) - LockManager.getLock project_id, (error, lockValue) => - return callback(error) if error? - releaseLock = (args...) => - LockManager.releaseLock project_id, lockValue, (lockError) -> - return callback(lockError) if lockError? - callback(args...) - @migrationClient.findQueue @queueKey, (err, rclient) => - return releaseLock(err) if err? - # add metric for updates - dest = (if rclient == @migrationClient.rclient_new then "new" else "old") - metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} - multi = rclient.multi() - for entry in @command_list - multi[entry.command](entry.args...) - multi.exec releaseLock - -class MigrationClient - constructor: (@old_settings, @new_settings) -> - @rclient_old = redis.createClient(@old_settings) - @rclient_new = redis.createClient(@new_settings) - @new_key_schema = new_settings.key_schema - # check that migration phase is valid on startup - logger.warn {migration_phase: @getMigrationPhase()}, "running with RedisMigrationManager" - - getMigrationPhase: () -> - @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing - throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] - return @migration_phase - - getMigrationStatus: (key, migrationKey, callback) -> - async.series [ - (cb) => @rclient_new.exists migrationKey, cb - (cb) => @rclient_new.exists key, cb - (cb) => @rclient_old.exists key, cb - ], (err, result) -> - return callback(err) if err? - migrationKeyExists = result[0] > 0 - newQueueExists = result[1] > 0 - oldQueueExists = result[2] > 0 - callback(null, migrationKeyExists, newQueueExists, oldQueueExists) - - findQueue: (key, callback) -> - project_id = getProjectId(key) - migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - migration_phase = @getMigrationPhase() # allow setting migration phase while running for testing - @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => - return callback(err) if err? - # In all cases, if the migration key exists we must always write to the - # new redis, unless we are rolling back. - if migration_phase is "prepare" - # in this phase we prepare for the switch, when some docupdaters will - # start setting the migration flag. We monitor the migration key and - # write to the new redis if the key is present, but we do not set the - # migration key. At this point no writes will be going into the new - # redis. When all the docupdaters are in the "prepare" phase we can - # begin deploying the "switch" phase. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else if migration_phase is "switch" - # As we deploy the "switch" phase new docupdaters will set the migration - # flag for projects which have an empty queue in the old redis, and - # write updates into the new redis. Existing docupdaters still in the - # "prepare" phase will pick up the migration flag and write new updates - # into the new redis when appropriate. When this deploy is complete - # writes will be going into the new redis for projects with an empty - # queue in the old redis. We have to remain in the switch phase until - # all projects are flushed from the old redis. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - if oldQueueExists - logger.debug {project_id}, "using old client because old queue exists" - return callback(null, @rclient_old) - else - @rclient_new.setnx migrationKey, "NEW", (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to new redis because old queue is empty" - return callback(null, @rclient_new) - else if migration_phase is "rollback" - # If we need to roll back gracefully we do the opposite of the "switch" - # phase. We use the new redis when the migration key is set and the - # queue exists in the new redis, but if the queue in the new redis is - # empty we delete the migration key and send further updates to the old - # redis. - if migrationKeyExists - if newQueueExists - logger.debug {project_id}, "using new client because migration key exists and new queue is present" - return callback(null, @rclient_new) - else - @rclient_new.del migrationKey, (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to old redis in rollback phase because new queue is empty" - return callback(null, @rclient_old) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else - logger.error {key: key, migration_phase: migration_phase}, "unknown migration phase" - callback(new Error('invalid migration phase')) - multi: () -> - new Multi(@) - -module.exports = RedisMigrationManager = - createClient: (args...) -> - new MigrationClient(args...) diff --git a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee deleted file mode 100644 index 2684a4a3d8..0000000000 --- a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee +++ /dev/null @@ -1,320 +0,0 @@ -sinon = require "sinon" -chai = require("chai") -chai.should() -expect = chai.expect -async = require "async" -Settings = require('settings-sharelatex') -rclient_old = require("redis-sharelatex").createClient(Settings.redis.project_history) -rclient_new = require("redis-sharelatex").createClient(Settings.redis.new_project_history) -rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) -Keys = Settings.redis.documentupdater.key_schema -HistoryKeys = Settings.redis.history.key_schema -ProjectHistoryKeys = Settings.redis.project_history.key_schema -NewProjectHistoryKeys = Settings.redis.new_project_history.key_schema - -MockTrackChangesApi = require "./helpers/MockTrackChangesApi" -MockWebApi = require "./helpers/MockWebApi" -DocUpdaterClient = require "./helpers/DocUpdaterClient" -DocUpdaterApp = require "./helpers/DocUpdaterApp" - -describe "RedisMigrationManager", -> - before (done) -> - @lines = ["one", "two", "three"] - @version = 42 - @update = - doc: @doc_id - op: [{ - i: "one and a half\n" - p: 4 - }] - v: @version - DocUpdaterApp.ensureRunning(done) - - describe "when the migration phase is 'prepare' (default)", -> - - describe "when there is no migration flag", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'switch'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'switch' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should set the migration flag for the project", (done) -> - rclient_new.get NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal "NEW" - done() - return null - - describe "when the old queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_old.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - JSON.parse(updates[1]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'rollback'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'rollback' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the new queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_new.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - updates.length.should.equal 1 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should delete the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - From 61da130cf401ef9eb2fe892553b74a660298dec8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 13:50:39 +0100 Subject: [PATCH 34/36] keep maxRetriesPerRequest for project_history redis config --- services/document-updater/config/settings.defaults.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 2fb398251a..1f1c951e1f 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -40,6 +40,7 @@ module.exports = port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" host: process.env["NEW_HISTORY_REDIS_HOST"] password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" From 2e24d1670c84c64cfe153383348e2b4299085cf7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 13:51:14 +0100 Subject: [PATCH 35/36] remove old unused ioredis keepalive option --- services/document-updater/config/settings.defaults.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 1f1c951e1f..f387a56122 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -44,8 +44,6 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - redisOptions: - keepAlive: 100 lock: port: process.env["LOCK_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" From 79c934759c37e16e42cfa4ea16a015f21a9c1721 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 14:04:28 +0100 Subject: [PATCH 36/36] add default redis settings for project history --- services/document-updater/config/settings.defaults.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index f387a56122..0ced9eeedd 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -37,9 +37,9 @@ module.exports = docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}" project_history: - port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" - host: process.env["NEW_HISTORY_REDIS_HOST"] - password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + port: process.env["NEW_HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" + host: process.env["NEW_HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" + password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}"