From d0c5eb569807ff14315c688fac5275742b9a0163 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 13 Sep 2019 15:06:42 +0100 Subject: [PATCH 01/17] support migration of project history keys to separate redis instance --- .../coffee/ProjectHistoryRedisManager.coffee | 3 +- .../app/coffee/RedisMigrationManager.coffee | 199 ++++++++++++++++++ .../config/settings.defaults.coffee | 12 ++ services/document-updater/docker-compose.yml | 9 +- .../coffee/helpers/DocUpdaterApp.coffee | 4 +- .../ProjectHistoryRedisManagerTests.coffee | 2 + 6 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 services/document-updater/app/coffee/RedisMigrationManager.coffee diff --git a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee index 1cc80ea722..c6362c4fbd 100644 --- a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee +++ b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee @@ -1,6 +1,7 @@ Settings = require('settings-sharelatex') projectHistoryKeys = Settings.redis?.project_history?.key_schema -rclient = require("redis-sharelatex").createClient(Settings.redis.documentupdater) +#rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) +rclient = require("./RedisMigrationManager").createClient(Settings.redis.project_history, Settings.redis.new_project_history) logger = require('logger-sharelatex') module.exports = ProjectHistoryRedisManager = diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee new file mode 100644 index 0000000000..c00d7727af --- /dev/null +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -0,0 +1,199 @@ +logger = require "logger-sharelatex" +Settings = require "settings-sharelatex" +redis = require("redis-sharelatex") +LockManager = require("./LockManager") +async = require("async") + +# The aim is to migrate the project history queues +# ProjectHistory:Ops:{project_id} from the existing redis to a new redis. +# +# This has to work in conjunction with changes in project history. +# +# The basic principles are: +# +# - project history is modified to read from an 'old' and 'new' queue. It reads +# from the 'old' queue first, and when that queue is empty it reads from the +# 'new' queue. +# - docupdater will migrate to writing to the 'new' queue when the 'old' queue +# is empty. +# +# Some facts about the update process: +# +# - project history has a lock on the project-id, so each queue is processed in +# isolation +# - docupdaters take a lock on the doc_id but not the project_id, therefore +# multiple docupdaters can be appending to the queue for a project at the same +# time (provided they updates for individual docs are in order this is +# acceptable) +# - as we want to do this without shutting down the site, we have to take into +# account that different versions of the code will be running while deploys +# are in progress. +# +# The migration has to be carried out with the following constraint: +# +# - a docupdater should never write to the "old" queue when there are updates in +# the "new" queue (there is a strict ordering on the versions, new > old) +# +# The deployment process for docupdater will be +# +# - add a project-level lock to the queuing in docupdater +# - use a per-project migration flag to determine when to write to the new redis +# - set the migration flag for projects with an empty queue in the old redis +# - when all docupdaters respect the flag, make a new deploy which starts to set +# the flag +# - when all docupdaters are setting the flag (and writing to the new redis), +# finish the migration by writing all data to the new redis +# +# Rollback +# +# Under the scheme above a project should only ever have data in the old redis +# or the new redis, but never both at the same time. +# +# Two scenarios: +# +# Hard rollback +# +# If we want to roll back to the old redis immediately, we need to get the data +# out of the new queues and back into the old queues, before appending to the +# old queues again. The actions to do this are: +# +# - close the site +# - revert docupdater so it only writes to the original redis (there will now +# be some data in the new redis for some projects which we need to recover) +# - run a script to move the new queues back into the old redis +# - revert project history to only read from the original redis +# +# Graceful rollback +# +# If we are prepared to keep the new redis running, but not add new projects to +# it we can do the following: +# +# - deploy all docupdaters to update from the "switch" phase into the +# "rollback" phase (projects in the new redis will continue to send data +# there, project not yet migrated will continue to go to the old redis) +# - deploy project history with the "old queue" pointing to the new redis and +# the "new queue" to the old redis to clear the new queue before processing +# the new queue (i.e. add a rollback:true property in new_project_history in +# the project-history settings) +# - projects will now clear gradually from the new redis back to the old redis +# - get a list of all the projects in the new redis and flush them, which will +# cause the new queues to be cleared and the old redis to be used for those +# projects. + +getProjectId = (key) -> + key.match(/\{([0-9a-f]{24})\}/)[1] + +class Multi + constructor: (@migrationClient) -> + @command_list = [] + @queueKey = null + rpush: (args...) -> + @queueKey = args[0] + @command_list.push { command:'rpush', args: args} + setnx: (args...) -> + @command_list.push { command: 'setnx', args: args} + exec: (callback) -> + # decide which client to use + project_id = getProjectId(@queueKey) + LockManager.getLock project_id, (error, lockValue) => + return callback(error) if error? + releaseLock = (args...) => + LockManager.releaseLock project_id, lockValue, (lockError) -> + return callback(lockError) if lockError? + callback(args...) + @migrationClient.findQueue @queueKey, (err, rclient) => + return releaseLock(err) if err? + multi = rclient.multi() + for entry in @command_list + multi[entry.command](entry.args...) + multi.exec releaseLock + +class MigrationClient + constructor: (old_settings, new_settings) -> + @rclient_old = redis.createClient(old_settings) + @rclient_new = redis.createClient(new_settings) + @new_key_schema = new_settings.key_schema + @migration_phase = new_settings.migration_phase + throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'start', 'switch', 'complete'] + + getMigrationStatus: (key, migrationKey, callback) -> + async.series [ + (cb) => @rclient_new.exists migrationKey, cb + (cb) => @rclient_new.exists key, cb + (cb) => @rclient_old.exists key, cb + ], (err, result) -> + return callback(err) if err? + migrationKeyExists = result[0] > 0 + newQueueExists = result[1] > 0 + oldQueueExists = result[2] > 0 + callback(null, migrationKeyExists, newQueueExists, oldQueueExists) + + findQueue: (key, callback) -> + project_id = getProjectId(key) + migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) + + @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) -> + return callback(err) if err? + # In all cases, if the migration key exists we must always write to the + # new redis, unless we are rolling back. + if @migration_phase is "prepare" + # in this phase we prepare for the switch, when some docupdaters will + # start setting the migration flag. We monitor the migration key and + # write to the new redis if the key is present, but we do not set the + # migration key. At this point no writes will be going into the new + # redis. When all the docupdaters are in the "prepare" phase we can + # begin deploying the "switch" phase. + if migrationKeyExists + logger.debug {project_id}, "using new client because migration key exists" + return callback(null, @rclient_new) + else + logger.debug {project_id}, "using old client because migration key does not exist" + return callback(null, @rclient_old) + else if @migration_phase is "switch" + # As we deploy the "switch" phase new docupdaters will set the migration + # flag for projects which have an empty queue in the old redis, and + # write updates into the new redis. Existing docupdaters still in the + # "prepare" phase will pick up the migration flag and write new updates + # into the new redis when appropriate. When this deploy is complete + # writes will be going into the new redis for projects with an empty + # queue in the old redis. We have to remain in the switch phase until + # all projects are flushed from the old redis. + if migrationKeyExists + logger.debug {project_id}, "using new client because migration key exists" + return callback(null, @rclient_new) + else + if oldQueueExists + logger.debug {project_id}, "using old client because old queue exists" + return callback(null, @rclient_old) + else + @rclient_new.setnx migrationKey, "NEW", (err) => + return callback(err) if err? + logger.debug {key: key}, "switching to new redis because old queue is empty" + return callback(null, @rclient_new) + else if @migration_phase is "rollback" + # If we need to roll back gracefully we do the opposite of the "switch" + # phase. We use the new redis when the migration key is set and the + # queue exists in the new redis, but if the queue in the new redis is + # empty we delete the migration key and send further updates to the old + # redis. + if migrationKeyExists + if newQueueExists + logger.debug {project_id}, "using new client because migration key exists and new queue is present" + return callback(null, @rclient_new) + else + @rclient_new.del migrationKey, (err) => + return callback(err) if err? + logger.debug {key: key}, "switching to old redis in rollback phase because new queue is empty" + return callback(null, @rclient_old) + else + logger.debug {project_id}, "using old client because migration key does not exist" + return callback(null, @rclient_old) + else + logger.error {key: key}, "unknown migration phase" + callback(new Error('invalid migration phase')) + multi: () -> + new Multi(@) + +module.exports = RedisMigrationManager = + createClient: (args...) -> + new MigrationClient(args...) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 9eebe86005..f890eb0f4a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -45,6 +45,18 @@ module.exports = projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + new_project_history: + port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" + host: process.env["NEW_HISTORY_REDIS_HOST"] + password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + key_schema: + projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" + projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" + migration_phase: "prepare" + redisOptions: + keepAlive: 100 + lock: port: process.env["LOCK_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" host: process.env["LOCK_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 6dc90009ca..31869acb50 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + NEW_HISTORY_REDIS_HOST: new_redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -34,10 +35,9 @@ services: depends_on: - mongo - redis + - new_redis command: npm run test:acceptance - - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -49,7 +49,8 @@ services: redis: image: redis + new_redis: + image: redis + mongo: image: mongo:3.4 - - diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee index 9819f9f99e..0f77199e73 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterApp.coffee @@ -13,8 +13,8 @@ module.exports = else @initing = true @callbacks.push callback - app.listen 3003, "localhost", (error) => + app.listen 3003, "localhost", (error) => throw error if error? @running = true for callback in @callbacks - callback() \ No newline at end of file + callback() diff --git a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee index a93545b250..6748f87af9 100644 --- a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee @@ -24,6 +24,8 @@ describe "ProjectHistoryRedisManager", -> } "redis-sharelatex": createClient: () => @rclient + "./RedisMigrationManager": + createClient: () => @rclient "logger-sharelatex": log:-> globals: From a85dffbcefad8d52e8508541eda4ee76a39471a7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 09:27:00 +0000 Subject: [PATCH 02/17] fix acceptance tests --- services/document-updater/docker-compose.ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index c78d90e8ed..d65f97b913 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -20,6 +20,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + NEW_HISTORY_REDIS_HOST: new_redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -27,6 +28,7 @@ services: depends_on: - mongo - redis + - new_redis user: node command: npm run test:acceptance:_run @@ -43,5 +45,8 @@ services: redis: image: redis + new_redis: + image: redis + mongo: image: mongo:3.4 From a2e63d009ee16a9064c0cb5ce3fd2ad7c9411812 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 09:55:26 +0000 Subject: [PATCH 03/17] fix migration phase check --- .../app/coffee/RedisMigrationManager.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index c00d7727af..f1ec00cf5c 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -114,7 +114,7 @@ class MigrationClient @rclient_new = redis.createClient(new_settings) @new_key_schema = new_settings.key_schema @migration_phase = new_settings.migration_phase - throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'start', 'switch', 'complete'] + throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] getMigrationStatus: (key, migrationKey, callback) -> async.series [ @@ -132,7 +132,7 @@ class MigrationClient project_id = getProjectId(key) migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) -> + @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => return callback(err) if err? # In all cases, if the migration key exists we must always write to the # new redis, unless we are rolling back. @@ -189,7 +189,7 @@ class MigrationClient logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) else - logger.error {key: key}, "unknown migration phase" + logger.error {key: key, migration_phase: @migration_phase}, "unknown migration phase" callback(new Error('invalid migration phase')) multi: () -> new Multi(@) From 97cbf461601add8ac9c024b9cbc189afe3fdc26b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 16 Dec 2019 11:46:35 +0000 Subject: [PATCH 04/17] add metrics for migration --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 5 +++++ services/document-updater/package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index f1ec00cf5c..553e707753 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -2,6 +2,7 @@ logger = require "logger-sharelatex" Settings = require "settings-sharelatex" redis = require("redis-sharelatex") LockManager = require("./LockManager") +metrics = require "./Metrics" async = require("async") # The aim is to migrate the project history queues @@ -89,6 +90,7 @@ class Multi @queueKey = null rpush: (args...) -> @queueKey = args[0] + @updates_count = args.length - 1 @command_list.push { command:'rpush', args: args} setnx: (args...) -> @command_list.push { command: 'setnx', args: args} @@ -103,6 +105,9 @@ class Multi callback(args...) @migrationClient.findQueue @queueKey, (err, rclient) => return releaseLock(err) if err? + # add metric for updates + dest = (if rclient == @rclient_new then "new" else "old") + metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} multi = rclient.multi() for entry in @command_list multi[entry.command](entry.args...) diff --git a/services/document-updater/package.json b/services/document-updater/package.json index f80251248b..ca79ec51ac 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -26,7 +26,7 @@ "lodash": "^4.17.4", "logger-sharelatex": "^1.7.0", "lynx": "0.0.11", - "metrics-sharelatex": "^2.2.0", + "metrics-sharelatex": "^2.4.0", "mongojs": "^2.6.0", "redis-sharelatex": "^1.0.11", "request": "2.47.0", From 8ae95ebf604734a4c3d2ddb96d7f3c29134fcc49 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:45:36 +0000 Subject: [PATCH 05/17] fix rclient check in migration metrics --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 553e707753..4978bf7e7b 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -106,7 +106,7 @@ class Multi @migrationClient.findQueue @queueKey, (err, rclient) => return releaseLock(err) if err? # add metric for updates - dest = (if rclient == @rclient_new then "new" else "old") + dest = (if rclient == @migrationClient.rclient_new then "new" else "old") metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} multi = rclient.multi() for entry in @command_list From 27044c2d02c5cd380dc9a29bf412d5e1d5c19d9c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:46:35 +0000 Subject: [PATCH 06/17] allow migration phase to be modified at runtime for testing --- .../app/coffee/RedisMigrationManager.coffee | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 4978bf7e7b..96c8ee049c 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -114,12 +114,17 @@ class Multi multi.exec releaseLock class MigrationClient - constructor: (old_settings, new_settings) -> - @rclient_old = redis.createClient(old_settings) - @rclient_new = redis.createClient(new_settings) + constructor: (@old_settings, @new_settings) -> + @rclient_old = redis.createClient(@old_settings) + @rclient_new = redis.createClient(@new_settings) @new_key_schema = new_settings.key_schema - @migration_phase = new_settings.migration_phase + # check that migration phase is valid on startup + @getMigrationPhase() + + getMigrationPhase: () -> + @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] + return @migration_phase getMigrationStatus: (key, migrationKey, callback) -> async.series [ @@ -136,12 +141,12 @@ class MigrationClient findQueue: (key, callback) -> project_id = getProjectId(key) migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - + migration_phase = @getMigrationPhase() # allow setting migration phase while running for testing @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => return callback(err) if err? # In all cases, if the migration key exists we must always write to the # new redis, unless we are rolling back. - if @migration_phase is "prepare" + if migration_phase is "prepare" # in this phase we prepare for the switch, when some docupdaters will # start setting the migration flag. We monitor the migration key and # write to the new redis if the key is present, but we do not set the @@ -154,7 +159,7 @@ class MigrationClient else logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) - else if @migration_phase is "switch" + else if migration_phase is "switch" # As we deploy the "switch" phase new docupdaters will set the migration # flag for projects which have an empty queue in the old redis, and # write updates into the new redis. Existing docupdaters still in the @@ -175,7 +180,7 @@ class MigrationClient return callback(err) if err? logger.debug {key: key}, "switching to new redis because old queue is empty" return callback(null, @rclient_new) - else if @migration_phase is "rollback" + else if migration_phase is "rollback" # If we need to roll back gracefully we do the opposite of the "switch" # phase. We use the new redis when the migration key is set and the # queue exists in the new redis, but if the queue in the new redis is @@ -194,7 +199,7 @@ class MigrationClient logger.debug {project_id}, "using old client because migration key does not exist" return callback(null, @rclient_old) else - logger.error {key: key, migration_phase: @migration_phase}, "unknown migration phase" + logger.error {key: key, migration_phase: migration_phase}, "unknown migration phase" callback(new Error('invalid migration phase')) multi: () -> new Multi(@) From c2714f9ae9c863890fc299915af4134d49676b4c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Jan 2020 16:50:06 +0000 Subject: [PATCH 07/17] add acceptance tests for RedisMigrationManager --- .../coffee/RedisMigrationManagerTests.coffee | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee diff --git a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee new file mode 100644 index 0000000000..2684a4a3d8 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee @@ -0,0 +1,320 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +expect = chai.expect +async = require "async" +Settings = require('settings-sharelatex') +rclient_old = require("redis-sharelatex").createClient(Settings.redis.project_history) +rclient_new = require("redis-sharelatex").createClient(Settings.redis.new_project_history) +rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) +Keys = Settings.redis.documentupdater.key_schema +HistoryKeys = Settings.redis.history.key_schema +ProjectHistoryKeys = Settings.redis.project_history.key_schema +NewProjectHistoryKeys = Settings.redis.new_project_history.key_schema + +MockTrackChangesApi = require "./helpers/MockTrackChangesApi" +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" +DocUpdaterApp = require "./helpers/DocUpdaterApp" + +describe "RedisMigrationManager", -> + before (done) -> + @lines = ["one", "two", "three"] + @version = 42 + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: @version + DocUpdaterApp.ensureRunning(done) + + describe "when the migration phase is 'prepare' (default)", -> + + describe "when there is no migration flag", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should not set the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should keep the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 1 + done() + return null + + describe "when the migration phase is 'switch'", -> + before -> + Settings.redis.new_project_history.migration_phase = 'switch' + + describe "when the old queue is empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should set the migration flag for the project", (done) -> + rclient_new.get NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal "NEW" + done() + return null + + describe "when the old queue is not empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + rclient_old.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal "dummy-op" + JSON.parse(updates[1]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should not set the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the old redis", (done) -> + rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should keep the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 1 + done() + return null + + describe "when the migration phase is 'rollback'", -> + before -> + Settings.redis.new_project_history.migration_phase = 'rollback' + + describe "when the old queue is empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + describe "when the new queue is not empty", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + rclient_new.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after -> + MockWebApi.getDocument.restore() + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal "dummy-op" + updates.length.should.equal 1 + done() + return null + + describe "when the migration flag is set for the project", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + + rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => + throw error if error? + MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + return null + + after (done) -> + MockWebApi.getDocument.restore() + rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done + return null + + it "should push the applied updates to the old redis", (done) -> + rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + JSON.parse(updates[0]).op.should.deep.equal @update.op + done() + return null + + it "should not push the applied updates to the new redis", (done) -> + rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + + it "should delete the migration flag for the project", (done) -> + rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => + result.should.equal 0 + done() + return null + From a638ef425146bb31e5ebfcd2b7d32e297cc75ab2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 13 Jan 2020 15:56:28 +0000 Subject: [PATCH 08/17] add comment about locking in redis migration --- .../app/coffee/RedisMigrationManager.coffee | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 96c8ee049c..a12593d03f 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -97,6 +97,14 @@ class Multi exec: (callback) -> # decide which client to use project_id = getProjectId(@queueKey) + # Put a lock around finding and updating the queue to avoid time-of-check to + # time-of-use problems. When running in the "switch" phase we need a lock to + # guarantee the order of operations. (Example: docupdater A sees an old + # queue at t=t0 and pushes onto it at t=t1, project history clears the queue + # between t0 and t1, and docupdater B sees the empty queue, sets the + # migration flag and pushes onto the new queue at t2. Without a lock it's + # possible to have t2 < t1 if docupdater A is slower than B - then there + # would be entries in the old and new queues, which we want to avoid.) LockManager.getLock project_id, (error, lockValue) => return callback(error) if error? releaseLock = (args...) => From 531d9b77b988e2991517c077979056369f3da402 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Jan 2020 09:49:41 +0000 Subject: [PATCH 09/17] add redislabs ca cert to repository --- services/document-updater/Dockerfile | 1 + services/document-updater/install_deps.sh | 2 + services/document-updater/redislabs_ca.pem | 77 ++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 services/document-updater/install_deps.sh create mode 100644 services/document-updater/redislabs_ca.pem diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 59f5e61889..2845544ae6 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -17,6 +17,7 @@ FROM node:6.9.5 COPY --from=app /app /app WORKDIR /app +RUN chmod 0755 ./install_deps.sh && ./install_deps.sh USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/install_deps.sh b/services/document-updater/install_deps.sh new file mode 100644 index 0000000000..8016ec6c85 --- /dev/null +++ b/services/document-updater/install_deps.sh @@ -0,0 +1,2 @@ +cp redislabs_ca.pem /usr/local/share/ca-certificates/redislabs_ca.crt +update-ca-certificates diff --git a/services/document-updater/redislabs_ca.pem b/services/document-updater/redislabs_ca.pem new file mode 100644 index 0000000000..a4af612d25 --- /dev/null +++ b/services/document-updater/redislabs_ca.pem @@ -0,0 +1,77 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 11859567854091286320 (0xa495a620ecc0b730) + Signature Algorithm: sha1WithRSAEncryption + Issuer: O=Garantia Data, CN=SSL Certification Authority + Validity + Not Before: Oct 1 12:14:55 2013 GMT + Not After : Sep 29 12:14:55 2023 GMT + Subject: O=Garantia Data, CN=SSL Certification Authority + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b6:6a:92:1f:c3:73:35:8f:26:7c:67:1c:b4:3b: + 40:bd:13:e0:1e:02:0c:a5:81:28:27:22:b2:b8:86: + 6c:0e:99:78:f5:95:36:8e:21:7c:a4:02:e8:9a:f3: + 7d:1f:b4:f3:53:5e:0f:a5:5c:59:48:b3:ae:67:7e: + 8e:d3:e1:21:8e:1c:f9:65:50:62:6e:4f:29:a3:7a: + 0d:3d:62:99:87:71:43:0e:da:a8:ee:63:d8:a5:02: + 12:1f:dc:ce:7a:4b:c5:e4:87:a1:3c:65:47:7e:04: + 43:01:76:f1:69:77:7a:0d:af:73:97:2d:f0:b8:d4: + dd:ea:33:59:59:37:81:be:da:97:1f:66:48:0d:92: + 82:6b:97:e6:51:10:6b:09:7e:fa:b4:a3:b0:14:ad: + 7a:66:36:04:3c:0e:a4:03:17:22:b7:44:c8:ff:dc: + 56:7f:26:92:f8:bf:04:3b:39:33:91:be:d3:d8:f4: + 81:f8:72:0b:34:56:31:0e:c7:9f:bd:6e:d5:ea:25: + 47:1c:15:c6:08:b7:4c:c9:fe:fe:f4:da:15:2a:b1: + 2a:38:1c:93:ac:ee:01:88:c1:44:f6:87:7b:ba:8b: + c4:73:6b:d5:2a:3f:31:cf:67:3f:2f:b7:c0:77:9b: + 17:06:c8:72:75:28:8f:06:e9:e2:77:2d:91:66:e3: + 6f:67 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 + X509v3 Authority Key Identifier: + keyid:FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 6d:9e:ad:78:70:44:06:bb:f9:93:81:b3:40:7a:5f:9e:c7:c3: + 27:75:47:89:1f:99:77:2c:d2:bb:5a:95:b3:e9:be:05:0b:4a: + 20:7e:4c:26:df:dc:46:e1:26:71:c6:ca:f7:42:63:5b:6f:95: + f7:cb:8d:d0:3b:1c:9d:0f:08:e9:fe:61:82:c1:03:4a:53:53: + f7:72:be:b3:7a:4a:ef:0d:b9:2e:72:b9:b9:ed:f6:66:f5:de: + 70:c6:62:8d:6b:9e:dd:18:45:fc:4d:fb:c0:cc:dd:f5:c8:56: + bd:37:f0:0d:f4:52:53:d7:d8:eb:b5:13:11:49:4f:43:19:b8: + 52:98:e9:9b:cb:74:8e:bf:d5:c6:e0:9a:0b:8c:94:08:4c:f8: + 38:4a:c9:5e:92:af:9e:bd:f4:b3:37:ce:a7:88:f3:5e:a9:66: + 69:51:10:44:d8:90:6a:fd:d6:ae:e4:06:95:c9:bb:f7:6d:1d: + a1:b1:83:56:46:bb:ac:3f:3c:2b:18:19:47:04:09:61:0d:60: + 3e:15:40:f7:7c:37:7d:89:8c:e7:ee:ea:f1:20:a0:40:30:7c: + f3:fe:de:81:a9:67:89:b7:7b:00:02:71:63:80:7a:7a:9f:95: + bf:9c:41:80:b8:3e:c1:7b:a9:b5:c3:99:16:96:ad:b2:a7:b4: + e9:59:de:7d +-----BEGIN CERTIFICATE----- +MIIDTzCCAjegAwIBAgIJAKSVpiDswLcwMA0GCSqGSIb3DQEBBQUAMD4xFjAUBgNV +BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTAeFw0xMzEwMDExMjE0NTVaFw0yMzA5MjkxMjE0NTVaMD4xFjAUBgNV +BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALZqkh/DczWP +JnxnHLQ7QL0T4B4CDKWBKCcisriGbA6ZePWVNo4hfKQC6JrzfR+081NeD6VcWUiz +rmd+jtPhIY4c+WVQYm5PKaN6DT1imYdxQw7aqO5j2KUCEh/cznpLxeSHoTxlR34E +QwF28Wl3eg2vc5ct8LjU3eozWVk3gb7alx9mSA2SgmuX5lEQawl++rSjsBStemY2 +BDwOpAMXIrdEyP/cVn8mkvi/BDs5M5G+09j0gfhyCzRWMQ7Hn71u1eolRxwVxgi3 +TMn+/vTaFSqxKjgck6zuAYjBRPaHe7qLxHNr1So/Mc9nPy+3wHebFwbIcnUojwbp +4nctkWbjb2cCAwEAAaNQME4wHQYDVR0OBBYEFP1whtcrydmW3ZJeuSoKZIKjze3w +MB8GA1UdIwQYMBaAFP1whtcrydmW3ZJeuSoKZIKjze3wMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAG2erXhwRAa7+ZOBs0B6X57Hwyd1R4kfmXcs0rta +lbPpvgULSiB+TCbf3EbhJnHGyvdCY1tvlffLjdA7HJ0PCOn+YYLBA0pTU/dyvrN6 +Su8NuS5yubnt9mb13nDGYo1rnt0YRfxN+8DM3fXIVr038A30UlPX2Ou1ExFJT0MZ +uFKY6ZvLdI6/1cbgmguMlAhM+DhKyV6Sr5699LM3zqeI816pZmlREETYkGr91q7k +BpXJu/dtHaGxg1ZGu6w/PCsYGUcECWENYD4VQPd8N32JjOfu6vEgoEAwfPP+3oGp +Z4m3ewACcWOAenqflb+cQYC4PsF7qbXDmRaWrbKntOlZ3n0= +-----END CERTIFICATE----- From ad58fe76b27ccfe3e49502407356f13edd09cc3f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Jan 2020 15:35:37 +0000 Subject: [PATCH 10/17] add tls settings --- .../config/settings.defaults.coffee | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index f890eb0f4a..0965c5bd3d 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -1,6 +1,7 @@ Path = require('path') http = require('http') http.globalAgent.maxSockets = 300 +fs = require('fs') module.exports = internal: @@ -44,6 +45,12 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" + tls: if process.env['REDIS_CA_CERT'] && process.env['REDIS_CLIENT_CERT'] && process.env['REDIS_CLIENT_KEY'] + ca: fs.readFileSync(process.env['REDIS_CA_CERT']), + cert: fs.readFileSync( + process.env['REDIS_CLIENT_CERT'] + ), + key: fs.readFileSync(process.env['REDIS_CLIENT_KEY']) new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" @@ -54,6 +61,12 @@ module.exports = projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" migration_phase: "prepare" + tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] + ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), + cert: fs.readFileSync( + process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] + ), + key: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CLIENT_KEY']) redisOptions: keepAlive: 100 From 31324fb65a585767fe61a80f9287f5dcc0c900c6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Jan 2020 15:28:34 +0000 Subject: [PATCH 11/17] add environment variable for migration_phase setting --- services/document-updater/config/settings.defaults.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 0965c5bd3d..a14a6d29fb 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -60,7 +60,7 @@ module.exports = projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" - migration_phase: "prepare" + migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), cert: fs.readFileSync( From d5a2b96df9c8c0f0ece5852a7931d7b0abd920f4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 14:36:59 +0000 Subject: [PATCH 12/17] add note about deleting the migration key entries --- .../app/coffee/RedisMigrationManager.coffee | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index a12593d03f..31ffa492c0 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -45,6 +45,12 @@ async = require("async") # - when all docupdaters are setting the flag (and writing to the new redis), # finish the migration by writing all data to the new redis # +# Final stage +# +# When all the queues are migrated, remove the migration code and return to a +# single client pointing at the new redis. Delete the +# ProjectHistory:MigrationKey:* entries in the new redis. +# # Rollback # # Under the scheme above a project should only ever have data in the old redis From 626e19ed1adc0d393c9395e3e0c38395eeef0f78 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 15:46:54 +0000 Subject: [PATCH 13/17] add logging of migration phase at startup --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index 31ffa492c0..af2eb7ad33 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -133,7 +133,7 @@ class MigrationClient @rclient_new = redis.createClient(@new_settings) @new_key_schema = new_settings.key_schema # check that migration phase is valid on startup - @getMigrationPhase() + logger.warn {migration_phase: @getMigrationPhase()}, "running with RedisMigrationManager" getMigrationPhase: () -> @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing From 544ae05212103925a7b55b160ac324b69057d9ba Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Jan 2020 16:22:26 +0000 Subject: [PATCH 14/17] added note about rollback --- .../document-updater/app/coffee/RedisMigrationManager.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee index af2eb7ad33..d11024fc94 100644 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ b/services/document-updater/app/coffee/RedisMigrationManager.coffee @@ -81,7 +81,8 @@ async = require("async") # - deploy project history with the "old queue" pointing to the new redis and # the "new queue" to the old redis to clear the new queue before processing # the new queue (i.e. add a rollback:true property in new_project_history in -# the project-history settings) +# the project-history settings via the environment variable +# MIGRATION_PHASE="rollback"). # - projects will now clear gradually from the new redis back to the old redis # - get a list of all the projects in the new redis and flush them, which will # cause the new queues to be cleared and the old redis to be used for those From 412eabc3063d37457c1f79e49e52f8848db1d9bc Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 Feb 2020 09:26:42 +0000 Subject: [PATCH 15/17] Revert "add tls settings" This reverts commit 72a4994cebab2731f99f0ada7a094c8a0acb3293. --- .../config/settings.defaults.coffee | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index a14a6d29fb..6724aa6a9a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -1,7 +1,6 @@ Path = require('path') http = require('http') http.globalAgent.maxSockets = 300 -fs = require('fs') module.exports = internal: @@ -45,12 +44,6 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - tls: if process.env['REDIS_CA_CERT'] && process.env['REDIS_CLIENT_CERT'] && process.env['REDIS_CLIENT_KEY'] - ca: fs.readFileSync(process.env['REDIS_CA_CERT']), - cert: fs.readFileSync( - process.env['REDIS_CLIENT_CERT'] - ), - key: fs.readFileSync(process.env['REDIS_CLIENT_KEY']) new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" @@ -61,12 +54,6 @@ module.exports = projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" - tls: if process.env['NEW_HISTORY_REDIS_CA_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] && process.env['NEW_HISTORY_REDIS_CLIENT_KEY'] - ca: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CA_CERT']), - cert: fs.readFileSync( - process.env['NEW_HISTORY_REDIS_CLIENT_CERT'] - ), - key: fs.readFileSync(process.env['NEW_HISTORY_REDIS_CLIENT_KEY']) redisOptions: keepAlive: 100 From 922f237c393e441b0124c44e4beabcc31d3bc9fe Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 Feb 2020 09:26:59 +0000 Subject: [PATCH 16/17] Revert "add redislabs ca cert to repository" This reverts commit eb7419b0f45992228393086caf7ed6f66296801f. --- services/document-updater/Dockerfile | 1 - services/document-updater/install_deps.sh | 2 - services/document-updater/redislabs_ca.pem | 77 ---------------------- 3 files changed, 80 deletions(-) delete mode 100644 services/document-updater/install_deps.sh delete mode 100644 services/document-updater/redislabs_ca.pem diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 2845544ae6..59f5e61889 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -17,7 +17,6 @@ FROM node:6.9.5 COPY --from=app /app /app WORKDIR /app -RUN chmod 0755 ./install_deps.sh && ./install_deps.sh USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/install_deps.sh b/services/document-updater/install_deps.sh deleted file mode 100644 index 8016ec6c85..0000000000 --- a/services/document-updater/install_deps.sh +++ /dev/null @@ -1,2 +0,0 @@ -cp redislabs_ca.pem /usr/local/share/ca-certificates/redislabs_ca.crt -update-ca-certificates diff --git a/services/document-updater/redislabs_ca.pem b/services/document-updater/redislabs_ca.pem deleted file mode 100644 index a4af612d25..0000000000 --- a/services/document-updater/redislabs_ca.pem +++ /dev/null @@ -1,77 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 11859567854091286320 (0xa495a620ecc0b730) - Signature Algorithm: sha1WithRSAEncryption - Issuer: O=Garantia Data, CN=SSL Certification Authority - Validity - Not Before: Oct 1 12:14:55 2013 GMT - Not After : Sep 29 12:14:55 2023 GMT - Subject: O=Garantia Data, CN=SSL Certification Authority - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:b6:6a:92:1f:c3:73:35:8f:26:7c:67:1c:b4:3b: - 40:bd:13:e0:1e:02:0c:a5:81:28:27:22:b2:b8:86: - 6c:0e:99:78:f5:95:36:8e:21:7c:a4:02:e8:9a:f3: - 7d:1f:b4:f3:53:5e:0f:a5:5c:59:48:b3:ae:67:7e: - 8e:d3:e1:21:8e:1c:f9:65:50:62:6e:4f:29:a3:7a: - 0d:3d:62:99:87:71:43:0e:da:a8:ee:63:d8:a5:02: - 12:1f:dc:ce:7a:4b:c5:e4:87:a1:3c:65:47:7e:04: - 43:01:76:f1:69:77:7a:0d:af:73:97:2d:f0:b8:d4: - dd:ea:33:59:59:37:81:be:da:97:1f:66:48:0d:92: - 82:6b:97:e6:51:10:6b:09:7e:fa:b4:a3:b0:14:ad: - 7a:66:36:04:3c:0e:a4:03:17:22:b7:44:c8:ff:dc: - 56:7f:26:92:f8:bf:04:3b:39:33:91:be:d3:d8:f4: - 81:f8:72:0b:34:56:31:0e:c7:9f:bd:6e:d5:ea:25: - 47:1c:15:c6:08:b7:4c:c9:fe:fe:f4:da:15:2a:b1: - 2a:38:1c:93:ac:ee:01:88:c1:44:f6:87:7b:ba:8b: - c4:73:6b:d5:2a:3f:31:cf:67:3f:2f:b7:c0:77:9b: - 17:06:c8:72:75:28:8f:06:e9:e2:77:2d:91:66:e3: - 6f:67 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 - X509v3 Authority Key Identifier: - keyid:FD:70:86:D7:2B:C9:D9:96:DD:92:5E:B9:2A:0A:64:82:A3:CD:ED:F0 - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 6d:9e:ad:78:70:44:06:bb:f9:93:81:b3:40:7a:5f:9e:c7:c3: - 27:75:47:89:1f:99:77:2c:d2:bb:5a:95:b3:e9:be:05:0b:4a: - 20:7e:4c:26:df:dc:46:e1:26:71:c6:ca:f7:42:63:5b:6f:95: - f7:cb:8d:d0:3b:1c:9d:0f:08:e9:fe:61:82:c1:03:4a:53:53: - f7:72:be:b3:7a:4a:ef:0d:b9:2e:72:b9:b9:ed:f6:66:f5:de: - 70:c6:62:8d:6b:9e:dd:18:45:fc:4d:fb:c0:cc:dd:f5:c8:56: - bd:37:f0:0d:f4:52:53:d7:d8:eb:b5:13:11:49:4f:43:19:b8: - 52:98:e9:9b:cb:74:8e:bf:d5:c6:e0:9a:0b:8c:94:08:4c:f8: - 38:4a:c9:5e:92:af:9e:bd:f4:b3:37:ce:a7:88:f3:5e:a9:66: - 69:51:10:44:d8:90:6a:fd:d6:ae:e4:06:95:c9:bb:f7:6d:1d: - a1:b1:83:56:46:bb:ac:3f:3c:2b:18:19:47:04:09:61:0d:60: - 3e:15:40:f7:7c:37:7d:89:8c:e7:ee:ea:f1:20:a0:40:30:7c: - f3:fe:de:81:a9:67:89:b7:7b:00:02:71:63:80:7a:7a:9f:95: - bf:9c:41:80:b8:3e:c1:7b:a9:b5:c3:99:16:96:ad:b2:a7:b4: - e9:59:de:7d ------BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIJAKSVpiDswLcwMA0GCSqGSIb3DQEBBQUAMD4xFjAUBgNV -BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 -dGhvcml0eTAeFw0xMzEwMDExMjE0NTVaFw0yMzA5MjkxMjE0NTVaMD4xFjAUBgNV -BAoMDUdhcmFudGlhIERhdGExJDAiBgNVBAMMG1NTTCBDZXJ0aWZpY2F0aW9uIEF1 -dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALZqkh/DczWP -JnxnHLQ7QL0T4B4CDKWBKCcisriGbA6ZePWVNo4hfKQC6JrzfR+081NeD6VcWUiz -rmd+jtPhIY4c+WVQYm5PKaN6DT1imYdxQw7aqO5j2KUCEh/cznpLxeSHoTxlR34E -QwF28Wl3eg2vc5ct8LjU3eozWVk3gb7alx9mSA2SgmuX5lEQawl++rSjsBStemY2 -BDwOpAMXIrdEyP/cVn8mkvi/BDs5M5G+09j0gfhyCzRWMQ7Hn71u1eolRxwVxgi3 -TMn+/vTaFSqxKjgck6zuAYjBRPaHe7qLxHNr1So/Mc9nPy+3wHebFwbIcnUojwbp -4nctkWbjb2cCAwEAAaNQME4wHQYDVR0OBBYEFP1whtcrydmW3ZJeuSoKZIKjze3w -MB8GA1UdIwQYMBaAFP1whtcrydmW3ZJeuSoKZIKjze3wMAwGA1UdEwQFMAMBAf8w -DQYJKoZIhvcNAQEFBQADggEBAG2erXhwRAa7+ZOBs0B6X57Hwyd1R4kfmXcs0rta -lbPpvgULSiB+TCbf3EbhJnHGyvdCY1tvlffLjdA7HJ0PCOn+YYLBA0pTU/dyvrN6 -Su8NuS5yubnt9mb13nDGYo1rnt0YRfxN+8DM3fXIVr038A30UlPX2Ou1ExFJT0MZ -uFKY6ZvLdI6/1cbgmguMlAhM+DhKyV6Sr5699LM3zqeI816pZmlREETYkGr91q7k -BpXJu/dtHaGxg1ZGu6w/PCsYGUcECWENYD4VQPd8N32JjOfu6vEgoEAwfPP+3oGp -Z4m3ewACcWOAenqflb+cQYC4PsF7qbXDmRaWrbKntOlZ3n0= ------END CERTIFICATE----- From 2e178b0e2d48e26935af69848baa3c4c80d72c3f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 21 Feb 2020 14:06:58 +0000 Subject: [PATCH 17/17] resolve merge conflicts --- services/document-updater/Dockerfile | 13 ++++++--- .../document-updater/docker-compose.ci.yml | 19 +++++++------ services/document-updater/docker-compose.yml | 27 ++++++++----------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 59f5e61889..e538fb48d9 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -1,7 +1,14 @@ -FROM node:6.9.5 as app +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3.5 + +FROM node:10.19.0 as base WORKDIR /app +FROM base as app + #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ @@ -12,11 +19,9 @@ COPY . /app RUN npm run compile:all -FROM node:6.9.5 +FROM base COPY --from=app /app /app - -WORKDIR /app USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index d65f97b913..56f3e1c42e 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: @@ -13,7 +13,6 @@ services: environment: NODE_ENV: test - test_acceptance: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -26,14 +25,15 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test depends_on: - - mongo - - redis - - new_redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + new_redis: + condition: service_healthy user: node command: npm run test:acceptance:_run - - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -41,7 +41,6 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: image: redis @@ -49,4 +48,4 @@ services: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 31869acb50..805e3b0d06 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -1,13 +1,13 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: - image: node:6.9.5 + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - build: . + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -33,19 +33,14 @@ services: NODE_ENV: test user: node depends_on: - - mongo - - redis - - new_redis + mongo: + condition: service_healthy + redis: + condition: service_healthy + new_redis: + condition: service_healthy command: npm run test:acceptance - tar: - build: . - image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - volumes: - - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . - user: root - redis: image: redis @@ -53,4 +48,4 @@ services: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6