From 8e210fe44140ca5fee302410924b2282068f72e2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:41:30 +0100 Subject: [PATCH 1/7] update unit tests --- .../coffee/ApplyingUpdatesToADocTests.coffee | 13 +++++++------ .../ApplyingUpdatesToProjectStructureTests.coffee | 14 +++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee index 0b28dea7a7..489f8d98eb 100644 --- a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee @@ -4,7 +4,8 @@ chai.should() expect = chai.expect async = require "async" Settings = require('settings-sharelatex') -rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) +rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) # note: this is track changes, not project-history +rclient_project_history = require("redis-sharelatex").createClient(Settings.redis.project_history) rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) Keys = Settings.redis.documentupdater.key_schema HistoryKeys = Settings.redis.history.key_schema @@ -65,14 +66,14 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null it "should set the first op timestamp", (done) -> - rclient_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => + rclient_project_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => throw error if error? result.should.be.within(@startTime, Date.now()) @firstOpTimestamp = result @@ -90,7 +91,7 @@ describe "Applying updates to a doc", -> return null it "should not change the first op timestamp", (done) -> - rclient_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => + rclient_project_history.get ProjectHistoryKeys.projectHistoryFirstOpTimestamp({@project_id}), (error, result) => throw error if error? result.should.equal @firstOpTimestamp done() @@ -130,7 +131,7 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null @@ -164,7 +165,7 @@ describe "Applying updates to a doc", -> return null it "should push the applied updates to the project history changes api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => JSON.parse(updates[0]).op.should.deep.equal @update.op done() return null diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee index cbb9fd9ea5..e18aa2e6a1 100644 --- a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToProjectStructureTests.coffee @@ -2,7 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() Settings = require('settings-sharelatex') -rclient_history = require("redis-sharelatex").createClient(Settings.redis.history) +rclient_project_history = require("redis-sharelatex").createClient(Settings.redis.project_history) ProjectHistoryKeys = Settings.redis.project_history.key_schema MockProjectHistoryApi = require "./helpers/MockProjectHistoryApi" @@ -30,7 +30,7 @@ describe "Applying updates to a project's structure", -> setTimeout done, 200 it "should push the applied file renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -61,7 +61,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -97,7 +97,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -141,7 +141,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the applied doc renames to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -194,7 +194,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the file addition to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) @@ -222,7 +222,7 @@ describe "Applying updates to a project's structure", -> return null it "should push the doc addition to the project history api", (done) -> - rclient_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => + rclient_project_history.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => throw error if error? update = JSON.parse(updates[0]) From af93193d6e05fca73b5f3f658ac2e6e93078240f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:43:48 +0100 Subject: [PATCH 2/7] remove new_project_history and use project_history instead --- .../app/coffee/ProjectHistoryRedisManager.coffee | 3 +-- .../document-updater/config/settings.defaults.coffee | 11 ----------- .../ProjectHistoryRedisManagerTests.coffee | 2 -- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee index 31842a1c8f..af75487a90 100644 --- a/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee +++ b/services/document-updater/app/coffee/ProjectHistoryRedisManager.coffee @@ -1,7 +1,6 @@ Settings = require('settings-sharelatex') projectHistoryKeys = Settings.redis?.project_history?.key_schema -#rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) -rclient = require("./RedisMigrationManager").createClient(Settings.redis.project_history, Settings.redis.new_project_history) +rclient = require("redis-sharelatex").createClient(Settings.redis.project_history) logger = require('logger-sharelatex') metrics = require('./Metrics') diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 6724aa6a9a..2fb398251a 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -37,23 +37,12 @@ module.exports = docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}" project_history: - port: process.env["HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" - host: process.env["HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" - password: process.env["HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") - key_schema: - projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" - projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - - new_project_history: port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" host: process.env["NEW_HISTORY_REDIS_HOST"] password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - projectHistoryMigrationKey: ({project_id}) -> "ProjectHistory:MigrationKey:{#{project_id}}" - migration_phase: process.env["PROJECT_HISTORY_MIGRATION_PHASE"] or "prepare" redisOptions: keepAlive: 100 diff --git a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee index 7199002162..9810b77d5f 100644 --- a/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee @@ -24,8 +24,6 @@ describe "ProjectHistoryRedisManager", -> } "redis-sharelatex": createClient: () => @rclient - "./RedisMigrationManager": - createClient: () => @rclient "logger-sharelatex": log:-> "./Metrics": @metrics = { summary: sinon.stub()} From 248edc03faaa36e6464e28b79d1fcc464ef6fe8e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:44:19 +0100 Subject: [PATCH 3/7] add comment about the two history clients --- services/document-updater/app/coffee/RedisManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index f212cfbadc..3eeed78ffb 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -23,7 +23,7 @@ MEGABYTES = 1024 * 1024 MAX_RANGES_SIZE = 3 * MEGABYTES keys = Settings.redis.documentupdater.key_schema -historyKeys = Settings.redis.history.key_schema +historyKeys = Settings.redis.history.key_schema # note: this is track changes, not project-history module.exports = RedisManager = rclient: rclient From a51f61a5558a276585b1503168d7f8cf5b538c4a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:48:47 +0100 Subject: [PATCH 4/7] remove redis migration code --- .../app/coffee/RedisMigrationManager.coffee | 224 ------------ .../coffee/RedisMigrationManagerTests.coffee | 320 ------------------ 2 files changed, 544 deletions(-) delete mode 100644 services/document-updater/app/coffee/RedisMigrationManager.coffee delete mode 100644 services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee deleted file mode 100644 index d11024fc94..0000000000 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ /dev/null @@ -1,224 +0,0 @@ -logger = require "logger-sharelatex" -Settings = require "settings-sharelatex" -redis = require("redis-sharelatex") -LockManager = require("./LockManager") -metrics = require "./Metrics" -async = require("async") - -# The aim is to migrate the project history queues -# ProjectHistory:Ops:{project_id} from the existing redis to a new redis. -# -# This has to work in conjunction with changes in project history. -# -# The basic principles are: -# -# - project history is modified to read from an 'old' and 'new' queue. It reads -# from the 'old' queue first, and when that queue is empty it reads from the -# 'new' queue. -# - docupdater will migrate to writing to the 'new' queue when the 'old' queue -# is empty. -# -# Some facts about the update process: -# -# - project history has a lock on the project-id, so each queue is processed in -# isolation -# - docupdaters take a lock on the doc_id but not the project_id, therefore -# multiple docupdaters can be appending to the queue for a project at the same -# time (provided they updates for individual docs are in order this is -# acceptable) -# - as we want to do this without shutting down the site, we have to take into -# account that different versions of the code will be running while deploys -# are in progress. -# -# The migration has to be carried out with the following constraint: -# -# - a docupdater should never write to the "old" queue when there are updates in -# the "new" queue (there is a strict ordering on the versions, new > old) -# -# The deployment process for docupdater will be -# -# - add a project-level lock to the queuing in docupdater -# - use a per-project migration flag to determine when to write to the new redis -# - set the migration flag for projects with an empty queue in the old redis -# - when all docupdaters respect the flag, make a new deploy which starts to set -# the flag -# - when all docupdaters are setting the flag (and writing to the new redis), -# finish the migration by writing all data to the new redis -# -# Final stage -# -# When all the queues are migrated, remove the migration code and return to a -# single client pointing at the new redis. Delete the -# ProjectHistory:MigrationKey:* entries in the new redis. -# -# Rollback -# -# Under the scheme above a project should only ever have data in the old redis -# or the new redis, but never both at the same time. -# -# Two scenarios: -# -# Hard rollback -# -# If we want to roll back to the old redis immediately, we need to get the data -# out of the new queues and back into the old queues, before appending to the -# old queues again. The actions to do this are: -# -# - close the site -# - revert docupdater so it only writes to the original redis (there will now -# be some data in the new redis for some projects which we need to recover) -# - run a script to move the new queues back into the old redis -# - revert project history to only read from the original redis -# -# Graceful rollback -# -# If we are prepared to keep the new redis running, but not add new projects to -# it we can do the following: -# -# - deploy all docupdaters to update from the "switch" phase into the -# "rollback" phase (projects in the new redis will continue to send data -# there, project not yet migrated will continue to go to the old redis) -# - deploy project history with the "old queue" pointing to the new redis and -# the "new queue" to the old redis to clear the new queue before processing -# the new queue (i.e. add a rollback:true property in new_project_history in -# the project-history settings via the environment variable -# MIGRATION_PHASE="rollback"). -# - projects will now clear gradually from the new redis back to the old redis -# - get a list of all the projects in the new redis and flush them, which will -# cause the new queues to be cleared and the old redis to be used for those -# projects. - -getProjectId = (key) -> - key.match(/\{([0-9a-f]{24})\}/)[1] - -class Multi - constructor: (@migrationClient) -> - @command_list = [] - @queueKey = null - rpush: (args...) -> - @queueKey = args[0] - @updates_count = args.length - 1 - @command_list.push { command:'rpush', args: args} - setnx: (args...) -> - @command_list.push { command: 'setnx', args: args} - exec: (callback) -> - # decide which client to use - project_id = getProjectId(@queueKey) - # Put a lock around finding and updating the queue to avoid time-of-check to - # time-of-use problems. When running in the "switch" phase we need a lock to - # guarantee the order of operations. (Example: docupdater A sees an old - # queue at t=t0 and pushes onto it at t=t1, project history clears the queue - # between t0 and t1, and docupdater B sees the empty queue, sets the - # migration flag and pushes onto the new queue at t2. Without a lock it's - # possible to have t2 < t1 if docupdater A is slower than B - then there - # would be entries in the old and new queues, which we want to avoid.) - LockManager.getLock project_id, (error, lockValue) => - return callback(error) if error? - releaseLock = (args...) => - LockManager.releaseLock project_id, lockValue, (lockError) -> - return callback(lockError) if lockError? - callback(args...) - @migrationClient.findQueue @queueKey, (err, rclient) => - return releaseLock(err) if err? - # add metric for updates - dest = (if rclient == @migrationClient.rclient_new then "new" else "old") - metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} - multi = rclient.multi() - for entry in @command_list - multi[entry.command](entry.args...) - multi.exec releaseLock - -class MigrationClient - constructor: (@old_settings, @new_settings) -> - @rclient_old = redis.createClient(@old_settings) - @rclient_new = redis.createClient(@new_settings) - @new_key_schema = new_settings.key_schema - # check that migration phase is valid on startup - logger.warn {migration_phase: @getMigrationPhase()}, "running with RedisMigrationManager" - - getMigrationPhase: () -> - @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing - throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] - return @migration_phase - - getMigrationStatus: (key, migrationKey, callback) -> - async.series [ - (cb) => @rclient_new.exists migrationKey, cb - (cb) => @rclient_new.exists key, cb - (cb) => @rclient_old.exists key, cb - ], (err, result) -> - return callback(err) if err? - migrationKeyExists = result[0] > 0 - newQueueExists = result[1] > 0 - oldQueueExists = result[2] > 0 - callback(null, migrationKeyExists, newQueueExists, oldQueueExists) - - findQueue: (key, callback) -> - project_id = getProjectId(key) - migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - migration_phase = @getMigrationPhase() # allow setting migration phase while running for testing - @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => - return callback(err) if err? - # In all cases, if the migration key exists we must always write to the - # new redis, unless we are rolling back. - if migration_phase is "prepare" - # in this phase we prepare for the switch, when some docupdaters will - # start setting the migration flag. We monitor the migration key and - # write to the new redis if the key is present, but we do not set the - # migration key. At this point no writes will be going into the new - # redis. When all the docupdaters are in the "prepare" phase we can - # begin deploying the "switch" phase. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else if migration_phase is "switch" - # As we deploy the "switch" phase new docupdaters will set the migration - # flag for projects which have an empty queue in the old redis, and - # write updates into the new redis. Existing docupdaters still in the - # "prepare" phase will pick up the migration flag and write new updates - # into the new redis when appropriate. When this deploy is complete - # writes will be going into the new redis for projects with an empty - # queue in the old redis. We have to remain in the switch phase until - # all projects are flushed from the old redis. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - if oldQueueExists - logger.debug {project_id}, "using old client because old queue exists" - return callback(null, @rclient_old) - else - @rclient_new.setnx migrationKey, "NEW", (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to new redis because old queue is empty" - return callback(null, @rclient_new) - else if migration_phase is "rollback" - # If we need to roll back gracefully we do the opposite of the "switch" - # phase. We use the new redis when the migration key is set and the - # queue exists in the new redis, but if the queue in the new redis is - # empty we delete the migration key and send further updates to the old - # redis. - if migrationKeyExists - if newQueueExists - logger.debug {project_id}, "using new client because migration key exists and new queue is present" - return callback(null, @rclient_new) - else - @rclient_new.del migrationKey, (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to old redis in rollback phase because new queue is empty" - return callback(null, @rclient_old) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else - logger.error {key: key, migration_phase: migration_phase}, "unknown migration phase" - callback(new Error('invalid migration phase')) - multi: () -> - new Multi(@) - -module.exports = RedisMigrationManager = - createClient: (args...) -> - new MigrationClient(args...) diff --git a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee deleted file mode 100644 index 2684a4a3d8..0000000000 --- a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee +++ /dev/null @@ -1,320 +0,0 @@ -sinon = require "sinon" -chai = require("chai") -chai.should() -expect = chai.expect -async = require "async" -Settings = require('settings-sharelatex') -rclient_old = require("redis-sharelatex").createClient(Settings.redis.project_history) -rclient_new = require("redis-sharelatex").createClient(Settings.redis.new_project_history) -rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) -Keys = Settings.redis.documentupdater.key_schema -HistoryKeys = Settings.redis.history.key_schema -ProjectHistoryKeys = Settings.redis.project_history.key_schema -NewProjectHistoryKeys = Settings.redis.new_project_history.key_schema - -MockTrackChangesApi = require "./helpers/MockTrackChangesApi" -MockWebApi = require "./helpers/MockWebApi" -DocUpdaterClient = require "./helpers/DocUpdaterClient" -DocUpdaterApp = require "./helpers/DocUpdaterApp" - -describe "RedisMigrationManager", -> - before (done) -> - @lines = ["one", "two", "three"] - @version = 42 - @update = - doc: @doc_id - op: [{ - i: "one and a half\n" - p: 4 - }] - v: @version - DocUpdaterApp.ensureRunning(done) - - describe "when the migration phase is 'prepare' (default)", -> - - describe "when there is no migration flag", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'switch'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'switch' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should set the migration flag for the project", (done) -> - rclient_new.get NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal "NEW" - done() - return null - - describe "when the old queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_old.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - JSON.parse(updates[1]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'rollback'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'rollback' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the new queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_new.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - updates.length.should.equal 1 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should delete the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - From 61da130cf401ef9eb2fe892553b74a660298dec8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 13:50:39 +0100 Subject: [PATCH 5/7] keep maxRetriesPerRequest for project_history redis config --- services/document-updater/config/settings.defaults.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 2fb398251a..1f1c951e1f 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -40,6 +40,7 @@ module.exports = port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" host: process.env["NEW_HISTORY_REDIS_HOST"] password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" From 2e24d1670c84c64cfe153383348e2b4299085cf7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 13:51:14 +0100 Subject: [PATCH 6/7] remove old unused ioredis keepalive option --- services/document-updater/config/settings.defaults.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 1f1c951e1f..f387a56122 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -44,8 +44,6 @@ module.exports = key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}" projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}" - redisOptions: - keepAlive: 100 lock: port: process.env["LOCK_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" From 79c934759c37e16e42cfa4ea16a015f21a9c1721 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 22 Apr 2020 14:04:28 +0100 Subject: [PATCH 7/7] add default redis settings for project history --- services/document-updater/config/settings.defaults.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index f387a56122..0ced9eeedd 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -37,9 +37,9 @@ module.exports = docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}" project_history: - port: process.env["NEW_HISTORY_REDIS_PORT"] or "6379" - host: process.env["NEW_HISTORY_REDIS_HOST"] - password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or "" + port: process.env["NEW_HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379" + host: process.env["NEW_HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost" + password: process.env["NEW_HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" maxRetriesPerRequest: parseInt(process.env['REDIS_MAX_RETRIES_PER_REQUEST'] or "20") key_schema: projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}"