From 9bd7c0017dcef9e00534b716745d8c51abc331b2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 7 Jul 2016 16:29:01 +0100 Subject: [PATCH 1/4] Allow migrate script to process multi docs at once --- .../config/settings.defaults.coffee | 24 +++++++++---------- services/document-updater/migrate.coffee | 21 ++++++++++------ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 15456db932..3f7abf90c7 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -32,18 +32,18 @@ module.exports = docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" - # }, { - # cluster: [{ - # port: "7000" - # host: "localhost" - # }] - # key_schema: - # blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" - # docLines: ({doc_id}) -> "doclines:{#{doc_id}}" - # docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" - # docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" - # projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" - # docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" + }, { + cluster: [{ + port: "7000" + host: "localhost" + }] + key_schema: + blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" + docLines: ({doc_id}) -> "doclines:{#{doc_id}}" + docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" + docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" + projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" + docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" }] max_doc_length: 2 * 1024 * 1024 # 2mb diff --git a/services/document-updater/migrate.coffee b/services/document-updater/migrate.coffee index 13f20e50c2..d0ede23a22 100644 --- a/services/document-updater/migrate.coffee +++ b/services/document-updater/migrate.coffee @@ -2,6 +2,8 @@ RedisManager = require "./app/coffee/RedisManager" UpdateManager = require "./app/coffee/UpdateManager" LockManager = require "./app/coffee/LockManager" +async = require "async" + handleErrorInsideLock = (doc_id, lockValue, original_error, callback = (error) ->) -> LockManager.releaseLock doc_id, lockValue, (lock_error) -> callback(original_error) @@ -17,13 +19,18 @@ migrateDoc = (doc_id, callback = (error) ->) -> return callback(error) if error? UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id, callback -doc_id = process.argv[2] -if !doc_id? - console.log "Usage: coffee migrate.coffee DOC_ID" +doc_ids = process.argv.slice(2) +if doc_ids.length == 0 + console.log "Usage: coffee migrate.coffee DOC_ID [DOC_ID ...]" process.exit(1) -migrateDoc doc_id, (error) -> +jobs = [] +for doc_id in doc_ids + do (doc_id) -> + jobs.push (cb) -> + console.log "MIGRATING #{doc_id}" + migrateDoc doc_id, cb + +async.series jobs, (error) -> throw error if error? - setTimeout () -> - process.exit(0) - , 200 \ No newline at end of file + process.exit(0) From 8b090c0a601af9cde5d2f543423106a4749b2e78 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 7 Jul 2016 16:41:38 +0100 Subject: [PATCH 2/4] Don't call sadd with blank args --- services/document-updater/app/coffee/RedisManager.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 257b889680..6878d8a17d 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -143,4 +143,7 @@ module.exports = RedisManager = getAndSetProject: (project_id, callback = (error) ->) -> rclient.smembers keys.docsInProject(project_id: project_id), (error, doc_ids) -> return callback(error) if error? - rclient.sadd keys.docsInProject(project_id: project_id), doc_ids..., callback + if doc_ids.length > 0 + rclient.sadd keys.docsInProject(project_id: project_id), doc_ids..., callback + else + callback() From bd392f568207b3df7ca1e85001d1fdef43aa4f74 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 8 Jul 2016 10:54:56 +0100 Subject: [PATCH 3/4] Comment out cluster config (again...) --- .../config/settings.defaults.coffee | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 3f7abf90c7..15456db932 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -32,18 +32,18 @@ module.exports = docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" - }, { - cluster: [{ - port: "7000" - host: "localhost" - }] - key_schema: - blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" - docLines: ({doc_id}) -> "doclines:{#{doc_id}}" - docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" - docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" - projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" - docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" + # }, { + # cluster: [{ + # port: "7000" + # host: "localhost" + # }] + # key_schema: + # blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" + # docLines: ({doc_id}) -> "doclines:{#{doc_id}}" + # docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" + # docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" + # projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" + # docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" }] max_doc_length: 2 * 1024 * 1024 # 2mb From 4f34cb7363dde4986fc6de6a1fe55ae3efe1ed11 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 8 Jul 2016 11:45:36 +0100 Subject: [PATCH 4/4] Only set keys in migration if they exist --- services/document-updater/app/coffee/RedisManager.coffee | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 6878d8a17d..2e59fefd57 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -130,12 +130,15 @@ module.exports = RedisManager = return callback(error) if error? [lines, version, ops, project_id] = results multi = rclient.multi() - multi.set keys.docLines(doc_id:doc_id), lines - multi.set keys.docVersion(doc_id:doc_id), version + if lines? + multi.set keys.docLines(doc_id:doc_id), lines + if version? + multi.set keys.docVersion(doc_id:doc_id), version multi.del keys.docOps(doc_id:doc_id) if ops.length > 0 multi.rpush keys.docOps(doc_id:doc_id), ops... - multi.set keys.projectKey(doc_id:doc_id), project_id + if project_id? + multi.set keys.projectKey(doc_id:doc_id), project_id multi.exec (error) -> return callback(error) if error? return callback null, project_id