From b5b61b98d0d8e19d14f36cd1f4c6ef622644fcac Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 Apr 2017 15:39:33 +0100 Subject: [PATCH] avoid blocking when fetching redis keys use scan instead of keys method --- .../app/coffee/RedisManager.coffee | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/services/track-changes/app/coffee/RedisManager.coffee b/services/track-changes/app/coffee/RedisManager.coffee index f29c9bc7ce..b58b99f11f 100644 --- a/services/track-changes/app/coffee/RedisManager.coffee +++ b/services/track-changes/app/coffee/RedisManager.coffee @@ -33,6 +33,23 @@ module.exports = RedisManager = getDocIdsWithHistoryOps: (project_id, callback = (error, doc_ids) ->) -> rclient.smembers docsWithHistoryOpsKey(project_id), callback + # iterate over keys asynchronously using redis scan (non-blocking) + _getKeys: (pattern, callback) -> + cursor = 0 # redis iterator + keySet = {} # use hash to avoid duplicate results + # scan over all keys looking for pattern + doIteration = (cb) -> + rclient.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) -> + return callback(error) if error? + [cursor, keys] = reply + for key in keys + keySet[key] = true + if cursor == '0' # note redis returns string result not numeric + return callback(null, Object.keys(keySet)) + else + doIteration() + doIteration() + # extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b _extractIds: (keyList) -> ids = (key.split(":")[1] for key in keyList) @@ -40,7 +57,7 @@ module.exports = RedisManager = # this will only work on single node redis, not redis cluster getProjectIdsWithHistoryOps: (callback = (error, project_ids) ->) -> - rclient.keys docsWithHistoryOpsKey("*"), (error, project_keys) -> + RedisManager._getKeys docsWithHistoryOpsKey("*"), (error, project_keys) -> return callback(error) if error? project_ids = RedisManager._extractIds project_keys callback(error, project_ids) @@ -49,7 +66,7 @@ module.exports = RedisManager = getAllDocIdsWithHistoryOps: (callback = (error, doc_ids) ->) -> # return all the docids, to find dangling history entries after # everything is flushed. - rclient.keys rawUpdatesKey("*"), (error, doc_keys) -> + RedisManager._getKeys rawUpdatesKey("*"), (error, doc_keys) -> return callback(error) if error? doc_ids = RedisManager._extractIds doc_keys callback(error, doc_ids)