diff --git a/services/document-updater/expire_docops.coffee b/services/document-updater/expire_docops.coffee deleted file mode 100644 index ff25b6f842..0000000000 --- a/services/document-updater/expire_docops.coffee +++ /dev/null @@ -1,44 +0,0 @@ -Settings = require "settings-sharelatex" -rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater) -keys = Settings.redis.documentupdater.key_schema -async = require "async" -RedisManager = require "./app/js/RedisManager" - -getKeysFromNode = (node, pattern, callback) -> - cursor = 0 # redis iterator - keySet = {} # use hash to avoid duplicate results - # scan over all keys looking for pattern - doIteration = (cb) -> - node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) -> - return callback(error) if error? - [cursor, keys] = reply - console.log "SCAN", keys.length - for key in keys - keySet[key] = true - if cursor == '0' # note redis returns string result not numeric - return callback(null, Object.keys(keySet)) - else - doIteration() - doIteration() - -getKeys = (pattern, callback) -> - nodes = rclient.nodes?('master') || [ rclient ]; - console.log "GOT NODES", nodes.length - doKeyLookupForNode = (node, cb) -> - getKeysFromNode node, pattern, cb - async.concatSeries nodes, doKeyLookupForNode, callback - -TTL = 60 * 60 # 1 hour -expireDocOps = (callback) -> - getKeys keys.docOps(doc_id: "*"), (error, keys) -> - async.mapSeries keys, - (key, cb) -> - console.log "EXPIRE #{key} #{RedisManager.DOC_OPS_TTL}" - rclient.expire key, RedisManager.DOC_OPS_TTL, cb - callback - -setTimeout () -> # Give redis a chance to connect - expireDocOps (error) -> - throw error if error? - process.exit() -, 1000 diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js new file mode 100644 index 0000000000..398bf5229a --- /dev/null +++ b/services/document-updater/expire_docops.js @@ -0,0 +1,65 @@ +const Settings = require('settings-sharelatex') +const rclient = require('@overleaf/redis-wrapper').createClient( + Settings.redis.documentupdater +) +let keys = Settings.redis.documentupdater.key_schema +const async = require('async') +const RedisManager = require('./app/js/RedisManager') + +const getKeysFromNode = function (node, pattern, callback) { + let cursor = 0 // redis iterator + const keySet = {} // use hash to avoid duplicate results + // scan over all keys looking for pattern + const doIteration = () => + node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { + if (error) { + return callback(error) + } + ;[cursor, keys] = reply + console.log('SCAN', keys.length) + for (const key of keys) { + keySet[key] = true + } + if (cursor === '0') { + // note redis returns string result not numeric + return callback(null, Object.keys(keySet)) + } else { + return doIteration() + } + }) + return doIteration() +} + +const getKeys = function (pattern, callback) { + const nodes = (typeof rclient.nodes === 'function' + ? rclient.nodes('master') + : undefined) || [rclient] + console.log('GOT NODES', nodes.length) + const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb) + return async.concatSeries(nodes, doKeyLookupForNode, callback) +} + +const expireDocOps = (callback) => + // eslint-disable-next-line handle-callback-err + getKeys(keys.docOps({ doc_id: '*' }), (error, keys) => + async.mapSeries( + keys, + function (key, cb) { + console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`) + return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb) + }, + callback + ) + ) + +setTimeout( + () => + // Give redis a chance to connect + expireDocOps(function (error) { + if (error) { + throw error + } + return process.exit() + }), + 1000 +) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee deleted file mode 100644 index e36f31f670..0000000000 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee +++ /dev/null @@ -1,41 +0,0 @@ -redis = require "@overleaf/redis-wrapper" -rclient1 = redis.createClient(cluster: [{ - port: "7000" - host: "localhost" -}]) - -rclient2 = redis.createClient(cluster: [{ - port: "7000" - host: "localhost" -}]) - -counter = 0 -sendPing = (cb = () ->) -> - rclient1.rpush "test-blpop", counter, (error) -> - console.error "[SENDING ERROR]", error.message if error? - if !error? - counter += 1 - cb() - -previous = null -listenForPing = (cb) -> - rclient2.blpop "test-blpop", 200, (error, result) -> - return cb(error) if error? - [key, value] = result - value = parseInt(value, 10) - if value % 10 == 0 - console.log "." - if previous? and value != previous + 1 - error = new Error("Counter not in order. Got #{value}, expected #{previous + 1}") - previous = value - return cb(error, value) - -PING_DELAY = 100 -do sendPings = () -> - sendPing () -> - setTimeout sendPings, PING_DELAY - -do listenInBackground = (cb = () ->) -> - listenForPing (error, value) -> - console.error "[RECEIVING ERROR]", error.message if error - setTimeout listenInBackground diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee deleted file mode 100644 index eccf952504..0000000000 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee +++ /dev/null @@ -1,33 +0,0 @@ -redis = require "@overleaf/redis-wrapper" -rclient1 = redis.createClient(cluster: [{ - port: "7000" - host: "localhost" -}]) - -rclient2 = redis.createClient(cluster: [{ - port: "7000" - host: "localhost" -}]) - -counter = 0 -sendPing = (cb = () ->) -> - rclient1.publish "test-pubsub", counter, (error) -> - console.error "[SENDING ERROR]", error.message if error? - if !error? - counter += 1 - cb() - -previous = null -rclient2.subscribe "test-pubsub" -rclient2.on "message", (channel, value) -> - value = parseInt(value, 10) - if value % 10 == 0 - console.log "." - if previous? and value != previous + 1 - console.error "[RECEIVING ERROR]", "Counter not in order. Got #{value}, expected #{previous + 1}" - previous = value - -PING_DELAY = 100 -do sendPings = () -> - sendPing () -> - setTimeout sendPings, PING_DELAY diff --git a/services/document-updater/test/cluster_failover/js/test_blpop_failover.js b/services/document-updater/test/cluster_failover/js/test_blpop_failover.js new file mode 100644 index 0000000000..e3f52f6339 --- /dev/null +++ b/services/document-updater/test/cluster_failover/js/test_blpop_failover.js @@ -0,0 +1,65 @@ +let listenInBackground, sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) + +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) + +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.rpush('test-blpop', counter, (error) => { + if (error != null) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} + +let previous = null +const listenForPing = (cb) => + rclient2.blpop('test-blpop', 200, (error, result) => { + if (error != null) { + return cb(error) + } + let [, value] = Array.from(result) + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + error = new Error( + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + previous = value + return cb(error, value) + }) + +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() +;(listenInBackground = () => + listenForPing((error) => { + if (error) { + console.error('[RECEIVING ERROR]', error.message) + } + return setTimeout(listenInBackground) + }))() diff --git a/services/document-updater/test/cluster_failover/js/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/js/test_pubsub_failover.js new file mode 100644 index 0000000000..3da52be287 --- /dev/null +++ b/services/document-updater/test/cluster_failover/js/test_pubsub_failover.js @@ -0,0 +1,54 @@ +let sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) + +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) + +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.publish('test-pubsub', counter, (error) => { + if (error) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} + +let previous = null +rclient2.subscribe('test-pubsub') +rclient2.on('message', (channel, value) => { + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + console.error( + '[RECEIVING ERROR]', + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + return (previous = value) +}) + +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() diff --git a/services/document-updater/test/stress/js/run.js b/services/document-updater/test/stress/js/run.js index 3ce482b19b..da78735a95 100644 --- a/services/document-updater/test/stress/js/run.js +++ b/services/document-updater/test/stress/js/run.js @@ -16,7 +16,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const DocUpdaterClient = require('../../acceptance/coffee/helpers/DocUpdaterClient') +const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient') // MockTrackChangesApi = require "../../acceptance/js/helpers/MockTrackChangesApi" // MockWebApi = require "../../acceptance/js/helpers/MockWebApi" const assert = require('assert')