From edacebb53cbd5bd17c312d19eaa44430063478fe Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Wed, 19 May 2021 20:25:04 +0100 Subject: [PATCH 01/10] Remove unused parameter --- .../test/cluster_failover/coffee/test_blpop_failover.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee index e36f31f670..60a1ddaaa5 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee @@ -35,7 +35,7 @@ do sendPings = () -> sendPing () -> setTimeout sendPings, PING_DELAY -do listenInBackground = (cb = () ->) -> +do listenInBackground = () -> listenForPing (error, value) -> console.error "[RECEIVING ERROR]", error.message if error setTimeout listenInBackground From e1e0d26b1fc0b85184805a9d6947d7f416a1e043 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Wed, 19 May 2021 20:25:06 +0100 Subject: [PATCH 02/10] decaffeinate: Rename expire_docops.coffee and 2 other files from .coffee to .js --- .../document-updater/{expire_docops.coffee => expire_docops.js} | 0 .../coffee/{test_blpop_failover.coffee => test_blpop_failover.js} | 0 .../{test_pubsub_failover.coffee => test_pubsub_failover.js} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename services/document-updater/{expire_docops.coffee => expire_docops.js} (100%) rename services/document-updater/test/cluster_failover/coffee/{test_blpop_failover.coffee => test_blpop_failover.js} (100%) rename services/document-updater/test/cluster_failover/coffee/{test_pubsub_failover.coffee => test_pubsub_failover.js} (100%) diff --git a/services/document-updater/expire_docops.coffee b/services/document-updater/expire_docops.js similarity index 100% rename from services/document-updater/expire_docops.coffee rename to services/document-updater/expire_docops.js diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js similarity index 100% rename from services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee rename to services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js similarity index 100% rename from services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee rename to services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js From 5ad0b6eea23be24fe69cc1155b60c792a718258f Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Wed, 19 May 2021 20:25:07 +0100 Subject: [PATCH 03/10] decaffeinate: Convert expire_docops.coffee and 2 other files to JS --- services/document-updater/expire_docops.js | 90 ++++++++++--------- .../coffee/test_blpop_failover.js | 80 ++++++++++------- .../coffee/test_pubsub_failover.js | 66 ++++++++------ 3 files changed, 135 insertions(+), 101 deletions(-) diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index ff25b6f842..59f498a181 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -1,44 +1,54 @@ -Settings = require "settings-sharelatex" -rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater) -keys = Settings.redis.documentupdater.key_schema -async = require "async" -RedisManager = require "./app/js/RedisManager" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const Settings = require("settings-sharelatex"); +const rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater); +let keys = Settings.redis.documentupdater.key_schema; +const async = require("async"); +const RedisManager = require("./app/js/RedisManager"); -getKeysFromNode = (node, pattern, callback) -> - cursor = 0 # redis iterator - keySet = {} # use hash to avoid duplicate results - # scan over all keys looking for pattern - doIteration = (cb) -> - node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) -> - return callback(error) if error? - [cursor, keys] = reply - console.log "SCAN", keys.length - for key in keys - keySet[key] = true - if cursor == '0' # note redis returns string result not numeric - return callback(null, Object.keys(keySet)) - else - doIteration() - doIteration() +const getKeysFromNode = function(node, pattern, callback) { + let cursor = 0; // redis iterator + const keySet = {}; // use hash to avoid duplicate results + // scan over all keys looking for pattern + var doIteration = cb => node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) { + if (error != null) { return callback(error); } + [cursor, keys] = Array.from(reply); + console.log("SCAN", keys.length); + for (let key of Array.from(keys)) { + keySet[key] = true; + } + if (cursor === '0') { // note redis returns string result not numeric + return callback(null, Object.keys(keySet)); + } else { + return doIteration(); + } + }); + return doIteration(); +}; -getKeys = (pattern, callback) -> - nodes = rclient.nodes?('master') || [ rclient ]; - console.log "GOT NODES", nodes.length - doKeyLookupForNode = (node, cb) -> - getKeysFromNode node, pattern, cb - async.concatSeries nodes, doKeyLookupForNode, callback +const getKeys = function(pattern, callback) { + const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ]; + console.log("GOT NODES", nodes.length); + const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb); + return async.concatSeries(nodes, doKeyLookupForNode, callback); +}; -TTL = 60 * 60 # 1 hour -expireDocOps = (callback) -> - getKeys keys.docOps(doc_id: "*"), (error, keys) -> - async.mapSeries keys, - (key, cb) -> - console.log "EXPIRE #{key} #{RedisManager.DOC_OPS_TTL}" - rclient.expire key, RedisManager.DOC_OPS_TTL, cb - callback +const TTL = 60 * 60; // 1 hour +const expireDocOps = callback => getKeys(keys.docOps({doc_id: "*"}), (error, keys) => async.mapSeries(keys, + function(key, cb) { + console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`); + return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb); + }, + callback)); -setTimeout () -> # Give redis a chance to connect - expireDocOps (error) -> - throw error if error? - process.exit() -, 1000 +setTimeout(() => // Give redis a chance to connect +expireDocOps(function(error) { + if (error != null) { throw error; } + return process.exit(); +}) +, 1000); diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index 60a1ddaaa5..7c29240717 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -1,41 +1,53 @@ -redis = require "@overleaf/redis-wrapper" -rclient1 = redis.createClient(cluster: [{ - port: "7000" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let listenInBackground, sendPings; +const redis = require("@overleaf/redis-wrapper"); +const rclient1 = redis.createClient({cluster: [{ + port: "7000", host: "localhost" -}]) +}]}); -rclient2 = redis.createClient(cluster: [{ - port: "7000" +const rclient2 = redis.createClient({cluster: [{ + port: "7000", host: "localhost" -}]) +}]}); -counter = 0 -sendPing = (cb = () ->) -> - rclient1.rpush "test-blpop", counter, (error) -> - console.error "[SENDING ERROR]", error.message if error? - if !error? - counter += 1 - cb() +let counter = 0; +const sendPing = function(cb) { + if (cb == null) { cb = function() {}; } + return rclient1.rpush("test-blpop", counter, function(error) { + if (error != null) { console.error("[SENDING ERROR]", error.message); } + if ((error == null)) { + counter += 1; + } + return cb(); + }); +}; -previous = null -listenForPing = (cb) -> - rclient2.blpop "test-blpop", 200, (error, result) -> - return cb(error) if error? - [key, value] = result - value = parseInt(value, 10) - if value % 10 == 0 - console.log "." - if previous? and value != previous + 1 - error = new Error("Counter not in order. Got #{value}, expected #{previous + 1}") - previous = value - return cb(error, value) +let previous = null; +const listenForPing = cb => rclient2.blpop("test-blpop", 200, function(error, result) { + if (error != null) { return cb(error); } + let [key, value] = Array.from(result); + value = parseInt(value, 10); + if ((value % 10) === 0) { + console.log("."); + } + if ((previous != null) && (value !== (previous + 1))) { + error = new Error(`Counter not in order. Got ${value}, expected ${previous + 1}`); + } + previous = value; + return cb(error, value); +}); -PING_DELAY = 100 -do sendPings = () -> - sendPing () -> - setTimeout sendPings, PING_DELAY +const PING_DELAY = 100; +(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); -do listenInBackground = () -> - listenForPing (error, value) -> - console.error "[RECEIVING ERROR]", error.message if error - setTimeout listenInBackground +(listenInBackground = () => listenForPing(function(error, value) { + if (error) { console.error("[RECEIVING ERROR]", error.message); } + return setTimeout(listenInBackground); +}))(); diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js index eccf952504..295ffc5b7b 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js @@ -1,33 +1,45 @@ -redis = require "@overleaf/redis-wrapper" -rclient1 = redis.createClient(cluster: [{ - port: "7000" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let sendPings; +const redis = require("@overleaf/redis-wrapper"); +const rclient1 = redis.createClient({cluster: [{ + port: "7000", host: "localhost" -}]) +}]}); -rclient2 = redis.createClient(cluster: [{ - port: "7000" +const rclient2 = redis.createClient({cluster: [{ + port: "7000", host: "localhost" -}]) +}]}); -counter = 0 -sendPing = (cb = () ->) -> - rclient1.publish "test-pubsub", counter, (error) -> - console.error "[SENDING ERROR]", error.message if error? - if !error? - counter += 1 - cb() +let counter = 0; +const sendPing = function(cb) { + if (cb == null) { cb = function() {}; } + return rclient1.publish("test-pubsub", counter, function(error) { + if (error != null) { console.error("[SENDING ERROR]", error.message); } + if ((error == null)) { + counter += 1; + } + return cb(); + }); +}; -previous = null -rclient2.subscribe "test-pubsub" -rclient2.on "message", (channel, value) -> - value = parseInt(value, 10) - if value % 10 == 0 - console.log "." - if previous? and value != previous + 1 - console.error "[RECEIVING ERROR]", "Counter not in order. Got #{value}, expected #{previous + 1}" - previous = value +let previous = null; +rclient2.subscribe("test-pubsub"); +rclient2.on("message", function(channel, value) { + value = parseInt(value, 10); + if ((value % 10) === 0) { + console.log("."); + } + if ((previous != null) && (value !== (previous + 1))) { + console.error("[RECEIVING ERROR]", `Counter not in order. Got ${value}, expected ${previous + 1}`); + } + return previous = value; +}); -PING_DELAY = 100 -do sendPings = () -> - sendPing () -> - setTimeout sendPings, PING_DELAY +const PING_DELAY = 100; +(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); From 03cbad4e9f10cc7fbe89b5f620d4948c927a3657 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Wed, 19 May 2021 20:25:07 +0100 Subject: [PATCH 04/10] decaffeinate: Run post-processing cleanups on expire_docops.coffee and 2 other files --- services/document-updater/expire_docops.js | 8 +++++++- .../cluster_failover/coffee/test_blpop_failover.js | 11 ++++++++--- .../cluster_failover/coffee/test_pubsub_failover.js | 9 +++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index 59f498a181..811b0cc019 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -1,3 +1,9 @@ +/* eslint-disable + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -19,7 +25,7 @@ const getKeysFromNode = function(node, pattern, callback) { if (error != null) { return callback(error); } [cursor, keys] = Array.from(reply); console.log("SCAN", keys.length); - for (let key of Array.from(keys)) { + for (const key of Array.from(keys)) { keySet[key] = true; } if (cursor === '0') { // note redis returns string result not numeric diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index 7c29240717..a356e683d4 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -20,7 +25,7 @@ const rclient2 = redis.createClient({cluster: [{ let counter = 0; const sendPing = function(cb) { if (cb == null) { cb = function() {}; } - return rclient1.rpush("test-blpop", counter, function(error) { + return rclient1.rpush("test-blpop", counter, (error) => { if (error != null) { console.error("[SENDING ERROR]", error.message); } if ((error == null)) { counter += 1; @@ -30,7 +35,7 @@ const sendPing = function(cb) { }; let previous = null; -const listenForPing = cb => rclient2.blpop("test-blpop", 200, function(error, result) { +const listenForPing = cb => rclient2.blpop("test-blpop", 200, (error, result) => { if (error != null) { return cb(error); } let [key, value] = Array.from(result); value = parseInt(value, 10); @@ -47,7 +52,7 @@ const listenForPing = cb => rclient2.blpop("test-blpop", 200, function(error, re const PING_DELAY = 100; (sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); -(listenInBackground = () => listenForPing(function(error, value) { +(listenInBackground = () => listenForPing((error, value) => { if (error) { console.error("[RECEIVING ERROR]", error.message); } return setTimeout(listenInBackground); }))(); diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js index 295ffc5b7b..670c7afa3a 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -19,7 +24,7 @@ const rclient2 = redis.createClient({cluster: [{ let counter = 0; const sendPing = function(cb) { if (cb == null) { cb = function() {}; } - return rclient1.publish("test-pubsub", counter, function(error) { + return rclient1.publish("test-pubsub", counter, (error) => { if (error != null) { console.error("[SENDING ERROR]", error.message); } if ((error == null)) { counter += 1; @@ -30,7 +35,7 @@ const sendPing = function(cb) { let previous = null; rclient2.subscribe("test-pubsub"); -rclient2.on("message", function(channel, value) { +rclient2.on("message", (channel, value) => { value = parseInt(value, 10); if ((value % 10) === 0) { console.log("."); From de50e595574c552c6d13211244924151a935d096 Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 14:27:45 +0100 Subject: [PATCH 05/10] Run format:fix --- services/document-updater/expire_docops.js | 104 ++++++++++-------- .../coffee/test_blpop_failover.js | 104 +++++++++++------- .../coffee/test_pubsub_failover.js | 85 ++++++++------ 3 files changed, 173 insertions(+), 120 deletions(-) diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index 811b0cc019..ffafbe7255 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -11,50 +11,68 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const Settings = require("settings-sharelatex"); -const rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater); -let keys = Settings.redis.documentupdater.key_schema; -const async = require("async"); -const RedisManager = require("./app/js/RedisManager"); +const Settings = require('settings-sharelatex') +const rclient = require('@overleaf/redis-wrapper').createClient( + Settings.redis.documentupdater +) +let keys = Settings.redis.documentupdater.key_schema +const async = require('async') +const RedisManager = require('./app/js/RedisManager') -const getKeysFromNode = function(node, pattern, callback) { - let cursor = 0; // redis iterator - const keySet = {}; // use hash to avoid duplicate results - // scan over all keys looking for pattern - var doIteration = cb => node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) { - if (error != null) { return callback(error); } - [cursor, keys] = Array.from(reply); - console.log("SCAN", keys.length); - for (const key of Array.from(keys)) { - keySet[key] = true; - } - if (cursor === '0') { // note redis returns string result not numeric - return callback(null, Object.keys(keySet)); - } else { - return doIteration(); - } - }); - return doIteration(); -}; +const getKeysFromNode = function (node, pattern, callback) { + let cursor = 0 // redis iterator + const keySet = {} // use hash to avoid duplicate results + // scan over all keys looking for pattern + var doIteration = (cb) => + node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { + if (error != null) { + return callback(error) + } + ;[cursor, keys] = Array.from(reply) + console.log('SCAN', keys.length) + for (const key of Array.from(keys)) { + keySet[key] = true + } + if (cursor === '0') { + // note redis returns string result not numeric + return callback(null, Object.keys(keySet)) + } else { + return doIteration() + } + }) + return doIteration() +} -const getKeys = function(pattern, callback) { - const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ]; - console.log("GOT NODES", nodes.length); - const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb); - return async.concatSeries(nodes, doKeyLookupForNode, callback); -}; +const getKeys = function (pattern, callback) { + const nodes = (typeof rclient.nodes === 'function' + ? rclient.nodes('master') + : undefined) || [rclient] + console.log('GOT NODES', nodes.length) + const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb) + return async.concatSeries(nodes, doKeyLookupForNode, callback) +} -const TTL = 60 * 60; // 1 hour -const expireDocOps = callback => getKeys(keys.docOps({doc_id: "*"}), (error, keys) => async.mapSeries(keys, - function(key, cb) { - console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`); - return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb); - }, - callback)); +const TTL = 60 * 60 // 1 hour +const expireDocOps = (callback) => + getKeys(keys.docOps({ doc_id: '*' }), (error, keys) => + async.mapSeries( + keys, + function (key, cb) { + console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`) + return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb) + }, + callback + ) + ) -setTimeout(() => // Give redis a chance to connect -expireDocOps(function(error) { - if (error != null) { throw error; } - return process.exit(); -}) -, 1000); +setTimeout( + () => + // Give redis a chance to connect + expireDocOps(function (error) { + if (error != null) { + throw error + } + return process.exit() + }), + 1000 +) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index a356e683d4..3d7cae99da 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -10,49 +10,69 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let listenInBackground, sendPings; -const redis = require("@overleaf/redis-wrapper"); -const rclient1 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); - -const rclient2 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); - -let counter = 0; -const sendPing = function(cb) { - if (cb == null) { cb = function() {}; } - return rclient1.rpush("test-blpop", counter, (error) => { - if (error != null) { console.error("[SENDING ERROR]", error.message); } - if ((error == null)) { - counter += 1; - } - return cb(); - }); -}; - -let previous = null; -const listenForPing = cb => rclient2.blpop("test-blpop", 200, (error, result) => { - if (error != null) { return cb(error); } - let [key, value] = Array.from(result); - value = parseInt(value, 10); - if ((value % 10) === 0) { - console.log("."); +let listenInBackground, sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' } - if ((previous != null) && (value !== (previous + 1))) { - error = new Error(`Counter not in order. Got ${value}, expected ${previous + 1}`); + ] +}) + +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' } - previous = value; - return cb(error, value); -}); + ] +}) -const PING_DELAY = 100; -(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.rpush('test-blpop', counter, (error) => { + if (error != null) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} -(listenInBackground = () => listenForPing((error, value) => { - if (error) { console.error("[RECEIVING ERROR]", error.message); } - return setTimeout(listenInBackground); -}))(); +let previous = null +const listenForPing = (cb) => + rclient2.blpop('test-blpop', 200, (error, result) => { + if (error != null) { + return cb(error) + } + let [key, value] = Array.from(result) + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + error = new Error( + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + previous = value + return cb(error, value) + }) + +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() + +;(listenInBackground = () => + listenForPing((error, value) => { + if (error) { + console.error('[RECEIVING ERROR]', error.message) + } + return setTimeout(listenInBackground) + }))() diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js index 670c7afa3a..627fb82a0d 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js @@ -9,42 +9,57 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let sendPings; -const redis = require("@overleaf/redis-wrapper"); -const rclient1 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); +let sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) -const rclient2 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) -let counter = 0; -const sendPing = function(cb) { - if (cb == null) { cb = function() {}; } - return rclient1.publish("test-pubsub", counter, (error) => { - if (error != null) { console.error("[SENDING ERROR]", error.message); } - if ((error == null)) { - counter += 1; - } - return cb(); - }); -}; +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.publish('test-pubsub', counter, (error) => { + if (error != null) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} -let previous = null; -rclient2.subscribe("test-pubsub"); -rclient2.on("message", (channel, value) => { - value = parseInt(value, 10); - if ((value % 10) === 0) { - console.log("."); - } - if ((previous != null) && (value !== (previous + 1))) { - console.error("[RECEIVING ERROR]", `Counter not in order. Got ${value}, expected ${previous + 1}`); - } - return previous = value; -}); +let previous = null +rclient2.subscribe('test-pubsub') +rclient2.on('message', (channel, value) => { + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + console.error( + '[RECEIVING ERROR]', + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + return (previous = value) +}) -const PING_DELAY = 100; -(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() From f4db3d8f45a3356b319b873dcbe92b88aef51308 Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 14:32:03 +0100 Subject: [PATCH 06/10] Manual tidying --- services/document-updater/expire_docops.js | 29 +++++-------------- .../coffee/test_blpop_failover.js | 16 ++-------- .../coffee/test_pubsub_failover.js | 13 +-------- 3 files changed, 11 insertions(+), 47 deletions(-) diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index ffafbe7255..3eba0d547c 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -1,16 +1,3 @@ -/* eslint-disable - handle-callback-err, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const Settings = require('settings-sharelatex') const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.documentupdater @@ -23,17 +10,17 @@ const getKeysFromNode = function (node, pattern, callback) { let cursor = 0 // redis iterator const keySet = {} // use hash to avoid duplicate results // scan over all keys looking for pattern - var doIteration = (cb) => - node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { - if (error != null) { + const doIteration = () => + node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function(error, reply) { + if (error) { return callback(error) } - ;[cursor, keys] = Array.from(reply) + ;[cursor, keys] = reply console.log('SCAN', keys.length) - for (const key of Array.from(keys)) { + for (const key of keys) { keySet[key] = true } - if (cursor === '0') { + if (cursor==='0') { // note redis returns string result not numeric return callback(null, Object.keys(keySet)) } else { @@ -52,8 +39,8 @@ const getKeys = function (pattern, callback) { return async.concatSeries(nodes, doKeyLookupForNode, callback) } -const TTL = 60 * 60 // 1 hour const expireDocOps = (callback) => + // eslint-disable-next-line handle-callback-err getKeys(keys.docOps({ doc_id: '*' }), (error, keys) => async.mapSeries( keys, @@ -69,7 +56,7 @@ setTimeout( () => // Give redis a chance to connect expireDocOps(function (error) { - if (error != null) { + if (error) { throw error } return process.exit() diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index 3d7cae99da..6d4b3ee55d 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -1,15 +1,3 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ let listenInBackground, sendPings const redis = require('@overleaf/redis-wrapper') const rclient1 = redis.createClient({ @@ -52,7 +40,7 @@ const listenForPing = (cb) => if (error != null) { return cb(error) } - let [key, value] = Array.from(result) + let [, value] = Array.from(result) value = parseInt(value, 10) if (value % 10 === 0) { console.log('.') @@ -70,7 +58,7 @@ const PING_DELAY = 100 ;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() ;(listenInBackground = () => - listenForPing((error, value) => { + listenForPing((error) => { if (error) { console.error('[RECEIVING ERROR]', error.message) } diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js index 627fb82a0d..3da52be287 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js @@ -1,14 +1,3 @@ -/* eslint-disable - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ let sendPings const redis = require('@overleaf/redis-wrapper') const rclient1 = redis.createClient({ @@ -35,7 +24,7 @@ const sendPing = function (cb) { cb = function () {} } return rclient1.publish('test-pubsub', counter, (error) => { - if (error != null) { + if (error) { console.error('[SENDING ERROR]', error.message) } if (error == null) { From 358786fccd78cd20fe4bd576e019ce0b98a53b2e Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 15:45:42 +0100 Subject: [PATCH 07/10] Run format:fix --- services/document-updater/expire_docops.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index 3eba0d547c..398bf5229a 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -11,7 +11,7 @@ const getKeysFromNode = function (node, pattern, callback) { const keySet = {} // use hash to avoid duplicate results // scan over all keys looking for pattern const doIteration = () => - node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function(error, reply) { + node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { if (error) { return callback(error) } @@ -20,7 +20,7 @@ const getKeysFromNode = function (node, pattern, callback) { for (const key of keys) { keySet[key] = true } - if (cursor==='0') { + if (cursor === '0') { // note redis returns string result not numeric return callback(null, Object.keys(keySet)) } else { From 24fd9449f23a4e06cc4564ed0520adfd771785d3 Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 15:52:19 +0100 Subject: [PATCH 08/10] Run format:fix --- .../test/cluster_failover/coffee/test_blpop_failover.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index 6d4b3ee55d..e3f52f6339 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -56,7 +56,6 @@ const listenForPing = (cb) => const PING_DELAY = 100 ;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() - ;(listenInBackground = () => listenForPing((error) => { if (error) { From 5961e85b1002a6616dfec16e68d3e3c39525148b Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 16:56:57 +0100 Subject: [PATCH 09/10] Fix path in test script --- services/document-updater/test/stress/js/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/test/stress/js/run.js b/services/document-updater/test/stress/js/run.js index 3ce482b19b..da78735a95 100644 --- a/services/document-updater/test/stress/js/run.js +++ b/services/document-updater/test/stress/js/run.js @@ -16,7 +16,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const DocUpdaterClient = require('../../acceptance/coffee/helpers/DocUpdaterClient') +const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient') // MockTrackChangesApi = require "../../acceptance/js/helpers/MockTrackChangesApi" // MockWebApi = require "../../acceptance/js/helpers/MockWebApi" const assert = require('assert') From 674ad4acf05c0e1d4870973e4cad939d592f3618 Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 25 May 2021 16:57:24 +0100 Subject: [PATCH 10/10] Rename folder from coffee to js --- .../test/cluster_failover/{coffee => js}/test_blpop_failover.js | 0 .../test/cluster_failover/{coffee => js}/test_pubsub_failover.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename services/document-updater/test/cluster_failover/{coffee => js}/test_blpop_failover.js (100%) rename services/document-updater/test/cluster_failover/{coffee => js}/test_pubsub_failover.js (100%) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/js/test_blpop_failover.js similarity index 100% rename from services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js rename to services/document-updater/test/cluster_failover/js/test_blpop_failover.js diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/js/test_pubsub_failover.js similarity index 100% rename from services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js rename to services/document-updater/test/cluster_failover/js/test_pubsub_failover.js