From afd4a10a19342216a58360e525b0c8c74fd3fa0a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 4 May 2018 13:43:12 +0100 Subject: [PATCH] retry lock with exponential backoff --- services/web/app/coffee/infrastructure/LockManager.coffee | 6 +++++- .../infrastructure/LockManager/ReleasingTheLock.coffee | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/infrastructure/LockManager.coffee b/services/web/app/coffee/infrastructure/LockManager.coffee index 60f6d00cc0..b117ce49ea 100644 --- a/services/web/app/coffee/infrastructure/LockManager.coffee +++ b/services/web/app/coffee/infrastructure/LockManager.coffee @@ -13,6 +13,7 @@ COUNT = 0 module.exports = LockManager = LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock + MAX_TEST_INTERVAL: 1000 # back off to 1s between each test of the lock MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock REDIS_LOCK_EXPIRY: 30 # seconds. Time until lock auto expires in redis SLOW_EXECUTION_THRESHOLD: 5000 # 5s, if execution takes longer than this then log @@ -69,6 +70,7 @@ module.exports = LockManager = _getLock: (key, namespace, callback = (error, lockValue) ->) -> startTime = Date.now() + testInterval = LockManager.LOCK_TEST_INTERVAL attempts = 0 do attempt = () -> if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME @@ -82,7 +84,9 @@ module.exports = LockManager = metrics.gauge "lock.#{namespace}.get.success.tries", attempts callback(null, lockValue) else - setTimeout attempt, LockManager.LOCK_TEST_INTERVAL + setTimeout attempt, testInterval + # back off when the lock is taken to avoid overloading + testInterval = Math.min(testInterval * 2, LockManager.MAX_TEST_INTERVAL) _releaseLock: (key, lockValue, callback)-> rclient.eval LockManager.unlockScript, 1, key, lockValue, (err, result) -> diff --git a/services/web/test/unit/coffee/infrastructure/LockManager/ReleasingTheLock.coffee b/services/web/test/unit/coffee/infrastructure/LockManager/ReleasingTheLock.coffee index 1fbe7b07f4..76747defbb 100644 --- a/services/web/test/unit/coffee/infrastructure/LockManager/ReleasingTheLock.coffee +++ b/services/web/test/unit/coffee/infrastructure/LockManager/ReleasingTheLock.coffee @@ -17,7 +17,6 @@ describe 'LockManager - releasing the lock', ()-> auth:-> eval:deleteStub - LockManager = SandboxedModule.require(modulePath, requires: mocks) LockManager.unlockScript = "this is the unlock script"