From 65360a9a2b5c7976c9a9394f0bed22623d93edd8 Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Wed, 26 Feb 2014 11:18:26 +0000
Subject: [PATCH] Continue to process raw updates until redis is empty

---
 .../app/coffee/HistoryManager.coffee          |  9 ++-
 .../HistoryManager/HistoryManagerTests.coffee | 77 ++++++++++++++-----
 2 files changed, 65 insertions(+), 21 deletions(-)

diff --git a/services/track-changes/app/coffee/HistoryManager.coffee b/services/track-changes/app/coffee/HistoryManager.coffee
index e11e911698..89319ca9e7 100644
--- a/services/track-changes/app/coffee/HistoryManager.coffee
+++ b/services/track-changes/app/coffee/HistoryManager.coffee
@@ -32,11 +32,18 @@ module.exports = HistoryManager =
 	processUncompressedUpdates: (doc_id, callback = (error) ->) ->
 		RedisManager.getOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) ->
 			return callback(error) if error?
+			length = rawUpdates.length
 			HistoryManager.compressAndSaveRawUpdates doc_id, rawUpdates, (error) ->
 				return callback(error) if error?
 				RedisManager.deleteOldestRawUpdates doc_id, HistoryManager.REDIS_READ_BATCH_SIZE, (error) ->
 					return callback(error) if error?
-					callback()
+					if length == HistoryManager.REDIS_READ_BATCH_SIZE
+						# There might be more updates
+						setTimeout () ->
+							HistoryManager.processUncompressedUpdates doc_id, callback
+						, 0
+					else
+						callback()
 
 	processUncompressedUpdatesWithLock: (doc_id, callback = (error) ->) ->
 		LockManager.runWithLock(
diff --git a/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee b/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
index 0770b0518e..82d61cd21d 100644
--- a/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
+++ b/services/track-changes/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee
@@ -112,30 +112,67 @@ describe "HistoryManager", ->
 						.should.equal true
 
 	describe "processUncompressedUpdates", ->
-		beforeEach ->
-			@updates = ["mock-update"]
-			@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
-			@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
-			@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
-			@HistoryManager.processUncompressedUpdates @doc_id, @callback
+		describe "when there is fewer than one batch to send", ->
+			beforeEach ->
+				@updates = ["mock-update"]
+				@RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates)
+				@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
+				@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
+				@HistoryManager.processUncompressedUpdates @doc_id, @callback
 
-		it "should get the oldest updates", ->
-			@RedisManager.getOldestRawUpdates
-				.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
-				.should.equal true
+			it "should get the oldest updates", ->
+				@RedisManager.getOldestRawUpdates
+					.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
+					.should.equal true
 
-		it "should compress and save the updates", ->
-			@HistoryManager.compressAndSaveRawUpdates
-				.calledWith(@doc_id, @updates)
-				.should.equal true
+			it "should compress and save the updates", ->
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates)
+					.should.equal true
 
-		it "should delete the batch of uncompressed updates that was just processed", ->
-			@RedisManager.deleteOldestRawUpdates
-				.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
-				.should.equal true
+			it "should delete the batch of uncompressed updates that was just processed", ->
+				@RedisManager.deleteOldestRawUpdates
+					.calledWith(@doc_id, @HistoryManager.REDIS_READ_BATCH_SIZE)
+					.should.equal true
 
-		it "should call the callback", ->
-			@callback.called.should.equal true
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "when there are multiple batches to send", ->
+			beforeEach (done) ->
+				@HistoryManager.REDIS_READ_BATCH_SIZE = 2
+				@updates = ["mock-update-0", "mock-update-1", "mock-update-2", "mock-update-3", "mock-update-4"]
+				@redisArray = @updates.slice()
+				@RedisManager.getOldestRawUpdates = (doc_id, batchSize, callback = (error, updates) ->) =>
+					updates = @redisArray.slice(0, batchSize)
+					@redisArray = @redisArray.slice(batchSize)
+					callback null, updates
+				sinon.spy @RedisManager, "getOldestRawUpdates"
+				@HistoryManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(2)
+				@RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(2)
+				@HistoryManager.processUncompressedUpdates @doc_id, (args...) =>
+					@callback(args...)
+					done()
+
+			it "should get the oldest updates in three batches ", ->
+				@RedisManager.getOldestRawUpdates.callCount.should.equal 3
+
+			it "should compress and save the updates in batches", ->
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(0,2))
+					.should.equal true
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(2,4))
+					.should.equal true
+				@HistoryManager.compressAndSaveRawUpdates
+					.calledWith(@doc_id, @updates.slice(4,5))
+					.should.equal true
+
+			it "should delete the batches of uncompressed updates", ->
+				@RedisManager.deleteOldestRawUpdates.callCount.should.equal 3
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
 
 	describe "processCompressedUpdatesWithLock", ->
 		beforeEach ->