From 8b0b79bc322758e9f7bd1905edcf147859940392 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 16 May 2014 15:59:12 +0100 Subject: [PATCH] Use TTL index to auto-delete updates after a week when versioning is not enabled --- .../app/coffee/MongoManager.coffee | 18 ++++-- .../app/coffee/UpdateTrimmer.coffee | 4 +- .../app/coffee/UpdatesManager.coffee | 38 ++++++------ .../coffee/AppendingUpdatesTests.coffee | 46 +++++++++++++++ .../coffee/FlushingUpdatesTests.coffee | 1 + .../coffee/GettingADiffTests.coffee | 1 + .../coffee/RestoringVersions.coffee | 1 + .../MongoManager/MongoManagerTests.coffee | 59 ++++++++++++++----- .../UpdateTrimmer/UpdateTrimmerTests.coffee | 16 ++--- .../UpdatesManager/UpdatesManagerTests.coffee | 37 +++++++----- 10 files changed, 157 insertions(+), 64 deletions(-) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 0ded71fb63..7bdedcbe98 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -24,21 +24,24 @@ module.exports = MongoManager = else callback null, null - insertCompressedUpdates: (project_id, doc_id, updates, callback = (error) ->) -> + insertCompressedUpdates: (project_id, doc_id, updates, permanent, callback = (error) ->) -> jobs = [] for update in updates do (update) -> - jobs.push (callback) -> MongoManager.insertCompressedUpdate project_id, doc_id, update, callback + jobs.push (callback) -> MongoManager.insertCompressedUpdate project_id, doc_id, update, permanent, callback async.series jobs, callback - insertCompressedUpdate: (project_id, doc_id, update, callback = (error) ->) -> - db.docHistory.insert { + insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) -> + update = { doc_id: ObjectId(doc_id.toString()) project_id: ObjectId(project_id.toString()) op: update.op meta: update.meta v: update.v - }, callback + } + if temporary + update.tempCreatedAt = new Date() + db.docHistory.insert update, callback getDocUpdates:(doc_id, options = {}, callback = (error, updates) ->) -> query = @@ -116,3 +119,8 @@ module.exports = MongoManager = db.docHistory.ensureIndex { doc_id: 1, project_id: 1 } # For finding project meta-data db.projectHistoryMetaData.ensureIndex { project_id: 1 } + # TTL index for auto deleting week old temporary ops + minutes = 60 + hours = 60 * minutes + days = 24 * hours + db.docHistory.ensureIndex { tempCreatedAt: 1 }, { expireAfterSeconds: 7 * days } diff --git a/services/track-changes/app/coffee/UpdateTrimmer.coffee b/services/track-changes/app/coffee/UpdateTrimmer.coffee index 12a673eddc..649a801a43 100644 --- a/services/track-changes/app/coffee/UpdateTrimmer.coffee +++ b/services/track-changes/app/coffee/UpdateTrimmer.coffee @@ -3,7 +3,7 @@ WebApiManager = require "./WebApiManager" logger = require "logger-sharelatex" module.exports = UpdateTrimmer = - _shouldTrimUpdates: (project_id, callback = (error, shouldTrim) ->) -> + shouldTrimUpdates: (project_id, callback = (error, shouldTrim) ->) -> MongoManager.getProjectMetaData project_id, (error, metadata) -> return callback(error) if error? if metadata?.preserveHistory @@ -20,7 +20,7 @@ module.exports = UpdateTrimmer = callback null, true deleteOldProjectUpdates: (project_id, callback = (error) ->) -> - UpdateTrimmer._shouldTrimUpdates project_id, (error, shouldTrim) -> + UpdateTrimmer.shouldTrimUpdates project_id, (error, shouldTrim) -> return callback(error) if error? if shouldTrim logger.log project_id: project_id, "deleting old updates" diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index e0e531e957..0aae777695 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -8,7 +8,7 @@ logger = require "logger-sharelatex" async = require "async" module.exports = UpdatesManager = - compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, callback = (error) ->) -> + compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) -> length = rawUpdates.length if length == 0 return callback() @@ -27,37 +27,39 @@ module.exports = UpdatesManager = logger.error err: error, doc_id: doc_id, project_id: project_id, "inconsistent doc versions" # Push the update back into Mongo - catching errors at this # point is useless, we're already bailing - MongoManager.insertCompressedUpdates project_id, doc_id, [lastCompressedUpdate], () -> + MongoManager.insertCompressedUpdates project_id, doc_id, [lastCompressedUpdate], temporary, () -> return callback error return compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates - MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, (error) -> + MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) -> return callback(error) if error? logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" callback() REDIS_READ_BATCH_SIZE: 100 processUncompressedUpdates: (project_id, doc_id, callback = (error) ->) -> - MongoManager.backportProjectId project_id, doc_id, (error) -> + UpdateTrimmer.shouldTrimUpdates project_id, (error, temporary) -> return callback(error) if error? - RedisManager.getOldestRawUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) -> + MongoManager.backportProjectId project_id, doc_id, (error) -> return callback(error) if error? - length = rawUpdates.length - UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, (error) -> + RedisManager.getOldestRawUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) -> return callback(error) if error? - logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates" - RedisManager.deleteOldestRawUpdates project_id, doc_id, length, (error) -> + length = rawUpdates.length + UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, temporary, (error) -> return callback(error) if error? - if length == UpdatesManager.REDIS_READ_BATCH_SIZE - # There might be more updates - logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates" - setTimeout () -> - UpdatesManager.processUncompressedUpdates project_id, doc_id, callback - , 0 - else - logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed" - callback() + logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates" + RedisManager.deleteOldestRawUpdates project_id, doc_id, length, (error) -> + return callback(error) if error? + if length == UpdatesManager.REDIS_READ_BATCH_SIZE + # There might be more updates + logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates" + setTimeout () -> + UpdatesManager.processUncompressedUpdates project_id, doc_id, callback + , 0 + else + logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed" + callback() processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) -> LockManager.runWithLock( diff --git a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee index b86da90c19..070ac4eb66 100644 --- a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee @@ -9,6 +9,7 @@ request = require "request" rclient = require("redis").createClient() # Only works locally for now TrackChangesClient = require "./helpers/TrackChangesClient" +MockWebApi = require "./helpers/MockWebApi" describe "Appending doc ops to the history", -> describe "when the history does not exist yet", -> @@ -16,6 +17,7 @@ describe "Appending doc ops to the history", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [{ i: "f", p: 3 }] meta: { ts: Date.now(), user_id: @user_id } @@ -58,6 +60,7 @@ describe "Appending doc ops to the history", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [{ i: "f", p: 3 }] meta: { ts: Date.now(), user_id: @user_id } @@ -139,6 +142,7 @@ describe "Appending doc ops to the history", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false updates = [] @expectedOp = [{ p:0, i: "" }] for i in [0..250] @@ -167,6 +171,7 @@ describe "Appending doc ops to the history", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false oneDay = 24 * 60 * 60 * 1000 TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [{ i: "f", p: 3 }, { i: "o", p: 4 }, { i: "o", p: 5 }] @@ -199,6 +204,7 @@ describe "Appending doc ops to the history", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false oneDay = 24 * 60 * 60 * 1000 TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [] @@ -226,3 +232,43 @@ describe "Appending doc ops to the history", -> it "should insert the correct version numbers into mongo", -> expect(@updates[0].v).to.equal 3 expect(@updates[1].v).to.equal 4 + + describe "when the project has versioning enabled", -> + before (done) -> + @project_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: true + + TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ + op: [{ i: "f", p: 3 }] + meta: { ts: Date.now(), user_id: @user_id } + v: 3 + }], (error) => + throw error if error? + TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) => + throw error if error? + done() + + it "should not add a tempCreatedAt entry in the update in mongo", -> + expect(@updates[0].tempCreatedAt).to.be.undefined + + describe "when the project does not have versioning enabled", -> + before (done) -> + @project_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: false + + TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ + op: [{ i: "f", p: 3 }] + meta: { ts: Date.now(), user_id: @user_id } + v: 3 + }], (error) => + throw error if error? + TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) => + throw error if error? + done() + + it "should add a tempCreatedAt entry in the update in mongo", -> + expect(@updates[0].tempCreatedAt).to.exist diff --git a/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee index ee225858ee..80b6275962 100644 --- a/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee @@ -17,6 +17,7 @@ describe "Flushing updates", -> @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @user_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: true TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{ op: [{ i: "f", p: 3 }] diff --git a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee index 7561b68673..1285177bf9 100644 --- a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee @@ -21,6 +21,7 @@ describe "Getting a diff", -> @user_id = ObjectId().toString() @doc_id = ObjectId().toString() @project_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: true MockWebApi.users[@user_id] = @user = email: "user@sharelatex.com" diff --git a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee index 7e3c29a0e7..ebe9a7795e 100644 --- a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee +++ b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee @@ -19,6 +19,7 @@ describe "Restoring a version", -> @user_id = ObjectId().toString() @doc_id = ObjectId().toString() @project_id = ObjectId().toString() + MockWebApi.projects[@project_id] = features: versioning: true minutes = 60 * 1000 diff --git a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee index 2228147b75..737500ee48 100644 --- a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee @@ -5,15 +5,20 @@ expect = chai.expect modulePath = "../../../../app/js/MongoManager.js" SandboxedModule = require('sandboxed-module') {ObjectId} = require("mongojs") +tk = require "timekeeper" describe "MongoManager", -> beforeEach -> + tk.freeze(new Date()) @MongoManager = SandboxedModule.require modulePath, requires: "./mongojs" : { db: @db = {}, ObjectId: ObjectId } @callback = sinon.stub() @doc_id = ObjectId().toString() @project_id = ObjectId().toString() + afterEach -> + tk.reset() + describe "getLastCompressedUpdate", -> beforeEach -> @update = "mock-update" @@ -101,34 +106,56 @@ describe "MongoManager", -> @update = { op: "op", meta: "meta", v: "v"} @db.docHistory = insert: sinon.stub().callsArg(1) - @MongoManager.insertCompressedUpdate @project_id, @doc_id, @update, @callback - it "should insert the update", -> - @db.docHistory.insert - .calledWith({ - project_id: ObjectId(@project_id), - doc_id: ObjectId(@doc_id), - op: @update.op, - meta: @update.meta, - v: @update.v - }) - .should.equal true + describe "temporarly", -> + beforeEach -> + @MongoManager.insertCompressedUpdate @project_id, @doc_id, @update, true, @callback - it "should call the callback", -> - @callback.called.should.equal true + it "should insert the update with a tempCreatedAt field", -> + @db.docHistory.insert + .calledWith({ + project_id: ObjectId(@project_id), + doc_id: ObjectId(@doc_id), + op: @update.op, + meta: @update.meta, + v: @update.v + tempCreatedAt: new Date() + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "permanenty", -> + beforeEach -> + @MongoManager.insertCompressedUpdate @project_id, @doc_id, @update, false, @callback + + it "should insert the update with no tempCreatedAt field", -> + @db.docHistory.insert + .calledWith({ + project_id: ObjectId(@project_id), + doc_id: ObjectId(@doc_id), + op: @update.op, + meta: @update.meta, + v: @update.v + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true describe "insertCompressedUpdates", -> beforeEach (done) -> @updates = [ "mock-update-1", "mock-update-2" ] - @MongoManager.insertCompressedUpdate = sinon.stub().callsArg(3) - @MongoManager.insertCompressedUpdates @project_id, @doc_id, @updates, (args...) => + @MongoManager.insertCompressedUpdate = sinon.stub().callsArg(4) + @MongoManager.insertCompressedUpdates @project_id, @doc_id, @updates, @temporary = true, (args...) => @callback(args...) done() it "should insert each update", -> for update in @updates @MongoManager.insertCompressedUpdate - .calledWith(@project_id, @doc_id, update) + .calledWith(@project_id, @doc_id, update, @temporary) .should.equal true it "should call the callback", -> diff --git a/services/track-changes/test/unit/coffee/UpdateTrimmer/UpdateTrimmerTests.coffee b/services/track-changes/test/unit/coffee/UpdateTrimmer/UpdateTrimmerTests.coffee index aa0ac00f78..10b0599c23 100644 --- a/services/track-changes/test/unit/coffee/UpdateTrimmer/UpdateTrimmerTests.coffee +++ b/services/track-changes/test/unit/coffee/UpdateTrimmer/UpdateTrimmerTests.coffee @@ -22,7 +22,7 @@ describe "UpdateTrimmer", -> afterEach -> tk.reset() - describe "_shouldTrimUpdates", -> + describe "shouldTrimUpdates", -> beforeEach -> @metadata = {} @details = @@ -34,7 +34,7 @@ describe "UpdateTrimmer", -> describe "with preserveHistory set in the project meta data", -> beforeEach -> @metadata.preserveHistory = true - @UpdateTrimmer._shouldTrimUpdates @project_id, @callback + @UpdateTrimmer.shouldTrimUpdates @project_id, @callback it "should look up the meta data", -> @MongoManager.getProjectMetaData @@ -56,7 +56,7 @@ describe "UpdateTrimmer", -> describe "when the project has the versioning feature", -> beforeEach -> @details.features.versioning = true - @UpdateTrimmer._shouldTrimUpdates @project_id, @callback + @UpdateTrimmer.shouldTrimUpdates @project_id, @callback it "should look up the meta data", -> @MongoManager.getProjectMetaData @@ -79,7 +79,7 @@ describe "UpdateTrimmer", -> describe "when the project does not have the versioning feature", -> beforeEach -> @details.features.versioning = false - @UpdateTrimmer._shouldTrimUpdates @project_id, @callback + @UpdateTrimmer.shouldTrimUpdates @project_id, @callback it "should return true", -> @callback.calledWith(null, true).should.equal true @@ -91,7 +91,7 @@ describe "UpdateTrimmer", -> describe "when the project has the versioning feature", -> beforeEach -> @details.features.versioning = true - @UpdateTrimmer._shouldTrimUpdates @project_id, @callback + @UpdateTrimmer.shouldTrimUpdates @project_id, @callback it "should insert preserveHistory into the metadata", -> @MongoManager.setProjectMetaData @@ -104,7 +104,7 @@ describe "UpdateTrimmer", -> describe "when the project does not have the versioning feature", -> beforeEach -> @details.features.versioning = false - @UpdateTrimmer._shouldTrimUpdates @project_id, @callback + @UpdateTrimmer.shouldTrimUpdates @project_id, @callback it "should return true", -> @callback.calledWith(null, true).should.equal true @@ -116,7 +116,7 @@ describe "UpdateTrimmer", -> describe "when the updates should be trimmed", -> beforeEach -> - @UpdateTrimmer._shouldTrimUpdates = sinon.stub().callsArgWith(1, null, true) + @UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, true) @UpdateTrimmer.deleteOldProjectUpdates @project_id, @callback it "should delete week old updates in mongo", -> @@ -130,7 +130,7 @@ describe "UpdateTrimmer", -> describe "when the updates should not be trimmed", -> beforeEach -> - @UpdateTrimmer._shouldTrimUpdates = sinon.stub().callsArgWith(1, null, false) + @UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, false) @UpdateTrimmer.deleteOldProjectUpdates @project_id, @callback it "should not delete any updates in mongo", -> diff --git a/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee b/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee index 0e50ee3e20..12554c817b 100644 --- a/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee @@ -18,13 +18,14 @@ describe "UpdatesManager", -> @doc_id = "doc-id-123" @project_id = "project-id-123" @callback = sinon.stub() + @temporary = "temp-mock" describe "compressAndSaveRawUpdates", -> describe "when there are no raw ops", -> beforeEach -> @MongoManager.popLastCompressedUpdate = sinon.stub() @MongoManager.insertCompressedUpdates = sinon.stub() - @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, [], @callback + @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, [], @temporary, @callback it "should not need to access the database", -> @MongoManager.popLastCompressedUpdate.called.should.equal false @@ -39,9 +40,9 @@ describe "UpdatesManager", -> @compressedUpdates = { v: 13, op: "compressed-op-12" } @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null) - @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(3) + @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(4) @UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates) - @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @callback + @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback it "should try to pop the last compressed op", -> @MongoManager.popLastCompressedUpdate @@ -55,7 +56,7 @@ describe "UpdatesManager", -> it "should save the compressed ops", -> @MongoManager.insertCompressedUpdates - .calledWith(@project_id, @doc_id, @compressedUpdates) + .calledWith(@project_id, @doc_id, @compressedUpdates, @temporary) .should.equal true it "should call the callback", -> @@ -67,13 +68,13 @@ describe "UpdatesManager", -> @compressedUpdates = { v: 13, op: "compressed-op-12" } @MongoManager.popLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate) - @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(3) + @MongoManager.insertCompressedUpdates = sinon.stub().callsArg(4) @UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates) describe "when the raw ops start where the existing history ends", -> beforeEach -> @rawUpdates = [{ v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] - @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @callback + @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback it "should try to pop the last compressed op", -> @MongoManager.popLastCompressedUpdate @@ -87,7 +88,7 @@ describe "UpdatesManager", -> it "should save the compressed ops", -> @MongoManager.insertCompressedUpdates - .calledWith(@project_id, @doc_id, @compressedUpdates) + .calledWith(@project_id, @doc_id, @compressedUpdates, @temporary) .should.equal true it "should call the callback", -> @@ -97,7 +98,7 @@ describe "UpdatesManager", -> beforeEach -> @rawUpdates = [{ v: 10, op: "mock-op-10" }, { v: 11, op: "mock-op-11"}, { v: 12, op: "mock-op-12" }, { v: 13, op: "mock-op-13" }] - @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @callback + @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback it "should only compress the more recent raw ops", -> @UpdateCompressor.compressRawUpdates @@ -107,7 +108,7 @@ describe "UpdatesManager", -> describe "when the raw ops do not follow from the last compressed op version", -> beforeEach -> @rawUpdates = [{ v: 13, op: "mock-op-13" }] - @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @callback + @UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, @rawUpdates, @temporary, @callback it "should call the callback with an error", -> @callback @@ -117,14 +118,15 @@ describe "UpdatesManager", -> it "should put the popped update back into mongo", -> @MongoManager.insertCompressedUpdates.calledOnce.should.equal true @MongoManager.insertCompressedUpdates - .calledWith(@project_id, @doc_id, [@lastCompressedUpdate]) + .calledWith(@project_id, @doc_id, [@lastCompressedUpdate], @temporary) .should.equal true describe "processUncompressedUpdates", -> beforeEach -> - @UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(3) + @UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(4) @RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(3) @MongoManager.backportProjectId = sinon.stub().callsArg(2) + @UpdateTrimmer.shouldTrimUpdates = sinon.stub().callsArgWith(1, null, @temporary = "temp mock") describe "when there is fewer than one batch to send", -> beforeEach -> @@ -132,6 +134,11 @@ describe "UpdatesManager", -> @RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates) @UpdatesManager.processUncompressedUpdates @project_id, @doc_id, @callback + it "should check if the updates are temporary", -> + @UpdateTrimmer.shouldTrimUpdates + .calledWith(@project_id) + .should.equal true + it "should backport the project id", -> @MongoManager.backportProjectId .calledWith(@project_id, @doc_id) @@ -144,7 +151,7 @@ describe "UpdatesManager", -> it "should compress and save the updates", -> @UpdatesManager.compressAndSaveRawUpdates - .calledWith(@project_id, @doc_id, @updates) + .calledWith(@project_id, @doc_id, @updates, @temporary) .should.equal true it "should delete the batch of uncompressed updates that was just processed", -> @@ -174,13 +181,13 @@ describe "UpdatesManager", -> it "should compress and save the updates in batches", -> @UpdatesManager.compressAndSaveRawUpdates - .calledWith(@project_id, @doc_id, @updates.slice(0,2)) + .calledWith(@project_id, @doc_id, @updates.slice(0,2), @temporary) .should.equal true @UpdatesManager.compressAndSaveRawUpdates - .calledWith(@project_id, @doc_id, @updates.slice(2,4)) + .calledWith(@project_id, @doc_id, @updates.slice(2,4), @temporary) .should.equal true @UpdatesManager.compressAndSaveRawUpdates - .calledWith(@project_id, @doc_id, @updates.slice(4,5)) + .calledWith(@project_id, @doc_id, @updates.slice(4,5), @temporary) .should.equal true it "should delete the batches of uncompressed updates", ->