diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 71f926a745..330a903a8a 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -75,7 +75,21 @@ module.exports = MongoManager = cursor.toArray callback - ensureIndices: (callback = (error) ->) -> - db.docHistory.ensureIndex { doc_id: 1, v: 1 }, callback - db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, callback + backportProjectId: (project_id, doc_id, callback = (error) ->) -> + db.docHistory.update { + doc_id: ObjectId(doc_id.toString()) + project_id: { $exists: false } + }, { + $set: { project_id: ObjectId(project_id.toString()) } + }, { + multi: true + }, callback + + ensureIndices: (callback = (error) ->) -> + # For finding all updates that go into a diff for a doc + db.docHistory.ensureIndex { doc_id: 1, v: 1 }, callback + # For finding all updates that affect a project + db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, callback + # For finding updates that don't yet have a project_id and need it inserting + db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, callback diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 42cd38fad4..c13b42d326 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -38,23 +38,25 @@ module.exports = UpdatesManager = REDIS_READ_BATCH_SIZE: 100 processUncompressedUpdates: (project_id, doc_id, callback = (error) ->) -> - RedisManager.getOldestRawUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) -> + MongoManager.backportProjectId project_id, doc_id, (error) -> return callback(error) if error? - length = rawUpdates.length - UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, (error) -> + RedisManager.getOldestRawUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, rawUpdates) -> return callback(error) if error? - logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates" - RedisManager.deleteOldestRawUpdates project_id, doc_id, length, (error) -> + length = rawUpdates.length + UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, (error) -> return callback(error) if error? - if length == UpdatesManager.REDIS_READ_BATCH_SIZE - # There might be more updates - logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates" - setTimeout () -> - UpdatesManager.processUncompressedUpdates project_id, doc_id, callback - , 0 - else - logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed" - callback() + logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates" + RedisManager.deleteOldestRawUpdates project_id, doc_id, length, (error) -> + return callback(error) if error? + if length == UpdatesManager.REDIS_READ_BATCH_SIZE + # There might be more updates + logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates" + setTimeout () -> + UpdatesManager.processUncompressedUpdates project_id, doc_id, callback + , 0 + else + logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed" + callback() processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) -> LockManager.runWithLock( diff --git a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee index c5609994a1..b3a8fc04ca 100644 --- a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee @@ -252,3 +252,25 @@ describe "MongoManager", -> @db.docHistory.limit .calledWith(@limit) .should.equal true + + describe "backportProjectId", -> + beforeEach -> + @db.docHistory = + update: sinon.stub().callsArg(3) + @MongoManager.backportProjectId @project_id, @doc_id, @callback + + it "should insert the project_id into all entries for the doc_id which don't have it set", -> + @db.docHistory.update + .calledWith({ + doc_id: ObjectId(@doc_id) + project_id: { $exists: false } + }, { + $set: { project_id: ObjectId(@project_id) } + }, { + multi: true + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + diff --git a/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee b/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee index 9bdf5fd635..6ff9f64e11 100644 --- a/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee @@ -120,14 +120,22 @@ describe "UpdatesManager", -> .should.equal true describe "processUncompressedUpdates", -> + beforeEach -> + @UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(3) + @RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(3) + @MongoManager.backportProjectId = sinon.stub().callsArg(2) + describe "when there is fewer than one batch to send", -> beforeEach -> @updates = ["mock-update"] @RedisManager.getOldestRawUpdates = sinon.stub().callsArgWith(2, null, @updates) - @UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(3) - @RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(3) @UpdatesManager.processUncompressedUpdates @project_id, @doc_id, @callback + it "should backport the project id", -> + @MongoManager.backportProjectId + .calledWith(@project_id, @doc_id) + .should.equal true + it "should get the oldest updates", -> @RedisManager.getOldestRawUpdates .calledWith(@doc_id, @UpdatesManager.REDIS_READ_BATCH_SIZE) @@ -156,8 +164,6 @@ describe "UpdatesManager", -> @redisArray = @redisArray.slice(batchSize) callback null, updates sinon.spy @RedisManager, "getOldestRawUpdates" - @UpdatesManager.compressAndSaveRawUpdates = sinon.stub().callsArgWith(3) - @RedisManager.deleteOldestRawUpdates = sinon.stub().callsArg(3) @UpdatesManager.processUncompressedUpdates @project_id, @doc_id, (args...) => @callback(args...) done()