diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index 68e11cc486..c8c0be3c56 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -12,6 +12,8 @@ app.use express.logger() app.post "/doc/:doc_id/flush", HttpController.flushUpdatesWithLock +app.get "/project/:project_id/doc/:doc_id/diff", HttpController.getDiff + app.get "/status", (req, res, next) -> res.send "track-changes is alive" diff --git a/services/track-changes/app/coffee/DiffManager.coffee b/services/track-changes/app/coffee/DiffManager.coffee index 4c45661aa3..eddb461c53 100644 --- a/services/track-changes/app/coffee/DiffManager.coffee +++ b/services/track-changes/app/coffee/DiffManager.coffee @@ -2,6 +2,7 @@ HistoryManager = require "./HistoryManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" MongoManager = require "./MongoManager" DiffGenerator = require "./DiffGenerator" +logger = require "logger-sharelatex" module.exports = DiffManager = getLatestDocAndUpdates: (project_id, doc_id, fromDate, toDate, callback = (error, lines, version, updates) ->) -> @@ -14,20 +15,26 @@ module.exports = DiffManager = callback(null, lines, version, updates) getDiff: (project_id, doc_id, fromDate, toDate, callback = (error, diff) ->) -> + logger.log project_id: project_id, doc_id: doc_id, from: fromDate, to: toDate, "getting diff" DiffManager.getLatestDocAndUpdates project_id, doc_id, fromDate, null, (error, lines, version, updates) -> return callback(error) if error? - lastUpdate = updates[updates.length - 1] + + logger.log lines: lines, version: version, updates: updates, "got doc and updates" + + lastUpdate = updates[0] if lastUpdate? and lastUpdate.v != version return callback new Error("latest update version, #{lastUpdate.v}, does not match doc version, #{version}") - updatesToApply = [] - for update in updates - if update.meta.end_ts <= toDate + for update in updates.reverse() + if update.meta.start_ts <= toDate updatesToApply.push update + logger.log project_id: project_id, doc_id: doc_id, updatesToApply: updatesToApply, "got updates to apply" + try startingContent = DiffGenerator.rewindUpdates lines.join("\n"), updates + logger.log project_id: project_id, doc_id: doc_id, startingContent: startingContent, "rewound doc" diff = DiffGenerator.buildDiff startingContent, updatesToApply catch e return callback(e) diff --git a/services/track-changes/app/coffee/HttpController.coffee b/services/track-changes/app/coffee/HttpController.coffee index 1fd1926317..5b742e8891 100644 --- a/services/track-changes/app/coffee/HttpController.coffee +++ b/services/track-changes/app/coffee/HttpController.coffee @@ -1,4 +1,5 @@ HistoryManager = require "./HistoryManager" +DiffManager = require "./DiffManager" logger = require "logger-sharelatex" module.exports = HttpController = @@ -7,5 +8,23 @@ module.exports = HttpController = logger.log doc_id: doc_id, "compressing doc history" HistoryManager.processUncompressedUpdatesWithLock doc_id, (error) -> return next(error) if error? - logger.log "done http request" res.send 204 + + getDiff: (req, res, next = (error) ->) -> + doc_id = req.params.doc_id + project_id = req.params.project_id + + if req.query.from? + from = parseInt(req.query.from, 10) + else + from = null + if req.query.to? + to = parseInt(req.query.to, 10) + else + to = null + + logger.log project_id, doc_id: doc_id, from: from, to: to, "getting diff" + DiffManager.getDiff project_id, doc_id, from, to, (error, diff) -> + return next(error) if error? + res.send JSON.stringify(diff: diff) + diff --git a/services/track-changes/config/settings.development.coffee b/services/track-changes/config/settings.development.coffee index 56d0a90eb8..d3cf2978ec 100755 --- a/services/track-changes/config/settings.development.coffee +++ b/services/track-changes/config/settings.development.coffee @@ -5,3 +5,6 @@ module.exports = trackchanges: port: 3015 host: "localhost" + apis: + documentupdater: + url: "http://localhost:3003" diff --git a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee index 31c4a012eb..031659f24b 100644 --- a/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee @@ -3,31 +3,19 @@ chai = require("chai") chai.should() expect = chai.expect mongojs = require "../../../app/js/mongojs" -db = mongojs.db ObjectId = mongojs.ObjectId Settings = require "settings-sharelatex" request = require "request" rclient = require("redis").createClient() # Only works locally for now -flushAndGetCompressedUpdates = (doc_id, callback = (error, updates) ->) -> - request.post { - url: "http://localhost:3015/doc/#{doc_id}/flush" - }, (error, response, body) => - response.statusCode.should.equal 204 - db.docHistory - .find(doc_id: ObjectId(doc_id)) - .sort("meta.end_ts": 1) - .toArray callback - -pushRawUpdates = (doc_id, updates, callback = (error) ->) -> - rclient.rpush "UncompressedHistoryOps:#{doc_id}", (JSON.stringify(u) for u in updates)..., callback +TrackChangesClient = require "./helpers/TrackChangesClient" describe "Appending doc ops to the history", -> describe "when the history does not exist yet", -> before (done) -> @doc_id = ObjectId().toString() @user_id = ObjectId().toString() - pushRawUpdates @doc_id, [{ + TrackChangesClient.pushRawUpdates @doc_id, [{ op: [{ i: "f", p: 3 }] meta: { ts: Date.now(), user_id: @user_id } v: 3 @@ -41,7 +29,7 @@ describe "Appending doc ops to the history", -> v: 5 }], (error) => throw error if error? - flushAndGetCompressedUpdates @doc_id, (error, @updates) => + TrackChangesClient.flushAndGetCompressedUpdates @doc_id, (error, @updates) => throw error if error? done() @@ -57,7 +45,7 @@ describe "Appending doc ops to the history", -> beforeEach (done) -> @doc_id = ObjectId().toString() @user_id = ObjectId().toString() - pushRawUpdates @doc_id, [{ + TrackChangesClient.pushRawUpdates @doc_id, [{ op: [{ i: "f", p: 3 }] meta: { ts: Date.now(), user_id: @user_id } v: 3 @@ -71,13 +59,13 @@ describe "Appending doc ops to the history", -> v: 5 }], (error) => throw error if error? - flushAndGetCompressedUpdates @doc_id, (error, updates) => + TrackChangesClient.flushAndGetCompressedUpdates @doc_id, (error, updates) => throw error if error? done() describe "when the updates are recent and from the same user", -> beforeEach (done) -> - pushRawUpdates @doc_id, [{ + TrackChangesClient.pushRawUpdates @doc_id, [{ op: [{ i: "b", p: 6 }] meta: { ts: Date.now(), user_id: @user_id } v: 6 @@ -91,7 +79,7 @@ describe "Appending doc ops to the history", -> v: 8 }], (error) => throw error if error? - flushAndGetCompressedUpdates @doc_id, (error, @updates) => + TrackChangesClient.flushAndGetCompressedUpdates @doc_id, (error, @updates) => throw error if error? done() @@ -107,7 +95,7 @@ describe "Appending doc ops to the history", -> describe "when the updates are far apart", -> beforeEach (done) -> oneDay = 24 * 60 * 60 * 1000 - pushRawUpdates @doc_id, [{ + TrackChangesClient.pushRawUpdates @doc_id, [{ op: [{ i: "b", p: 6 }] meta: { ts: Date.now() + oneDay, user_id: @user_id } v: 6 @@ -121,7 +109,7 @@ describe "Appending doc ops to the history", -> v: 8 }], (error) => throw error if error? - flushAndGetCompressedUpdates @doc_id, (error, @updates) => + TrackChangesClient.flushAndGetCompressedUpdates @doc_id, (error, @updates) => throw error if error? done() @@ -147,9 +135,9 @@ describe "Appending doc ops to the history", -> } @expectedOp.i = "a" + @expectedOp.i - pushRawUpdates @doc_id, updates, (error) => + TrackChangesClient.pushRawUpdates @doc_id, updates, (error) => throw error if error? - flushAndGetCompressedUpdates @doc_id, (error, @updates) => + TrackChangesClient.flushAndGetCompressedUpdates @doc_id, (error, @updates) => throw error if error? done() diff --git a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee new file mode 100644 index 0000000000..02d83f0044 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee @@ -0,0 +1,70 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +expect = chai.expect +mongojs = require "../../../app/js/mongojs" +db = mongojs.db +ObjectId = mongojs.ObjectId +Settings = require "settings-sharelatex" + +TrackChangesClient = require "./helpers/TrackChangesClient" +MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi" + +describe "Getting a diff", -> + before (done) -> + sinon.spy MockDocUpdaterApi, "getDoc" + + @now = Date.now() + @from = @now - 100000000 + @to = @now + @user_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @project_id = ObjectId().toString() + + twoMinutes = 2 * 60 * 1000 + + @updates = [{ + op: [{ i: "one ", p: 0 }] + meta: { ts: @from - twoMinutes, user_id: @user_id } + v: 3 + }, { + op: [{ i: "two ", p: 4 }] + meta: { ts: @from + twoMinutes, user_id: @user_id } + v: 4 + }, { + op: [{ i: "three ", p: 8 }] + meta: { ts: @to - twoMinutes, user_id: @user_id } + v: 5 + }, { + op: [{ i: "four", p: 14 }] + meta: { ts: @to + twoMinutes, user_id: @user_id } + v: 6 + }] + @lines = ["one two three four"] + @expected_diff = [ + { u: "one " } + { i: "two ", meta: { start_ts: @from + twoMinutes, end_ts: @from + twoMinutes, user_id: @user_id } } + { i: "three ", meta: { start_ts: @to - twoMinutes, end_ts: @to - twoMinutes, user_id: @user_id } } + ] + + MockDocUpdaterApi.docs[@doc_id] = + lines: @lines + version: 6 + + TrackChangesClient.pushRawUpdates @doc_id, @updates, (error) => + throw error if error? + TrackChangesClient.getDiff @project_id, @doc_id, @from, @to, (error, diff) => + throw error if error? + @diff = diff.diff + done() + + after () -> + MockDocUpdaterApi.getDoc.restore() + + it "should return the diff", -> + expect(@diff).to.deep.equal @expected_diff + + it "should get the doc from the doc updater", -> + MockDocUpdaterApi.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true diff --git a/services/track-changes/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee b/services/track-changes/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee new file mode 100644 index 0000000000..a64098503c --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee @@ -0,0 +1,24 @@ +express = require("express") +app = express() + +module.exports = MockDocUpdaterApi = + docs: {} + + getDoc: (project_id, doc_id, callback = (error) ->) -> + callback null, @docs[doc_id] + + run: () -> + app.get "/project/:project_id/doc/:doc_id", (req, res, next) => + @getDoc req.params.project_id, req.params.doc_id, (error, doc) -> + if error? + res.send 500 + if !doc? + res.send 404 + else + res.send JSON.stringify doc + + app.listen 3003, (error) -> + throw error if error? + +MockDocUpdaterApi.run() + diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee new file mode 100644 index 0000000000..52a5552dd6 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -0,0 +1,24 @@ +request = require "request" +rclient = require("redis").createClient() # Only works locally for now +{db, ObjectId} = require "../../../../app/js/mongojs" + +module.exports = TrackChangesClient = + flushAndGetCompressedUpdates: (doc_id, callback = (error, updates) ->) -> + request.post { + url: "http://localhost:3015/doc/#{doc_id}/flush" + }, (error, response, body) => + response.statusCode.should.equal 204 + db.docHistory + .find(doc_id: ObjectId(doc_id)) + .sort("meta.end_ts": 1) + .toArray callback + + pushRawUpdates: (doc_id, updates, callback = (error) ->) -> + rclient.rpush "UncompressedHistoryOps:#{doc_id}", (JSON.stringify(u) for u in updates)..., callback + + getDiff: (project_id, doc_id, from, to, callback = (error, diff) ->) -> + request.get { + url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/diff?from=#{from}&to=#{to}" + }, (error, response, body) => + response.statusCode.should.equal 200 + callback null, JSON.parse(body) \ No newline at end of file diff --git a/services/track-changes/test/unit/coffee/DiffManager/DiffManagerTests.coffee b/services/track-changes/test/unit/coffee/DiffManager/DiffManagerTests.coffee index 332a73120d..391ad138cb 100644 --- a/services/track-changes/test/unit/coffee/DiffManager/DiffManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/DiffManager/DiffManagerTests.coffee @@ -53,10 +53,12 @@ describe "DiffManager", -> @lines = [ "hello", "world" ] @version = 42 @updates = [ - { op: "mock-1", v: 41, meta: { end_ts: new Date(@to.getTime() - 10)} } - { op: "mock-2", v: 42, meta: { end_ts: new Date(@to.getTime() + 10)} } + { op: "mock-4", v: 42, meta: { start_ts: new Date(@to.getTime() + 20)} } + { op: "mock-3", v: 41, meta: { start_ts: new Date(@to.getTime() + 10)} } + { op: "mock-2", v: 40, meta: { start_ts: new Date(@to.getTime() - 10)} } + { op: "mock-1", v: 39, meta: { start_ts: new Date(@to.getTime() - 20)} } ] - @diffed_updates = @updates.slice(0,1) + @diffed_updates = @updates.slice(2) @rewound_content = "rewound-content" @diff = [ u: "mock-diff" ] @@ -79,7 +81,7 @@ describe "DiffManager", -> it "should generate the diff", -> @DiffGenerator.buildDiff - .calledWith(@rewound_content, @diffed_updates) + .calledWith(@rewound_content, @diffed_updates.reverse()) .should.equal true it "should call the callback with the diff", -> @@ -88,7 +90,7 @@ describe "DiffManager", -> describe "with mismatching versions", -> beforeEach -> @version = 42 - @updates = [ { op: "mock-1", v: 39 }, { op: "mock-1", v: 40 } ] + @updates = [ { op: "mock-1", v: 40 }, { op: "mock-1", v: 39 } ] @DiffManager.getLatestDocAndUpdates = sinon.stub().callsArgWith(4, null, @lines, @version, @updates) @DiffManager.getDiff @project_id, @doc_id, @from, @to, @callback diff --git a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee index a0d7c23166..89e03ff6ea 100644 --- a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee +++ b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee @@ -10,7 +10,9 @@ describe "HttpController", -> @HttpController = SandboxedModule.require modulePath, requires: "logger-sharelatex": { log: sinon.stub() } "./HistoryManager": @HistoryManager = {} + "./DiffManager": @DiffManager = {} @doc_id = "doc-id-123" + @project_id = "project-id-123" @version = 42 @next = sinon.stub() @@ -30,4 +32,29 @@ describe "HttpController", -> .should.equal true it "should return a success code", -> - @res.send.calledWith(204).should.equal true \ No newline at end of file + @res.send.calledWith(204).should.equal true + + describe "getDiff", -> + beforeEach -> + @from = Date.now() - 10000 + @to = Date.now() + @req = + params: + doc_id: @doc_id + project_id: @project_id + query: + from: @from.toString() + to: @to.toString() + @res = + send: sinon.stub() + @diff = [ u: "mock-diff" ] + @DiffManager.getDiff = sinon.stub().callsArgWith(4, null, @diff) + @HttpController.getDiff @req, @res, @next + + it "should get the diff", -> + @DiffManager.getDiff + .calledWith(@project_id, @doc_id, parseInt(@from, 10), parseInt(@to, 10)) + .should.equal true + + it "should return the diff", -> + @res.send.calledWith(JSON.stringify(diff: @diff)).should.equal true