diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index 8d5fd9c082..f5f12078ac 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -27,6 +27,9 @@ app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpContro app.post "/doc/:doc_id/pack", HttpController.packDoc +app.get '/project/:project_id/archive', HttpController.archiveProject +app.get '/project/:project_id/unarchive', HttpController.unArchiveProject + packWorker = null # use a single packing worker app.post "/pack", (req, res, next) -> diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee new file mode 100644 index 0000000000..1ae3b22363 --- /dev/null +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -0,0 +1,74 @@ +MongoManager = require "./MongoManager" +MongoAWS = require "./MongoAWS" +LockManager = require "./LockManager" +DocstoreHandler = require "./DocstoreHandler" +logger = require "logger-sharelatex" +_ = require "underscore" +async = require "async" +settings = require("settings-sharelatex") + +# increase lock timeouts because archiving can be slow +LockManager.LOCK_TEST_INTERVAL = 500 # 500ms between each test of the lock +LockManager.MAX_LOCK_WAIT_TIME = 30000 # 30s maximum time to spend trying to get the lock +LockManager.LOCK_TTL = 30 # seconds + +module.exports = DocArchiveManager = + + archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + DocstoreHandler.getAllDocs project_id, (error, docs) -> + if error? + return callback(error) + else if !docs? + return callback new Error("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> DocArchiveManager.archiveDocChangesWithLock project_id, doc._id, cb + async.series jobs, callback + + archiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> + job = (releaseLock) -> + DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock + LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) + + archiveDocChanges: (project_id, doc_id, callback)-> + MongoManager.getDocChangesCount doc_id, (error, count) -> + return callback(error) if error? + if count == 0 + logger.log {project_id, doc_id}, "document history is empty, not archiving" + return callback() + else + MongoManager.getLastCompressedUpdate doc_id, (error, update) -> + return callback(error) if error? + MongoAWS.archiveDocHistory project_id, doc_id, (error) -> + return callback(error) if error? + logger.log doc_id:doc_id, project_id:project_id, "exported document to S3" + MongoManager.markDocHistoryAsArchived doc_id, update, (error) -> + return callback(error) if error? + callback() + + unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + DocstoreHandler.getAllDocs project_id, (error, docs) -> + if error? + return callback(error) + else if !docs? + return callback new Error("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> DocArchiveManager.unArchiveDocChangesWithLock project_id, doc._id, cb + async.parallelLimit jobs, 4, callback + + unArchiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> + job = (releaseLock) -> + DocArchiveManager.unArchiveDocChanges project_id, doc_id, releaseLock + LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) + + unArchiveDocChanges: (project_id, doc_id, callback)-> + MongoManager.getArchivedDocChanges doc_id, (error, count) -> + return callback(error) if error? + if count == 0 + return callback() + else + MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> + return callback(error) if error? + logger.log doc_id:doc_id, project_id:project_id, "imported document from S3" + MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> + return callback(error) if error? + callback() diff --git a/services/track-changes/app/coffee/DocstoreHandler.coffee b/services/track-changes/app/coffee/DocstoreHandler.coffee new file mode 100644 index 0000000000..9e3b10d258 --- /dev/null +++ b/services/track-changes/app/coffee/DocstoreHandler.coffee @@ -0,0 +1,21 @@ +request = require("request").defaults(jar: false) +logger = require "logger-sharelatex" +settings = require "settings-sharelatex" + +module.exports = DocstoreHandler = + + getAllDocs: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "getting all docs for project in docstore api" + url = "#{settings.apis.docstore.url}/project/#{project_id}/doc" + request.get { + url: url + json: true + }, (error, res, docs) -> + return callback(error) if error? + logger.log {error, res, docs: if docs?.length then docs.map (d) -> d._id else []}, "docstore response" + if 200 <= res.statusCode < 300 + callback(null, docs) + else + error = new Error("docstore api responded with non-success code: #{res.statusCode}") + logger.error err: error, project_id: project_id, "error getting all docs from docstore" + callback(error) diff --git a/services/track-changes/app/coffee/HttpController.coffee b/services/track-changes/app/coffee/HttpController.coffee index d802f193bc..298311c180 100644 --- a/services/track-changes/app/coffee/HttpController.coffee +++ b/services/track-changes/app/coffee/HttpController.coffee @@ -3,6 +3,7 @@ DiffManager = require "./DiffManager" PackManager = require "./PackManager" RestoreManager = require "./RestoreManager" logger = require "logger-sharelatex" +DocArchiveManager = require "./DocArchiveManager" module.exports = HttpController = flushDoc: (req, res, next = (error) ->) -> @@ -66,3 +67,17 @@ module.exports = HttpController = RestoreManager.restoreToBeforeVersion project_id, doc_id, version, user_id, (error) -> return next(error) if error? res.send 204 + + archiveProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "archiving all track changes to s3" + DocArchiveManager.archiveAllDocsChanges project_id, (error) -> + return next(error) if error? + res.send 204 + + unArchiveProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "unarchiving all track changes from s3" + DocArchiveManager.unArchiveAllDocsChanges project_id, (error) -> + return next(error) if error? + res.send 204 \ No newline at end of file diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee new file mode 100644 index 0000000000..f2ad22bdf4 --- /dev/null +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -0,0 +1,105 @@ +settings = require "settings-sharelatex" +logger = require "logger-sharelatex" +AWS = require 'aws-sdk' +S3S = require 's3-streams' +{db, ObjectId} = require "./mongojs" +JSONStream = require "JSONStream" +ReadlineStream = require "readline-stream" + +module.exports = MongoAWS = + + MAX_SIZE: 1024*1024 # almost max size + MAX_COUNT: 1024 # almost max count + + archiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> + + callback = (args...) -> + _callback(args...) + _callback = () -> + + query = { + doc_id: ObjectId(doc_id) + expiresAt: {$exists : false} + } + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + upload = S3S.WriteStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + db.docHistory.find(query) + .on 'error', (err) -> + callback(err) + .pipe JSONStream.stringify() + .pipe upload + .on 'error', (err) -> + callback(err) + .on 'finish', () -> + return callback(null) + + unArchiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> + + callback = (args...) -> + _callback(args...) + _callback = () -> + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + download = S3S.ReadStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + }, { + encoding: "utf8" + } + + lineStream = new ReadlineStream(); + ops = [] + sz = 0 + + download + .on 'open', (obj) -> + return 1 + .on 'error', (err) -> + callback(err) + .pipe lineStream + .on 'data', (line) -> + if line.length > 2 + ops.push(JSON.parse(line)) + sz += line.length + if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE + download.pause() + MongoAWS.handleBulk ops.slice(0), sz, () -> + download.resume() + ops.splice(0,ops.length) + sz = 0 + .on 'end', () -> + MongoAWS.handleBulk ops, sz, callback + .on 'error', (err) -> + return callback(err) + + handleBulk: (ops, size, cb) -> + bulk = db.docHistory.initializeUnorderedBulkOp(); + + for op in ops + op._id = ObjectId(op._id) + op.doc_id = ObjectId(op.doc_id) + op.project_id = ObjectId(op.project_id) + bulk.find({_id:op._id}).upsert().updateOne(op) + + if ops.length > 0 + bulk.execute (err, result) -> + if err? + logger.error err:err, "error bulking ReadlineStream" + else + logger.log count:ops.length, result:result, size: size, "bulked ReadlineStream" + cb(err) + else + cb() diff --git a/services/track-changes/app/coffee/MongoAWSexternal.coffee b/services/track-changes/app/coffee/MongoAWSexternal.coffee new file mode 100644 index 0000000000..f422a583b5 --- /dev/null +++ b/services/track-changes/app/coffee/MongoAWSexternal.coffee @@ -0,0 +1,123 @@ +settings = require "settings-sharelatex" +child_process = require "child_process" +mongoUri = require "mongo-uri"; +logger = require "logger-sharelatex" +AWS = require 'aws-sdk' +fs = require 'fs' +S3S = require 's3-streams' + +module.exports = MongoAWSexternal = + + archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> + MongoAWS.s3upStream project_id, doc_id, filepath, callback + #delete temp file? + + + unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> + if error == null + MongoAWS.mongoImportDocHistory filepath, callback + #delete temp file? + else + callback + + mongoExportDocHistory: (doc_id, callback = (error, filepath) ->) -> + uriData = mongoUri.parse(settings.mongo.url); + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonUp' + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '-q' + args.push "{doc_id: ObjectId('#{doc_id}') , expiresAt: {$exists : false} }" + args.push '-o' + args.push filepath + + proc = child_process.spawn "mongoexport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + + mongoImportDocHistory: (filepath, callback = (error) ->) -> + + uriData = mongoUri.parse(settings.mongo.url); + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '--file' + args.push filepath + + proc = child_process.spawn "mongoimport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + + s3upStream: (project_id, doc_id, filepath, callback = (error) ->) -> + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + upload = S3S.WriteStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + fs.createReadStream(filepath) + .on 'open', (obj) -> + return 1 + .pipe(upload) + .on 'finish', () -> + return callback(null) + .on 'error', (err) -> + return callback(err) + + s3downStream: (project_id, doc_id, callback = (error, filepath) ->) -> + + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + download = S3S.ReadStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + download + .on 'open', (obj) -> + return 1 + .pipe(fs.createWriteStream(filepath)) + .on 'finish', () -> + return callback(null, filepath) + .on 'error', (err) -> + return callback(err, null) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index b322a5d878..3bcf50af2a 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -1,6 +1,7 @@ {db, ObjectId} = require "./mongojs" PackManager = require "./PackManager" async = require "async" +logger = require "logger-sharelatex" module.exports = MongoManager = getLastCompressedUpdate: (doc_id, callback = (error, update) ->) -> @@ -47,6 +48,7 @@ module.exports = MongoManager = insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) -> + inS3 = update.inS3? update = { doc_id: ObjectId(doc_id.toString()) project_id: ObjectId(project_id.toString()) @@ -54,6 +56,9 @@ module.exports = MongoManager = meta: update.meta v: update.v } + if inS3 + update.inS3 = true + if temporary seconds = 1000 minutes = 60 * seconds @@ -126,3 +131,20 @@ module.exports = MongoManager = # For finding documents which need packing db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true } db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true } + + getDocChangesCount: (doc_id, callback)-> + db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, {}, callback + + getArchivedDocChanges: (doc_id, callback)-> + db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback + + markDocHistoryAsArchived: (doc_id, update, callback)-> + db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> + return callback(error) if error? + db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)-> + return callback(error) if error? + callback(error) + + markDocHistoryAsUnarchived: (doc_id, callback)-> + db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)-> + callback(error) diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 6d19db7fbd..ffec3d1efe 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -6,6 +6,8 @@ WebApiManager = require "./WebApiManager" UpdateTrimmer = require "./UpdateTrimmer" logger = require "logger-sharelatex" async = require "async" +DocArchiveManager = require "./DocArchiveManager" +_ = require "underscore" module.exports = UpdatesManager = compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) -> @@ -32,6 +34,9 @@ module.exports = UpdatesManager = return compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates + if lastCompressedUpdate?.inS3? and not _.some(compressedUpdates, (update) -> update.inS3) + compressedUpdates[compressedUpdates.length-1].inS3 = lastCompressedUpdate.inS3 + MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) -> return callback(error) if error? logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" @@ -94,7 +99,17 @@ module.exports = UpdatesManager = getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.processUncompressedUpdatesForProject project_id, (error) -> return callback(error) if error? - MongoManager.getProjectUpdates project_id, options, callback + MongoManager.getProjectUpdates project_id, options, (error, updates) -> + jobs = [] + for update in updates + if update.inS3? + do (update) -> + jobs.push (callback) -> DocArchiveManager.unArchiveDocChanges update.project_id, update.doc_id, callback + if jobs.length? + async.series jobs, (err) -> + MongoManager.getProjectUpdates project_id, options, callback + else + callback(error, updates) getProjectUpdatesWithUserInfo: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.getProjectUpdates project_id, options, (error, updates) -> diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index 0a3e8f18a5..59fb94ea6a 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -1,3 +1,6 @@ +Path = require('path') +TMP_DIR = Path.resolve(Path.join(__dirname, "../../", "tmp")) + module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' @@ -19,3 +22,14 @@ module.exports = host: "localhost" port: 6379 pass: "" + + filestore: + backend: "s3" + stores: + user_files: "" + s3: + key: "" + secret: "" + + path: + dumpFolder: Path.join(TMP_DIR, "dumpFolder") diff --git a/services/track-changes/package.json b/services/track-changes/package.json index 89b2bd8496..4fe112262a 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -18,7 +18,11 @@ "request": "~2.33.0", "redis-sharelatex": "~0.0.4", "redis": "~0.10.1", - "underscore": "~1.7.0" + "underscore": "~1.7.0", + "mongo-uri": "^0.1.2", + "s3-streams": "^0.3.0", + "JSONStream": "^1.0.4", + "readline-stream": "^1.0.1" }, "devDependencies": { "chai": "~1.9.0", diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee new file mode 100644 index 0000000000..416b2de9d4 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -0,0 +1,113 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +expect = chai.expect +mongojs = require "../../../app/js/mongojs" +db = mongojs.db +ObjectId = mongojs.ObjectId +Settings = require "settings-sharelatex" +request = require "request" +rclient = require("redis").createClient() # Only works locally for now + +TrackChangesClient = require "./helpers/TrackChangesClient" +MockDocStoreApi = require "./helpers/MockDocStoreApi" +MockWebApi = require "./helpers/MockWebApi" + +describe "Archiving updates", -> + before (done) -> + @now = Date.now() + @to = @now + @user_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @project_id = ObjectId().toString() + + @minutes = 60 * 1000 + @hours = 60 * @minutes + + MockWebApi.projects[@project_id] = + features: + versioning: true + sinon.spy MockWebApi, "getProjectDetails" + + MockWebApi.users[@user_id] = @user = + email: "user@sharelatex.com" + first_name: "Leo" + last_name: "Lion" + id: @user_id + sinon.spy MockWebApi, "getUserInfo" + + MockDocStoreApi.docs[@doc_id] = @doc = + _id: @doc_id + project_id: @project_id + sinon.spy MockDocStoreApi, "getAllDoc" + + @updates = [] + for i in [0..9] + @updates.push { + op: [{ i: "a", p: 0 }] + meta: { ts: @now - (9 - i) * @hours - 2 * @minutes, user_id: @user_id } + v: 2 * i + 1 + } + @updates.push { + op: [{ i: "b", p: 0 }] + meta: { ts: @now - (9 - i) * @hours, user_id: @user_id } + v: 2 * i + 2 + } + + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => + throw error if error? + TrackChangesClient.flushDoc @project_id, @doc_id, (error) -> + throw error if error? + done() + + after (done) -> + MockWebApi.getUserInfo.restore() + db.docHistory.remove {project_id: ObjectId(@project_id)} + TrackChangesClient.removeS3Doc @project_id, @doc_id, done + + describe "archiving a doc's updates", -> + before (done) -> + TrackChangesClient.archiveProject @project_id, (error) -> + throw error if error? + done() + + it "should remain one doc change", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + throw error if error? + count.should.equal 1 + done() + + it "should remained doc marked as inS3", (done) -> + db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + throw error if error? + doc.inS3.should.equal true + done() + + it "should remained doc have last version", (done) -> + db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + throw error if error? + doc.v.should.equal 20 + done() + + it "should store twenty doc changes in S3", (done) -> + TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) => + doc.length.should.equal 20 + done() + + describe "unarchiving a doc's updates", -> + before (done) -> + TrackChangesClient.unarchiveProject @project_id, (error) -> + throw error if error? + done() + + it "should restore doc changes", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + throw error if error? + count.should.equal 20 + done() + + it "should remove doc marked as inS3", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) -> + throw error if error? + count.should.equal 0 + done() diff --git a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee index 1285177bf9..f64f42ff87 100644 --- a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee @@ -28,7 +28,7 @@ describe "Getting a diff", -> first_name: "Leo" last_name: "Lion" id: @user_id - sinon.spy MockWebApi, "getUser" + sinon.spy MockWebApi, "getUserInfo" twoMinutes = 2 * 60 * 1000 @@ -68,7 +68,7 @@ describe "Getting a diff", -> after () -> MockDocUpdaterApi.getDoc.restore() - MockWebApi.getUser.restore() + MockWebApi.getUserInfo.restore() it "should return the diff", -> expect(@diff).to.deep.equal @expected_diff diff --git a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee index 7c94bcdc34..d01797e25f 100644 --- a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee @@ -31,7 +31,7 @@ describe "Getting updates", -> first_name: "Leo" last_name: "Lion" id: @user_id - sinon.spy MockWebApi, "getUser" + sinon.spy MockWebApi, "getUserInfo" @updates = [] for i in [0..9] @@ -52,7 +52,7 @@ describe "Getting updates", -> done() after: () -> - MockWebApi.getUser.restore() + MockWebApi.getUserInfo.restore() describe "getting updates up to the limit", -> before (done) -> @@ -62,7 +62,7 @@ describe "Getting updates", -> done() it "should fetch the user details from the web api", -> - MockWebApi.getUser + MockWebApi.getUserInfo .calledWith(@user_id) .should.equal true diff --git a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee index ebe9a7795e..5dbe094cd8 100644 --- a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee +++ b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee @@ -20,7 +20,7 @@ describe "Restoring a version", -> @doc_id = ObjectId().toString() @project_id = ObjectId().toString() MockWebApi.projects[@project_id] = features: versioning: true - + minutes = 60 * 1000 @updates = [{ @@ -49,6 +49,7 @@ describe "Restoring a version", -> first_name: "Leo" last_name: "Lion" id: @user_id + MockDocUpdaterApi.docs[@doc_id] = lines: @lines version: 7 diff --git a/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee b/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee new file mode 100644 index 0000000000..29864479a4 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee @@ -0,0 +1,24 @@ +express = require("express") +app = express() + +module.exports = MockDocUpdaterApi = + docs: {} + + getAllDoc: (project_id, callback = (error) ->) -> + callback null, @docs + + run: () -> + app.get "/project/:project_id/doc", (req, res, next) => + @getAllDoc req.params.project_id, (error, docs) -> + if error? + res.send 500 + if !docs? + res.send 404 + else + res.send JSON.stringify docs + + app.listen 3016, (error) -> + throw error if error? + +MockDocUpdaterApi.run() + diff --git a/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee b/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee index 002d944dfd..7beb67075e 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee @@ -6,15 +6,15 @@ module.exports = MockWebApi = projects: {} - getUser: (user_id, callback = (error) ->) -> + getUserInfo: (user_id, callback = (error) ->) -> callback null, @users[user_id] or null - getProject: (project_id, callback = (error, project) ->) -> + getProjectDetails: (project_id, callback = (error, project) ->) -> callback null, @projects[project_id] run: () -> app.get "/user/:user_id/personal_info", (req, res, next) => - @getUser req.params.user_id, (error, user) -> + @getUserInfo req.params.user_id, (error, user) -> if error? res.send 500 if !user? @@ -23,7 +23,7 @@ module.exports = MockWebApi = res.send JSON.stringify user app.get "/project/:project_id/details", (req, res, next) => - @getProject req.params.project_id, (error, project) -> + @getProjectDetails req.params.project_id, (error, project) -> if error? res.send 500 if !project? diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee index 661b0eafb8..1e409908d9 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -1,6 +1,7 @@ request = require "request" rclient = require("redis").createClient() # Only works locally for now {db, ObjectId} = require "../../../../app/js/mongojs" +Settings = require "settings-sharelatex" module.exports = TrackChangesClient = flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) -> @@ -71,4 +72,37 @@ module.exports = TrackChangesClient = "X-User-Id": user_id }, (error, response, body) => response.statusCode.should.equal 204 - callback null \ No newline at end of file + callback null + + archiveProject: (project_id, callback = (error) ->) -> + request.get { + url: "http://localhost:3015/project/#{project_id}/archive" + }, (error, response, body) => + response.statusCode.should.equal 204 + callback(error) + + unarchiveProject: (project_id, callback = (error) ->) -> + request.get { + url: "http://localhost:3015/project/#{project_id}/unarchive" + }, (error, response, body) => + response.statusCode.should.equal 204 + callback(error) + + buildS3Options: (content, key)-> + return { + aws: + key: Settings.filestore.s3.key + secret: Settings.filestore.s3.secret + bucket: Settings.filestore.stores.user_files + timeout: 30 * 1000 + json: content + uri:"https://#{Settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } + + getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> + options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id) + request.get options, callback + + removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> + options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id) + request.del options, callback \ No newline at end of file diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee new file mode 100644 index 0000000000..f09889492d --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -0,0 +1,120 @@ +chai = require('chai') +sinon = require("sinon") +should = chai.should() +modulePath = "../../../../app/js/DocArchiveManager.js" +SandboxedModule = require('sandboxed-module') +ObjectId = require("mongojs").ObjectId + +describe "DocArchiveManager", -> + beforeEach -> + @DocArchiveManager = SandboxedModule.require modulePath, requires: + "./MongoManager" : @MongoManager = sinon.stub() + "./MongoAWS" : @MongoAWS = sinon.stub() + "./LockManager" : @LockManager = sinon.stub() + "./DocstoreHandler" : @DocstoreHandler = sinon.stub() + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + "settings-sharelatex": @settings = + filestore: + backend: 's3' + + @mongoDocs = [{ + _id: ObjectId() + }, { + _id: ObjectId() + }, { + _id: ObjectId() + }] + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + + describe "archiveAllDocsChanges", -> + it "should archive all project docs change", (done)-> + @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) + @DocArchiveManager.archiveDocChangesWithLock = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.archiveAllDocsChanges @project_id, (err)=> + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true + should.not.exist err + done() + + describe "archiveDocChangesWithLock", -> + beforeEach -> + @DocArchiveManager.archiveDocChanges = sinon.stub().callsArg(2) + @LockManager.runWithLock = sinon.stub().callsArg(2) + @DocArchiveManager.archiveDocChangesWithLock @project_id, @doc_id, @callback + + it "should run archiveDocChangesWithLock with the lock", -> + @LockManager.runWithLock + .calledWith( + "HistoryLock:#{@doc_id}" + ) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "archiveDocChanges", -> + beforeEach -> + @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} + @MongoManager.getDocChangesCount = sinon.stub().callsArg(1) + @MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update) + @MongoAWS.archiveDocHistory = sinon.stub().callsArg(2) + @MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2) + @DocArchiveManager.archiveDocChanges @project_id, @doc_id, @callback + + it "should run markDocHistoryAsArchived with doc_id and update", -> + @MongoManager.markDocHistoryAsArchived + .calledWith( + @doc_id, @update + ) + .should.equal true + it "should call the callback", -> + @callback.called.should.equal true + + describe "unArchiveAllDocsChanges", -> + it "should unarchive all project docs change", (done)-> + @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) + @DocArchiveManager.unArchiveDocChangesWithLock = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.unArchiveAllDocsChanges @project_id, (err)=> + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true + should.not.exist err + done() + + describe "unArchiveDocChangesWithLock", -> + beforeEach -> + @DocArchiveManager.unArchiveDocChanges = sinon.stub().callsArg(2) + @LockManager.runWithLock = sinon.stub().callsArg(2) + @DocArchiveManager.unArchiveDocChangesWithLock @project_id, @doc_id, @callback + + it "should run unArchiveDocChangesWithLock with the lock", -> + @LockManager.runWithLock + .calledWith( + "HistoryLock:#{@doc_id}" + ) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "unArchiveDocChanges", -> + beforeEach -> + @MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1) + @MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2) + @MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1) + @DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback + + it "should run markDocHistoryAsUnarchived with doc_id", -> + @MongoManager.markDocHistoryAsUnarchived + .calledWith( + @doc_id + ) + .should.equal true + it "should call the callback", -> + @callback.called.should.equal true diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee new file mode 100644 index 0000000000..279f2a979b --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee @@ -0,0 +1,55 @@ +chai = require('chai') +chai.should() +sinon = require("sinon") +modulePath = "../../../../app/js/DocstoreHandler.js" +SandboxedModule = require('sandboxed-module') + +describe "DocstoreHandler", -> + beforeEach -> + @requestDefaults = sinon.stub().returns(@request = sinon.stub()) + @DocstoreHandler = SandboxedModule.require modulePath, requires: + "request" : defaults: @requestDefaults + "settings-sharelatex": @settings = + apis: + docstore: + url: "docstore.sharelatex.com" + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + + @requestDefaults.calledWith(jar: false).should.equal true + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + + describe "getAllDocs", -> + describe "with a successful response code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, statusCode: 204, @docs = [{ _id: "mock-doc-id" }]) + @DocstoreHandler.getAllDocs @project_id, @callback + + it "should get all the project docs in the docstore api", -> + @request.get + .calledWith({ + url: "#{@settings.apis.docstore.url}/project/#{@project_id}/doc" + json: true + }) + .should.equal true + + it "should call the callback with the docs", -> + @callback.calledWith(null, @docs).should.equal true + + describe "with a failed response code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, statusCode: 500, "") + @DocstoreHandler.getAllDocs @project_id, @callback + + it "should call the callback with an error", -> + @callback.calledWith(new Error("docstore api responded with non-success code: 500")).should.equal true + + it "should log the error", -> + @logger.error + .calledWith({ + err: new Error("docstore api responded with a non-success code: 500") + project_id: @project_id + }, "error getting all docs from docstore") + .should.equal true \ No newline at end of file diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee new file mode 100644 index 0000000000..0bad498eb5 --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -0,0 +1,113 @@ +chai = require('chai') +chai.should() +sinon = require("sinon") +modulePath = "../../../../app/js/MongoAWS.js" +SandboxedModule = require('sandboxed-module') +{ObjectId} = require("mongojs") + +describe "MongoAWS", -> + beforeEach -> + @MongoAWS = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings = + filestore: + s3: + secret: "s3-secret" + key: "s3-key" + stores: + user_files: "s3-bucket" + "child_process": @child_process = {} + "mongo-uri": @mongouri = {} + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + "aws-sdk": @awssdk = {} + "fs": @fs = {} + "s3-streams": @s3streams = {} + "./mongojs" : { db: @db = {}, ObjectId: ObjectId } + "JSONStream": @JSONStream = {} + "readline-stream": @readline = sinon.stub() + + @project_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @callback = sinon.stub() + + describe "archiveDocHistory", -> + + beforeEach -> + @awssdk.config = { update: sinon.stub() } + @awssdk.S3 = sinon.stub() + @s3streams.WriteStream = sinon.stub() + @db.docHistory = {} + @db.docHistory.on = sinon.stub() + @db.docHistory.find = sinon.stub().returns @db.docHistory + @db.docHistory.on.returns + pipe:-> + pipe:-> + on: (type, cb)-> + on: (type, cb)-> + cb() + @JSONStream.stringify = sinon.stub() + + @MongoAWS.archiveDocHistory @project_id, @doc_id, @callback + + it "should call the callback", -> + @callback.called.should.equal true + + describe "unArchiveDocHistory", -> + + beforeEach -> + @awssdk.config = { update: sinon.stub() } + @awssdk.S3 = sinon.stub() + @s3streams.ReadStream = sinon.stub() + + @s3streams.ReadStream.returns + #describe on 'open' behavior + on: (type, cb)-> + #describe on 'error' behavior + on: (type, cb)-> + pipe:-> + #describe on 'data' behavior + on: (type, cb)-> + cb([]) + #describe on 'end' behavior + on: (type, cb)-> + cb() + #describe on 'error' behavior + on: sinon.stub() + + @MongoAWS.handleBulk = sinon.stub() + @MongoAWS.unArchiveDocHistory @project_id, @doc_id, @callback + + it "should call handleBulk", -> + @MongoAWS.handleBulk.called.should.equal true + + describe "handleBulk", -> + beforeEach -> + @bulkOps = [{ + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }, { + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }, { + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }] + @bulk = + find: sinon.stub().returns + upsert: sinon.stub().returns + updateOne: sinon.stub() + execute: sinon.stub().callsArgWith(0, null, {}) + @db.docHistory = {} + @db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk + @MongoAWS.handleBulk @bulkOps, @bulkOps.length, @callback + + it "should call updateOne for each operation", -> + @bulk.find.calledWith({_id:@bulkOps[0]._id}).should.equal true + @bulk.find.calledWith({_id:@bulkOps[1]._id}).should.equal true + @bulk.find.calledWith({_id:@bulkOps[2]._id}).should.equal true + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + diff --git a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee index 7c559c0937..e9533664e3 100644 --- a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee +++ b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee @@ -13,6 +13,7 @@ describe "HttpController", -> "./DiffManager": @DiffManager = {} "./RestoreManager": @RestoreManager = {} "./PackManager": @PackManager = {} + "./DocArchiveManager": @DocArchiveManager = {} @doc_id = "doc-id-123" @project_id = "project-id-123" @next = sinon.stub() @@ -130,3 +131,39 @@ describe "HttpController", -> it "should return a success code", -> @res.send.calledWith(204).should.equal true + + describe "archiveProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + @res = + send: sinon.stub() + @DocArchiveManager.archiveAllDocsChanges = sinon.stub().callsArg(1) + @HttpController.archiveProject @req, @res, @next + + it "should process archive doc changes", -> + @DocArchiveManager.archiveAllDocsChanges + .calledWith(@project_id) + .should.equal true + + it "should return a success code", -> + @res.send.calledWith(204).should.equal true + + describe "unArchiveProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + @res = + send: sinon.stub() + @DocArchiveManager.unArchiveAllDocsChanges = sinon.stub().callsArg(1) + @HttpController.unArchiveProject @req, @res, @next + + it "should process unarchive doc changes", -> + @DocArchiveManager.unArchiveAllDocsChanges + .calledWith(@project_id) + .should.equal true + + it "should return a success code", -> + @res.send.calledWith(204).should.equal true diff --git a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee index 24588de6ea..8db054b40f 100644 --- a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee @@ -398,3 +398,89 @@ describe "MongoManager", -> it "should call the callback", -> @callback.called.should.equal true + + describe "getDocChangesCount", -> + beforeEach -> + @db.docHistory = + count: sinon.stub().callsArg(2) + @MongoManager.getDocChangesCount @doc_id, @callback + + it "should return if there is any doc changes", -> + @db.docHistory.count + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : { $exists : false } + }, { + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "getArchivedDocChanges", -> + beforeEach -> + @db.docHistory = + count: sinon.stub().callsArg(2) + @MongoManager.getArchivedDocChanges @doc_id, @callback + + it "should return if there is any archived doc changes", -> + @db.docHistory.count + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : true + }, { + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "markDocHistoryAsArchived", -> + beforeEach -> + @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} + @db.docHistory = + update: sinon.stub().callsArg(2) + remove: sinon.stub().callsArg(1) + @MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback + + it "should update last doc change with inS3 flag", -> + @db.docHistory.update + .calledWith({ + _id: ObjectId(@update._id) + },{ + $set : { inS3 : true } + }) + .should.equal true + + it "should remove any other doc changes before last update", -> + @db.docHistory.remove + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : { $exists : false } + v: { $lt : @update.v } + expiresAt: {$exists : false} + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "markDocHistoryAsUnarchived", -> + beforeEach -> + @db.docHistory = + update: sinon.stub().callsArg(3) + @MongoManager.markDocHistoryAsUnarchived @doc_id, @callback + + it "should remove any doc changes inS3 flag", -> + @db.docHistory.update + .calledWith({ + doc_id: ObjectId(@doc_id) + },{ + $unset : { inS3 : true } + },{ + multi: true + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true \ No newline at end of file