From ae047ecf76c36fc44023d9c1ab565038ffe0f44d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 6 Aug 2015 10:00:09 -0300 Subject: [PATCH 01/39] init s3 feature --- .../app/coffee/DocArquiveManager.coffee | 24 +++++++++++++++++++ .../config/settings.defaults.coffee | 8 +++++++ 2 files changed, 32 insertions(+) create mode 100644 services/track-changes/app/coffee/DocArquiveManager.coffee diff --git a/services/track-changes/app/coffee/DocArquiveManager.coffee b/services/track-changes/app/coffee/DocArquiveManager.coffee new file mode 100644 index 0000000000..f088a5d57e --- /dev/null +++ b/services/track-changes/app/coffee/DocArquiveManager.coffee @@ -0,0 +1,24 @@ +MongoManager = require "./MongoManager" +Errors = require "./Errors" +logger = require "logger-sharelatex" +_ = require "underscore" +async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 + +module.exports = DocArchiveManager = + + buildS3Options: (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(JSON.stringify(content)).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } \ No newline at end of file diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index 0a3e8f18a5..24e9a7e976 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -19,3 +19,11 @@ module.exports = host: "localhost" port: 6379 pass: "" + + #filestore: + # backend: "s3" + # stores: + # user_files: "sharelatex-dev" + # s3: + # key: "" + # secret: "" From 028fe2fa030b3f5cb28e31dc454e491162583063 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 6 Aug 2015 11:11:43 -0300 Subject: [PATCH 02/39] archive docChanges list to s3 --- services/track-changes/app.coffee | 3 ++ .../app/coffee/DocArchiveManager.coffee | 49 +++++++++++++++++++ .../app/coffee/DocArquiveManager.coffee | 24 --------- .../app/coffee/HttpController.coffee | 8 +++ .../app/coffee/MongoManager.coffee | 6 +++ .../track-changes/app/coffee/mongojs.coffee | 2 +- 6 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 services/track-changes/app/coffee/DocArchiveManager.coffee delete mode 100644 services/track-changes/app/coffee/DocArquiveManager.coffee diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index 8d5fd9c082..63ed85df86 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -27,6 +27,9 @@ app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpContro app.post "/doc/:doc_id/pack", HttpController.packDoc +if Settings.filestore?.backend == "s3" + app.get '/project/:project_id/archive', HttpController.archiveProject + packWorker = null # use a single packing worker app.post "/pack", (req, res, next) -> diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee new file mode 100644 index 0000000000..1d8af8b8c0 --- /dev/null +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -0,0 +1,49 @@ +MongoManager = require "./MongoManager" +logger = require "logger-sharelatex" +_ = require "underscore" +async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 + +module.exports = DocArchiveManager = + + archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if error? + return callback(error) + else if !docs? + return callback new Error("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> DocArchiveManager.archiveDocChanges project_id, doc._id, cb + async.series jobs, callback + + + archiveDocChanges: (project_id, doc_id, callback)-> + MongoManager.getDocChanges doc_id, (error, docChanges) -> + logger.log project_id: project_id, doc_id: doc_id, "sending doc changes to s3" + options = DocArchiveManager.buildS3Options(docChanges, project_id+"/changes-"+doc_id) + request.put options, (err, res)-> + md5lines = crypto.createHash("md5").update(JSON.stringify(docChanges)).digest("hex") + md5response = res.headers.etag.toString().replace(/\"/g, '') + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving doc changes in aws" + return callback new Error("Error in S3 request") + if md5lines != md5response + logger.err responseMD5:md5response, linesMD5:md5lines, "error in response md5 from s3" + return callback new Error("Error in S3 md5 response") + #MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> + # return callback(error) if error? + callback() + + buildS3Options: (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } \ No newline at end of file diff --git a/services/track-changes/app/coffee/DocArquiveManager.coffee b/services/track-changes/app/coffee/DocArquiveManager.coffee deleted file mode 100644 index f088a5d57e..0000000000 --- a/services/track-changes/app/coffee/DocArquiveManager.coffee +++ /dev/null @@ -1,24 +0,0 @@ -MongoManager = require "./MongoManager" -Errors = require "./Errors" -logger = require "logger-sharelatex" -_ = require "underscore" -async = require "async" -settings = require("settings-sharelatex") -request = require("request") -crypto = require("crypto") -thirtySeconds = 30 * 1000 - -module.exports = DocArchiveManager = - - buildS3Options: (content, key)-> - return { - aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: settings.filestore.stores.user_files - timeout: thirtySeconds - json: content - #headers: - # 'content-md5': crypto.createHash("md5").update(JSON.stringify(content)).digest("hex") - uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" - } \ No newline at end of file diff --git a/services/track-changes/app/coffee/HttpController.coffee b/services/track-changes/app/coffee/HttpController.coffee index d802f193bc..901d96e5de 100644 --- a/services/track-changes/app/coffee/HttpController.coffee +++ b/services/track-changes/app/coffee/HttpController.coffee @@ -3,6 +3,7 @@ DiffManager = require "./DiffManager" PackManager = require "./PackManager" RestoreManager = require "./RestoreManager" logger = require "logger-sharelatex" +DocArchiveManager = require "./DocArchiveManager" module.exports = HttpController = flushDoc: (req, res, next = (error) ->) -> @@ -66,3 +67,10 @@ module.exports = HttpController = RestoreManager.restoreToBeforeVersion project_id, doc_id, version, user_id, (error) -> return next(error) if error? res.send 204 + + archiveProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "archiving all track changes" + DocArchiveManager.archiveAllDocsChanges project_id, (error) -> + return next(error) if error? + res.send 204 \ No newline at end of file diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index b322a5d878..4465ba215c 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -126,3 +126,9 @@ module.exports = MongoManager = # For finding documents which need packing db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true } db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true } + + getProjectsDocs: (project_id, callback)-> + db.docs.find project_id: ObjectId(project_id.toString()), {}, callback + + getDocChanges: (doc_id, callback)-> + db.docHistory.find doc_id: ObjectId(doc_id.toString()), {}, callback diff --git a/services/track-changes/app/coffee/mongojs.coffee b/services/track-changes/app/coffee/mongojs.coffee index 32efbc9a1d..5c1e9ea252 100644 --- a/services/track-changes/app/coffee/mongojs.coffee +++ b/services/track-changes/app/coffee/mongojs.coffee @@ -1,6 +1,6 @@ Settings = require "settings-sharelatex" mongojs = require "mongojs" -db = mongojs.connect(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryStats"]) +db = mongojs.connect(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryStats", "docs"]) module.exports = db: db ObjectId: mongojs.ObjectId From 438c4f4d0c114b47e8e3b14acb8fd2ce0ab486a9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 6 Aug 2015 15:46:44 -0300 Subject: [PATCH 03/39] using mongoexport for s3 archive --- .../app/coffee/DocArchiveManager.coffee | 30 ++------ .../track-changes/app/coffee/MongoAWS.coffee | 69 +++++++++++++++++++ .../config/settings.defaults.coffee | 8 ++- services/track-changes/package.json | 4 +- 4 files changed, 83 insertions(+), 28 deletions(-) create mode 100644 services/track-changes/app/coffee/MongoAWS.coffee diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 1d8af8b8c0..3a308d03d8 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -1,4 +1,5 @@ MongoManager = require "./MongoManager" +MongoAWS = require "./MongoAWS" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" @@ -21,29 +22,6 @@ module.exports = DocArchiveManager = archiveDocChanges: (project_id, doc_id, callback)-> - MongoManager.getDocChanges doc_id, (error, docChanges) -> - logger.log project_id: project_id, doc_id: doc_id, "sending doc changes to s3" - options = DocArchiveManager.buildS3Options(docChanges, project_id+"/changes-"+doc_id) - request.put options, (err, res)-> - md5lines = crypto.createHash("md5").update(JSON.stringify(docChanges)).digest("hex") - md5response = res.headers.etag.toString().replace(/\"/g, '') - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving doc changes in aws" - return callback new Error("Error in S3 request") - if md5lines != md5response - logger.err responseMD5:md5response, linesMD5:md5lines, "error in response md5 from s3" - return callback new Error("Error in S3 md5 response") - #MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> - # return callback(error) if error? - callback() - - buildS3Options: (content, key)-> - return { - aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: settings.filestore.stores.user_files - timeout: thirtySeconds - json: content - uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" - } \ No newline at end of file + MongoAWS.archiveDocHistory project_id, doc_id, (error) -> + logger.log doc_id:doc_id, error: error, "mongoexport" + callback() \ No newline at end of file diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee new file mode 100644 index 0000000000..775a4a9a22 --- /dev/null +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -0,0 +1,69 @@ +settings = require "settings-sharelatex" +child_process = require "child_process" +mongoUri = require "mongo-uri"; +logger = require "logger-sharelatex" +AWS = require 'aws-sdk' +fs = require 'fs' + + +module.exports = MongoAWS = + + archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.mongoDumpDocHistory doc_id, (error,filepath) -> + MongoAWS.s3upload project_id, doc_id, filepath, callback + + mongoDumpDocHistory: (doc_id, callback = (error, filepath) ->) -> + uriData = mongoUri.parse(settings.mongo.url); + filepath = settings.path.dumpFolder + '/' + doc_id + '.json' + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '-q' + args.push "{doc_id: ObjectId('#{doc_id}') }" + args.push '-o' + args.push filepath + + proc = child_process.spawn "mongoexport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + + s3upload: (project_id, doc_id, filepath, callback = (error) ->) -> + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + s3Stream = require('s3-upload-stream')(new AWS.S3()); + + upload = s3Stream.upload { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + read = fs.createReadStream filepath + + #Handle errors. + upload.on 'error', callback + + #Handle upload completion. + upload.on 'uploaded', (details) -> + return callback(null) + + #Pipe the incoming filestream and up to S3. + read.pipe(upload); + diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index 24e9a7e976..84c048dced 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -1,3 +1,6 @@ +Path = require('path') +TMP_DIR = Path.resolve(Path.join(__dirname, "../../", "tmp")) + module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' @@ -23,7 +26,10 @@ module.exports = #filestore: # backend: "s3" # stores: - # user_files: "sharelatex-dev" + # user_files: "" # s3: # key: "" # secret: "" + + path: + dumpFolder: Path.join(TMP_DIR, "dumpFolder") \ No newline at end of file diff --git a/services/track-changes/package.json b/services/track-changes/package.json index 89b2bd8496..79d2fa1254 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -18,7 +18,9 @@ "request": "~2.33.0", "redis-sharelatex": "~0.0.4", "redis": "~0.10.1", - "underscore": "~1.7.0" + "underscore": "~1.7.0", + "mongo-uri": "^0.1.2", + "s3-upload-stream": "^1.0.7" }, "devDependencies": { "chai": "~1.9.0", From bca48ac117923fae320b3cf55149629394abbc95 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 6 Aug 2015 17:09:36 -0300 Subject: [PATCH 04/39] add unarchive doc track from s3 --- services/track-changes/app.coffee | 1 + .../app/coffee/DocArchiveManager.coffee | 25 +++++++- .../app/coffee/HttpController.coffee | 9 ++- .../track-changes/app/coffee/MongoAWS.coffee | 63 +++++++++++++++++-- .../app/coffee/MongoManager.coffee | 3 + .../config/settings.defaults.coffee | 16 ++--- 6 files changed, 101 insertions(+), 16 deletions(-) diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index 63ed85df86..f7af275d62 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -29,6 +29,7 @@ app.post "/doc/:doc_id/pack", HttpController.packDoc if Settings.filestore?.backend == "s3" app.get '/project/:project_id/archive', HttpController.archiveProject + app.get '/project/:project_id/unarchive', HttpController.unArchiveProject packWorker = null # use a single packing worker diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 3a308d03d8..d7b71a8901 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -22,6 +22,25 @@ module.exports = DocArchiveManager = archiveDocChanges: (project_id, doc_id, callback)-> - MongoAWS.archiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "mongoexport" - callback() \ No newline at end of file + MongoManager.getDocChangesCount doc_id, (error, count) -> + if count == 0 + callback() + else + MongoAWS.archiveDocHistory project_id, doc_id, (error) -> + logger.log doc_id:doc_id, error: error, "mongoexport" + callback() + + unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if error? + return callback(error) + else if !docs? + return callback new Error("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> DocArchiveManager.unArchiveDocChanges project_id, doc._id, cb + async.series jobs, callback + + unArchiveDocChanges: (project_id, doc_id, callback)-> + MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> + logger.log doc_id:doc_id, error: error, "mongoimport" + callback() diff --git a/services/track-changes/app/coffee/HttpController.coffee b/services/track-changes/app/coffee/HttpController.coffee index 901d96e5de..298311c180 100644 --- a/services/track-changes/app/coffee/HttpController.coffee +++ b/services/track-changes/app/coffee/HttpController.coffee @@ -70,7 +70,14 @@ module.exports = HttpController = archiveProject: (req, res, next = (error) ->) -> project_id = req.params.project_id - logger.log project_id: project_id, "archiving all track changes" + logger.log project_id: project_id, "archiving all track changes to s3" DocArchiveManager.archiveAllDocsChanges project_id, (error) -> + return next(error) if error? + res.send 204 + + unArchiveProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "unarchiving all track changes from s3" + DocArchiveManager.unArchiveAllDocsChanges project_id, (error) -> return next(error) if error? res.send 204 \ No newline at end of file diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 775a4a9a22..3092209113 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,16 +5,22 @@ logger = require "logger-sharelatex" AWS = require 'aws-sdk' fs = require 'fs' - module.exports = MongoAWS = archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> - MongoAWS.mongoDumpDocHistory doc_id, (error,filepath) -> + MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> MongoAWS.s3upload project_id, doc_id, filepath, callback - mongoDumpDocHistory: (doc_id, callback = (error, filepath) ->) -> + unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.s3download project_id, doc_id, (error, filepath) -> + if error == null + MongoAWS.mongoImportDocHistory filepath, callback + else + callback + + mongoExportDocHistory: (doc_id, callback = (error, filepath) ->) -> uriData = mongoUri.parse(settings.mongo.url); - filepath = settings.path.dumpFolder + '/' + doc_id + '.json' + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonUp' args = [] args.push '-h' @@ -41,6 +47,33 @@ module.exports = MongoAWS = else return callback(new Error("mongodump failed: #{stderr}"),null) + mongoImportDocHistory: (filepath, callback = (error) ->) -> + + uriData = mongoUri.parse(settings.mongo.url); + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '--file' + args.push filepath + + proc = child_process.spawn "mongoimport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + s3upload: (project_id, doc_id, filepath, callback = (error) ->) -> AWS.config.update { @@ -67,3 +100,25 @@ module.exports = MongoAWS = #Pipe the incoming filestream and up to S3. read.pipe(upload); + s3download: (project_id, doc_id, callback = (error, filepath) ->) -> + + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + params = { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + s3 = new AWS.S3() + + s3.getObject params, (err, data) -> + if !err && data.ContentLength > 0 + fs.writeFile filepath, data.Body, (err) -> + return callback(null,filepath) + else + return callback(new Error("s3download failed: #{err}"),null) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 4465ba215c..a7898b79c4 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -132,3 +132,6 @@ module.exports = MongoManager = getDocChanges: (doc_id, callback)-> db.docHistory.find doc_id: ObjectId(doc_id.toString()), {}, callback + + getDocChangesCount: (doc_id, callback)-> + db.docHistory.count doc_id: ObjectId(doc_id.toString()), {}, callback diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index 84c048dced..59fb94ea6a 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -23,13 +23,13 @@ module.exports = port: 6379 pass: "" - #filestore: - # backend: "s3" - # stores: - # user_files: "" - # s3: - # key: "" - # secret: "" + filestore: + backend: "s3" + stores: + user_files: "" + s3: + key: "" + secret: "" path: - dumpFolder: Path.join(TMP_DIR, "dumpFolder") \ No newline at end of file + dumpFolder: Path.join(TMP_DIR, "dumpFolder") From daa42bcea0f7278a21e5fccfbcbb6db6e6febab8 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Sun, 9 Aug 2015 15:47:47 -0300 Subject: [PATCH 05/39] change s3Stream lib --- .../app/coffee/DocArchiveManager.coffee | 14 +++-- .../track-changes/app/coffee/MongoAWS.coffee | 58 +++++++++---------- services/track-changes/package.json | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index d7b71a8901..de769efa4f 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -24,11 +24,11 @@ module.exports = DocArchiveManager = archiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getDocChangesCount doc_id, (error, count) -> if count == 0 - callback() + return callback() else MongoAWS.archiveDocHistory project_id, doc_id, (error) -> logger.log doc_id:doc_id, error: error, "mongoexport" - callback() + return callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> MongoManager.getProjectsDocs project_id, (error, docs) -> @@ -41,6 +41,10 @@ module.exports = DocArchiveManager = async.series jobs, callback unArchiveDocChanges: (project_id, doc_id, callback)-> - MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "mongoimport" - callback() + MongoManager.getDocChangesCount doc_id, (error, count) -> + if count == 0 + return callback() + else + MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> + logger.log doc_id:doc_id, error: error, "mongoimport" + return callback() diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 3092209113..9a63ddab4d 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -4,15 +4,16 @@ mongoUri = require "mongo-uri"; logger = require "logger-sharelatex" AWS = require 'aws-sdk' fs = require 'fs' +S3S = require 's3-streams' module.exports = MongoAWS = archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> - MongoAWS.s3upload project_id, doc_id, filepath, callback + MongoAWS.s3upStream project_id, doc_id, filepath, callback unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> - MongoAWS.s3download project_id, doc_id, (error, filepath) -> + MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> if error == null MongoAWS.mongoImportDocHistory filepath, callback else @@ -74,33 +75,28 @@ module.exports = MongoAWS = else return callback(new Error("mongodump failed: #{stderr}"),null) - s3upload: (project_id, doc_id, filepath, callback = (error) ->) -> + s3upStream: (project_id, doc_id, filepath, callback = (error) ->) -> AWS.config.update { accessKeyId: settings.filestore.s3.key secretAccessKey: settings.filestore.s3.secret } - - s3Stream = require('s3-upload-stream')(new AWS.S3()); - - upload = s3Stream.upload { - "Bucket": settings.filestore.stores.user_files, - "Key": project_id+"/changes-"+doc_id + + upload = S3S.WriteStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id } - read = fs.createReadStream filepath + fs.createReadStream(filepath) + .on 'open', (obj) -> + return 1 + .pipe(upload) + .on 'finish', () -> + return callback(null) + .on 'error', (err) -> + return callback(err) - #Handle errors. - upload.on 'error', callback - - #Handle upload completion. - upload.on 'uploaded', (details) -> - return callback(null) - - #Pipe the incoming filestream and up to S3. - read.pipe(upload); - - s3download: (project_id, doc_id, callback = (error, filepath) ->) -> + s3downStream: (project_id, doc_id, callback = (error, filepath) ->) -> filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' @@ -108,17 +104,17 @@ module.exports = MongoAWS = accessKeyId: settings.filestore.s3.key secretAccessKey: settings.filestore.s3.secret } - - params = { + + download = S3S.ReadStream new AWS.S3(), { "Bucket": settings.filestore.stores.user_files, "Key": project_id+"/changes-"+doc_id } - s3 = new AWS.S3() - - s3.getObject params, (err, data) -> - if !err && data.ContentLength > 0 - fs.writeFile filepath, data.Body, (err) -> - return callback(null,filepath) - else - return callback(new Error("s3download failed: #{err}"),null) + download + .on 'open', (obj) -> + return 1 + .pipe(fs.createWriteStream(filepath)) + .on 'finish', () -> + return callback(null, filepath) + .on 'error', (err) -> + return callback(err, null) diff --git a/services/track-changes/package.json b/services/track-changes/package.json index 79d2fa1254..f82b618359 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -20,7 +20,7 @@ "redis": "~0.10.1", "underscore": "~1.7.0", "mongo-uri": "^0.1.2", - "s3-upload-stream": "^1.0.7" + "s3-streams": "^0.3.0" }, "devDependencies": { "chai": "~1.9.0", From 3bc538046886d39be704c56cfcd9533f9d7e1cbe Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Sun, 9 Aug 2015 17:50:15 -0300 Subject: [PATCH 06/39] handle inS3 flag --- .../app/coffee/DocArchiveManager.coffee | 8 ++++++-- services/track-changes/app/coffee/MongoAWS.coffee | 2 ++ .../track-changes/app/coffee/MongoManager.coffee | 15 ++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index de769efa4f..6ab60f7b72 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -28,7 +28,9 @@ module.exports = DocArchiveManager = else MongoAWS.archiveDocHistory project_id, doc_id, (error) -> logger.log doc_id:doc_id, error: error, "mongoexport" - return callback() + MongoManager.markDocHistoryAsArchived doc_id, (error) -> + return callback(error) if error? + callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> MongoManager.getProjectsDocs project_id, (error, docs) -> @@ -47,4 +49,6 @@ module.exports = DocArchiveManager = else MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> logger.log doc_id:doc_id, error: error, "mongoimport" - return callback() + MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> + return callback(error) if error? + callback() diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 9a63ddab4d..43dd59591b 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -11,11 +11,13 @@ module.exports = MongoAWS = archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> MongoAWS.s3upStream project_id, doc_id, filepath, callback + #delete temp file? unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> if error == null MongoAWS.mongoImportDocHistory filepath, callback + #delete temp file? else callback diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index a7898b79c4..a4011c1af3 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -130,8 +130,17 @@ module.exports = MongoManager = getProjectsDocs: (project_id, callback)-> db.docs.find project_id: ObjectId(project_id.toString()), {}, callback - getDocChanges: (doc_id, callback)-> - db.docHistory.find doc_id: ObjectId(doc_id.toString()), {}, callback - getDocChangesCount: (doc_id, callback)-> db.docHistory.count doc_id: ObjectId(doc_id.toString()), {}, callback + + markDocHistoryAsArchived: (doc_id, callback)-> + MongoManager.getLastCompressedUpdate doc_id, (error, update) -> + db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> + return callback(error) if error? + db.docHistory.remove { doc_id : doc_id, inS3 : { $exists : false } }, (error)-> + return callback(error) if error? + callback(error) + + markDocHistoryAsUnarchived: (doc_id, callback)-> + db.docHistory.update { doc_id: doc_id }, { $unset : { inS3 : true } }, { multi: true }, (error)-> + callback(error) From 6bc9c9010a9e1171dd40002d1e1047b336c71425 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Sun, 9 Aug 2015 19:52:32 -0300 Subject: [PATCH 07/39] handle auto unarchive track changes --- .../track-changes/app/coffee/DocArchiveManager.coffee | 6 +++++- services/track-changes/app/coffee/MongoManager.coffee | 9 ++++++++- services/track-changes/app/coffee/UpdatesManager.coffee | 5 ++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 6ab60f7b72..9ac6e1163f 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -11,6 +11,8 @@ thirtySeconds = 30 * 1000 module.exports = DocArchiveManager = archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + if settings.filestore?.backend != "s3" + return callback(null) MongoManager.getProjectsDocs project_id, (error, docs) -> if error? return callback(error) @@ -33,6 +35,8 @@ module.exports = DocArchiveManager = callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> + if settings.filestore?.backend != "s3" + return callback(null) MongoManager.getProjectsDocs project_id, (error, docs) -> if error? return callback(error) @@ -43,7 +47,7 @@ module.exports = DocArchiveManager = async.series jobs, callback unArchiveDocChanges: (project_id, doc_id, callback)-> - MongoManager.getDocChangesCount doc_id, (error, count) -> + MongoManager.getArchivedDocChanges doc_id, (error, count) -> if count == 0 return callback() else diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index a4011c1af3..536b94f876 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -47,6 +47,7 @@ module.exports = MongoManager = insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) -> + inS3 = update.inS3? update = { doc_id: ObjectId(doc_id.toString()) project_id: ObjectId(project_id.toString()) @@ -54,6 +55,9 @@ module.exports = MongoManager = meta: update.meta v: update.v } + if inS3? + update.inS3 = true + if temporary seconds = 1000 minutes = 60 * seconds @@ -117,7 +121,7 @@ module.exports = MongoManager = db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, { background: true } # For finding all packs that affect a project (use a sparse index so only packs are included) db.docHistory.ensureIndex { project_id: 1, "pack.0.meta.end_ts": 1, "meta.end_ts": 1}, { background: true, sparse: true } - # For finding updates that don't yet have a project_id and need it inserting + # For finding updates that dont yet have a project_id and need it inserting db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, { background: true } # For finding project meta-data db.projectHistoryMetaData.ensureIndex { project_id: 1 }, { background: true } @@ -133,6 +137,9 @@ module.exports = MongoManager = getDocChangesCount: (doc_id, callback)-> db.docHistory.count doc_id: ObjectId(doc_id.toString()), {}, callback + getArchivedDocChanges: (doc_id, callback)-> + db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback + markDocHistoryAsArchived: (doc_id, callback)-> MongoManager.getLastCompressedUpdate doc_id, (error, update) -> db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 92357fc2dc..1fbfe53d7e 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -6,6 +6,7 @@ WebApiManager = require "./WebApiManager" UpdateTrimmer = require "./UpdateTrimmer" logger = require "logger-sharelatex" async = require "async" +DocArchiveManager = require "./DocArchiveManager" module.exports = UpdatesManager = compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) -> @@ -94,7 +95,9 @@ module.exports = UpdatesManager = getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.processUncompressedUpdatesForProject project_id, (error) -> return callback(error) if error? - MongoManager.getProjectUpdates project_id, options, callback + DocArchiveManager.unArchiveAllDocsChanges project_id, (error) -> + return callback(error,null) if error? + MongoManager.getProjectUpdates project_id, options, callback getProjectUpdatesWithUserInfo: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.getProjectUpdates project_id, options, (error, updates) -> From fd4afb357449c7b5c1a56c94105463e8c3be36dc Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 14 Aug 2015 15:07:16 -0300 Subject: [PATCH 08/39] Archive changes, care about: versioin, expiresAt and Lock --- .../app/coffee/DocArchiveManager.coffee | 21 +++++++++++++------ .../track-changes/app/coffee/MongoAWS.coffee | 2 +- .../app/coffee/MongoManager.coffee | 11 +++++----- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 9ac6e1163f..54d05cab96 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -1,5 +1,6 @@ MongoManager = require "./MongoManager" MongoAWS = require "./MongoAWS" +LockManager = require "./LockManager" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" @@ -19,20 +20,28 @@ module.exports = DocArchiveManager = else if !docs? return callback new Error("No docs for project #{project_id}") jobs = _.map docs, (doc) -> - (cb)-> DocArchiveManager.archiveDocChanges project_id, doc._id, cb + (cb)-> DocArchiveManager.archiveDocChangesWithLock project_id, doc._id, cb async.series jobs, callback + archiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> + LockManager.runWithLock( + "HistoryArchiveLock:#{doc_id}", + (releaseLock) -> + DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock + callback + ) archiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getDocChangesCount doc_id, (error, count) -> if count == 0 return callback() else - MongoAWS.archiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "mongoexport" - MongoManager.markDocHistoryAsArchived doc_id, (error) -> - return callback(error) if error? - callback() + MongoManager.getLastCompressedUpdate doc_id, (error, update) -> + MongoAWS.archiveDocHistory project_id, doc_id, (error) -> + logger.log doc_id:doc_id, error: error, "mongoexport" + MongoManager.markDocHistoryAsArchived doc_id, update, (error) -> + return callback(error) if error? + callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> if settings.filestore?.backend != "s3" diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 43dd59591b..96435fd064 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -33,7 +33,7 @@ module.exports = MongoAWS = args.push '-c' args.push 'docHistory' args.push '-q' - args.push "{doc_id: ObjectId('#{doc_id}') }" + args.push "{doc_id: ObjectId('#{doc_id}') , expiresAt: {$exists : false} }" args.push '-o' args.push filepath diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 536b94f876..9163da8009 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -140,13 +140,12 @@ module.exports = MongoManager = getArchivedDocChanges: (doc_id, callback)-> db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback - markDocHistoryAsArchived: (doc_id, callback)-> - MongoManager.getLastCompressedUpdate doc_id, (error, update) -> - db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> + markDocHistoryAsArchived: (doc_id, update, callback)-> + db.docHistory.update { v: update.v }, { $set : { inS3 : true } }, (error)-> + return callback(error) if error? + db.docHistory.remove { doc_id : doc_id, inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)-> return callback(error) if error? - db.docHistory.remove { doc_id : doc_id, inS3 : { $exists : false } }, (error)-> - return callback(error) if error? - callback(error) + callback(error) markDocHistoryAsUnarchived: (doc_id, callback)-> db.docHistory.update { doc_id: doc_id }, { $unset : { inS3 : true } }, { multi: true }, (error)-> From 26c8048729da0013ad51b16e12e2662004a41784 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 14 Aug 2015 19:19:54 -0300 Subject: [PATCH 09/39] change mongo stream method (still have a bug in bulk insert limit) --- .../track-changes/app/coffee/MongoAWS.coffee | 79 ++++++++++++++++++- services/track-changes/package.json | 4 +- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 96435fd064..d5d243cb2d 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,15 +5,92 @@ logger = require "logger-sharelatex" AWS = require 'aws-sdk' fs = require 'fs' S3S = require 's3-streams' +{db, ObjectId} = require "./mongojs" +JSONStream = require "JSONStream" +ReadlineStream = require "readline-stream" module.exports = MongoAWS = + bulkLimit: 10 + archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + query = { + doc_id: ObjectId(doc_id) + expiresAt: {$exists : false} + } + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + upload = S3S.WriteStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + db.docHistory.find(query) + .pipe JSONStream.stringify() + .pipe upload + .on 'finish', () -> + return callback(null) + + unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + download = S3S.ReadStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + }, { + encoding: "utf8" + } + + lineStream = new ReadlineStream(); + ops = [] + + download + .on 'open', (obj) -> + return 1 + .pipe lineStream + .on 'data', (line) -> + if line.length > 2 + ops.push(JSON.parse(line)) + if ops.length > MongoAWS.bulkLimit + MongoAWS.handleBulk ops, () -> + ops.splice(0,ops.length) + .on 'end', () -> + MongoAWS.handleBulk ops, callback + .on 'error', (err) -> + return callback(err) + + handleBulk: (ops, cb) -> + bulk = db.docHistory.initializeUnorderedBulkOp(); + + for op in ops + op._id = ObjectId(op._id) + op.doc_id = ObjectId(op.doc_id) + op.project_id = ObjectId(op.project_id) + bulk.find({_id:op._id}).upsert().updateOne(op) + + bulk.execute (err, result) -> + if err? + logger.error err:err, "error bulking ReadlineStream" + else + logger.log result:result, "bulked ReadlineStream" + cb(err) + + + archiveDocHistoryExternal: (project_id, doc_id, callback = (error) ->) -> MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> MongoAWS.s3upStream project_id, doc_id, filepath, callback #delete temp file? - unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + + unArchiveDocHistoryExternal: (project_id, doc_id, callback = (error) ->) -> MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> if error == null MongoAWS.mongoImportDocHistory filepath, callback diff --git a/services/track-changes/package.json b/services/track-changes/package.json index f82b618359..4fe112262a 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -20,7 +20,9 @@ "redis": "~0.10.1", "underscore": "~1.7.0", "mongo-uri": "^0.1.2", - "s3-streams": "^0.3.0" + "s3-streams": "^0.3.0", + "JSONStream": "^1.0.4", + "readline-stream": "^1.0.1" }, "devDependencies": { "chai": "~1.9.0", From 20c3e15f930a0c7f4ff39b9f0acb21b04e75efb0 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 14 Aug 2015 19:58:38 -0300 Subject: [PATCH 10/39] fix bulk insert limit --- services/track-changes/app/coffee/MongoAWS.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index d5d243cb2d..9b4271c167 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -59,9 +59,9 @@ module.exports = MongoAWS = .on 'data', (line) -> if line.length > 2 ops.push(JSON.parse(line)) - if ops.length > MongoAWS.bulkLimit - MongoAWS.handleBulk ops, () -> - ops.splice(0,ops.length) + if ops.length == MongoAWS.bulkLimit + MongoAWS.handleBulk ops.slice(0), () -> + ops.splice(0,ops.length) .on 'end', () -> MongoAWS.handleBulk ops, callback .on 'error', (err) -> @@ -80,7 +80,7 @@ module.exports = MongoAWS = if err? logger.error err:err, "error bulking ReadlineStream" else - logger.log result:result, "bulked ReadlineStream" + logger.log count:ops.length, result:result, "bulked ReadlineStream" cb(err) From 04ec45529fa96a7dd485e6baab1703726cded0f3 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 18 Aug 2015 17:11:19 -0300 Subject: [PATCH 11/39] restore updates from S3 when exists fix: avoid rearchiving --- .../track-changes/app/coffee/MongoManager.coffee | 2 +- .../track-changes/app/coffee/UpdatesManager.coffee | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 9163da8009..44e1335e8c 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -135,7 +135,7 @@ module.exports = MongoManager = db.docs.find project_id: ObjectId(project_id.toString()), {}, callback getDocChangesCount: (doc_id, callback)-> - db.docHistory.count doc_id: ObjectId(doc_id.toString()), {}, callback + db.docHistory.count { doc_id : doc_id, inS3 : { $exists : false }}, {}, callback getArchivedDocChanges: (doc_id, callback)-> db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 1fbfe53d7e..c104ceed7a 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -95,9 +95,17 @@ module.exports = UpdatesManager = getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.processUncompressedUpdatesForProject project_id, (error) -> return callback(error) if error? - DocArchiveManager.unArchiveAllDocsChanges project_id, (error) -> - return callback(error,null) if error? - MongoManager.getProjectUpdates project_id, options, callback + MongoManager.getProjectUpdates project_id, options, (error, updates) -> + jobs = [] + for update in updates + if update.inS3? + do (update) -> + jobs.push (callback) -> DocArchiveManager.unArchiveDocChanges update.project_id, update.doc_id, callback + if jobs.length? + async.series jobs, (err) -> + MongoManager.getProjectUpdates project_id, options, callback + else + callback(error, updates) getProjectUpdatesWithUserInfo: (project_id, options = {}, callback = (error, updates) ->) -> UpdatesManager.getProjectUpdates project_id, options, (error, updates) -> From 98ce03b2f2ebf6523d9a299ba969fe1286fb71d8 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 24 Aug 2015 10:38:31 -0300 Subject: [PATCH 12/39] replace docs collection to DocstoreHandler --- .../app/coffee/DocArchiveManager.coffee | 11 +++++----- .../app/coffee/DocstoreHandler.coffee | 20 +++++++++++++++++++ .../app/coffee/MongoManager.coffee | 3 --- .../track-changes/app/coffee/mongojs.coffee | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 services/track-changes/app/coffee/DocstoreHandler.coffee diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 54d05cab96..e4aab49f40 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -1,20 +1,18 @@ MongoManager = require "./MongoManager" MongoAWS = require "./MongoAWS" LockManager = require "./LockManager" +DocstoreHandler = require "./DocstoreHandler" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" settings = require("settings-sharelatex") -request = require("request") -crypto = require("crypto") -thirtySeconds = 30 * 1000 module.exports = DocArchiveManager = archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> if settings.filestore?.backend != "s3" return callback(null) - MongoManager.getProjectsDocs project_id, (error, docs) -> + DocstoreHandler.getAllDocs project_id, (error, docs) -> if error? return callback(error) else if !docs? @@ -46,7 +44,7 @@ module.exports = DocArchiveManager = unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> if settings.filestore?.backend != "s3" return callback(null) - MongoManager.getProjectsDocs project_id, (error, docs) -> + DocstoreHandler.getAllDocs project_id, (error, docs) -> if error? return callback(error) else if !docs? @@ -65,3 +63,6 @@ module.exports = DocArchiveManager = MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> return callback(error) if error? callback() + + getProjectsDocs: (project_id, callback)-> + db.docs.find project_id: ObjectId(project_id.toString()), {}, callback diff --git a/services/track-changes/app/coffee/DocstoreHandler.coffee b/services/track-changes/app/coffee/DocstoreHandler.coffee new file mode 100644 index 0000000000..7ca270fb0f --- /dev/null +++ b/services/track-changes/app/coffee/DocstoreHandler.coffee @@ -0,0 +1,20 @@ +request = require("request").defaults(jar: false) +logger = require "logger-sharelatex" +settings = require "settings-sharelatex" + +module.exports = DocstoreHandler = + + getAllDocs: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "getting all docs for project in docstore api" + url = "#{settings.apis.docstore.url}/project/#{project_id}/doc" + request.get { + url: url + json: true + }, (error, res, docs) -> + return callback(error) if error? + if 200 <= res.statusCode < 300 + callback(null, docs) + else + error = new Error("docstore api responded with non-success code: #{res.statusCode}") + logger.error err: error, project_id: project_id, "error getting all docs from docstore" + callback(error) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 44e1335e8c..a18fe0f2ae 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -131,9 +131,6 @@ module.exports = MongoManager = db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true } db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true } - getProjectsDocs: (project_id, callback)-> - db.docs.find project_id: ObjectId(project_id.toString()), {}, callback - getDocChangesCount: (doc_id, callback)-> db.docHistory.count { doc_id : doc_id, inS3 : { $exists : false }}, {}, callback diff --git a/services/track-changes/app/coffee/mongojs.coffee b/services/track-changes/app/coffee/mongojs.coffee index 5c1e9ea252..32efbc9a1d 100644 --- a/services/track-changes/app/coffee/mongojs.coffee +++ b/services/track-changes/app/coffee/mongojs.coffee @@ -1,6 +1,6 @@ Settings = require "settings-sharelatex" mongojs = require "mongojs" -db = mongojs.connect(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryStats", "docs"]) +db = mongojs.connect(Settings.mongo.url, ["docHistory", "projectHistoryMetaData", "docHistoryStats"]) module.exports = db: db ObjectId: mongojs.ObjectId From 1ccba422c81074385d1285a10d6bd8c518c443bc Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 24 Aug 2015 10:55:27 -0300 Subject: [PATCH 13/39] remove unused function --- services/track-changes/app/coffee/DocArchiveManager.coffee | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index e4aab49f40..e0d0d1bf1a 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -63,6 +63,3 @@ module.exports = DocArchiveManager = MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> return callback(error) if error? callback() - - getProjectsDocs: (project_id, callback)-> - db.docs.find project_id: ObjectId(project_id.toString()), {}, callback From fcbe4aa92576b5a3d842a161724159eef285e3df Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 24 Aug 2015 12:19:19 -0300 Subject: [PATCH 14/39] fix inS3 propagation --- .../app/coffee/MongoManager.coffee | 19 ++++++++++++------- .../app/coffee/UpdatesManager.coffee | 5 ++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index a18fe0f2ae..b366ce51b9 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -1,6 +1,7 @@ {db, ObjectId} = require "./mongojs" PackManager = require "./PackManager" async = require "async" +logger = require "logger-sharelatex" module.exports = MongoManager = getLastCompressedUpdate: (doc_id, callback = (error, update) ->) -> @@ -47,7 +48,6 @@ module.exports = MongoManager = insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) -> - inS3 = update.inS3? update = { doc_id: ObjectId(doc_id.toString()) project_id: ObjectId(project_id.toString()) @@ -55,8 +55,6 @@ module.exports = MongoManager = meta: update.meta v: update.v } - if inS3? - update.inS3 = true if temporary seconds = 1000 @@ -132,18 +130,25 @@ module.exports = MongoManager = db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true } getDocChangesCount: (doc_id, callback)-> - db.docHistory.count { doc_id : doc_id, inS3 : { $exists : false }}, {}, callback + db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, {}, callback getArchivedDocChanges: (doc_id, callback)-> db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback + remarkDocHistoryAsArchived: (doc_id, callback)-> + logger.log doc_id: doc_id, "remarkDocHistoryAsArchived" + MongoManager.getLastCompressedUpdate doc_id, (error, update) -> + db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> + return callback(error) if error? + callback(error) + markDocHistoryAsArchived: (doc_id, update, callback)-> - db.docHistory.update { v: update.v }, { $set : { inS3 : true } }, (error)-> + db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> return callback(error) if error? - db.docHistory.remove { doc_id : doc_id, inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)-> + db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)-> return callback(error) if error? callback(error) markDocHistoryAsUnarchived: (doc_id, callback)-> - db.docHistory.update { doc_id: doc_id }, { $unset : { inS3 : true } }, { multi: true }, (error)-> + db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)-> callback(error) diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index c104ceed7a..7e89be9ea3 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -36,7 +36,10 @@ module.exports = UpdatesManager = MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) -> return callback(error) if error? logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" - callback() + if lastCompressedUpdate.inS3? + MongoManager.remarkDocHistoryAsArchived doc_id, callback + else + callback() REDIS_READ_BATCH_SIZE: 100 processUncompressedUpdates: (project_id, doc_id, callback = (error) ->) -> From f910e63e903af890307acbb17be1ec9587c38782 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 24 Aug 2015 12:22:17 -0300 Subject: [PATCH 15/39] fix null case --- services/track-changes/app/coffee/UpdatesManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 7e89be9ea3..38c97144a0 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -36,7 +36,7 @@ module.exports = UpdatesManager = MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) -> return callback(error) if error? logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" - if lastCompressedUpdate.inS3? + if lastCompressedUpdate?.inS3? MongoManager.remarkDocHistoryAsArchived doc_id, callback else callback() From efff026a79ed97b820df64fbdd499a075f71eafc Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 25 Aug 2015 16:52:28 -0300 Subject: [PATCH 16/39] handle easier propagation --- services/track-changes/app/coffee/MongoManager.coffee | 10 +++------- .../track-changes/app/coffee/UpdatesManager.coffee | 9 +++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index b366ce51b9..1077199c2c 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -48,6 +48,7 @@ module.exports = MongoManager = insertCompressedUpdate: (project_id, doc_id, update, temporary, callback = (error) ->) -> + inS3 = update.inS3? update = { doc_id: ObjectId(doc_id.toString()) project_id: ObjectId(project_id.toString()) @@ -55,6 +56,8 @@ module.exports = MongoManager = meta: update.meta v: update.v } + if inS3 + update.inS3 = true if temporary seconds = 1000 @@ -135,13 +138,6 @@ module.exports = MongoManager = getArchivedDocChanges: (doc_id, callback)-> db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: true }, {}, callback - remarkDocHistoryAsArchived: (doc_id, callback)-> - logger.log doc_id: doc_id, "remarkDocHistoryAsArchived" - MongoManager.getLastCompressedUpdate doc_id, (error, update) -> - db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> - return callback(error) if error? - callback(error) - markDocHistoryAsArchived: (doc_id, update, callback)-> db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> return callback(error) if error? diff --git a/services/track-changes/app/coffee/UpdatesManager.coffee b/services/track-changes/app/coffee/UpdatesManager.coffee index 38c97144a0..b500d5581c 100644 --- a/services/track-changes/app/coffee/UpdatesManager.coffee +++ b/services/track-changes/app/coffee/UpdatesManager.coffee @@ -7,6 +7,7 @@ UpdateTrimmer = require "./UpdateTrimmer" logger = require "logger-sharelatex" async = require "async" DocArchiveManager = require "./DocArchiveManager" +_ = require "underscore" module.exports = UpdatesManager = compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) -> @@ -33,13 +34,13 @@ module.exports = UpdatesManager = return compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates + if lastCompressedUpdate?.inS3? and not _.some(compressedUpdates, (update) -> update.inS3) + compressedUpdates[compressedUpdates.length-1].inS3 = lastCompressedUpdate.inS3 + MongoManager.insertCompressedUpdates project_id, doc_id, compressedUpdates, temporary,(error) -> return callback(error) if error? logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates" - if lastCompressedUpdate?.inS3? - MongoManager.remarkDocHistoryAsArchived doc_id, callback - else - callback() + callback() REDIS_READ_BATCH_SIZE: 100 processUncompressedUpdates: (project_id, doc_id, callback = (error) ->) -> From 1abcea1a66f4354e84ed16577d0dcbe05055e808 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 31 Aug 2015 18:13:18 -0300 Subject: [PATCH 17/39] add some unit test --- .../app/coffee/MongoManager.coffee | 2 +- .../DocArchive/DocArchiveManager.coffee | 44 ++++++++++ .../coffee/DocArchive/DocstoreHandler.coffee | 55 ++++++++++++ .../unit/coffee/DocArchive/MongoAWS.coffee | 78 +++++++++++++++++ .../HttpController/HttpControllerTests.coffee | 37 ++++++++ .../MongoManager/MongoManagerTests.coffee | 86 +++++++++++++++++++ 6 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee create mode 100644 services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee create mode 100644 services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 1077199c2c..3bcf50af2a 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -122,7 +122,7 @@ module.exports = MongoManager = db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, { background: true } # For finding all packs that affect a project (use a sparse index so only packs are included) db.docHistory.ensureIndex { project_id: 1, "pack.0.meta.end_ts": 1, "meta.end_ts": 1}, { background: true, sparse: true } - # For finding updates that dont yet have a project_id and need it inserting + # For finding updates that don't yet have a project_id and need it inserting db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, { background: true } # For finding project meta-data db.projectHistoryMetaData.ensureIndex { project_id: 1 }, { background: true } diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee new file mode 100644 index 0000000000..c76afab988 --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -0,0 +1,44 @@ +chai = require('chai') +sinon = require("sinon") +should = chai.should() +modulePath = "../../../../app/js/DocArchiveManager.js" +SandboxedModule = require('sandboxed-module') +ObjectId = require("mongojs").ObjectId + +describe "DocArchiveManager", -> + beforeEach -> + @DocArchiveManager = SandboxedModule.require modulePath, requires: + "./MongoManager" : @MongoManager = sinon.stub() + "./MongoAWS" : @MongoAWS = sinon.stub() + "./LockManager" : @LockManager = sinon.stub() + "./DocstoreHandler" : @DocstoreHandler = sinon.stub() + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + "settings-sharelatex": @settings = + filestore: + backend: 's3' + + @mongoDocs = [{ + _id: ObjectId() + }, { + _id: ObjectId() + }, { + _id: ObjectId() + }] + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + + describe "archiveAllDocsChanges", -> + + it "should archive all project docs change", (done)-> + + @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) + @DocArchiveManager.archiveDocChangesWithLock = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.archiveAllDocsChanges @project_id, (err)=> + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true + @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true + should.not.exist err + done() diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee new file mode 100644 index 0000000000..279f2a979b --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/DocstoreHandler.coffee @@ -0,0 +1,55 @@ +chai = require('chai') +chai.should() +sinon = require("sinon") +modulePath = "../../../../app/js/DocstoreHandler.js" +SandboxedModule = require('sandboxed-module') + +describe "DocstoreHandler", -> + beforeEach -> + @requestDefaults = sinon.stub().returns(@request = sinon.stub()) + @DocstoreHandler = SandboxedModule.require modulePath, requires: + "request" : defaults: @requestDefaults + "settings-sharelatex": @settings = + apis: + docstore: + url: "docstore.sharelatex.com" + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + + @requestDefaults.calledWith(jar: false).should.equal true + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + + describe "getAllDocs", -> + describe "with a successful response code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, statusCode: 204, @docs = [{ _id: "mock-doc-id" }]) + @DocstoreHandler.getAllDocs @project_id, @callback + + it "should get all the project docs in the docstore api", -> + @request.get + .calledWith({ + url: "#{@settings.apis.docstore.url}/project/#{@project_id}/doc" + json: true + }) + .should.equal true + + it "should call the callback with the docs", -> + @callback.calledWith(null, @docs).should.equal true + + describe "with a failed response code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, statusCode: 500, "") + @DocstoreHandler.getAllDocs @project_id, @callback + + it "should call the callback with an error", -> + @callback.calledWith(new Error("docstore api responded with non-success code: 500")).should.equal true + + it "should log the error", -> + @logger.error + .calledWith({ + err: new Error("docstore api responded with a non-success code: 500") + project_id: @project_id + }, "error getting all docs from docstore") + .should.equal true \ No newline at end of file diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee new file mode 100644 index 0000000000..88c9f4b7da --- /dev/null +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -0,0 +1,78 @@ +chai = require('chai') +chai.should() +sinon = require("sinon") +modulePath = "../../../../app/js/MongoAWS.js" +SandboxedModule = require('sandboxed-module') +{ObjectId} = require("mongojs") + +describe "MongoAWS", -> + beforeEach -> + @MongoAWS = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings = + filestore: + s3: + secret: "s3-secret" + key: "s3-key" + stores: + user_files: "s3-bucket" + "child_process": @child_process = {} + "mongo-uri": @mongouri = {} + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->} + "aws-sdk": @awssdk = {} + "fs": @fs = {} + "s3-streams": @s3streams = {} + "./mongojs" : { db: @db = {}, ObjectId: ObjectId } + "JSONStream": @JSONStream = {} + "readline-stream": @readline = sinon.stub() + + @bulkLimit = @MongoAWS.bulkLimit + @project_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @callback = sinon.stub() + + describe "archiveDocHistory", -> + + beforeEach -> + @awssdk.config = { update: sinon.stub() } + @awssdk.S3 = sinon.stub() + @s3streams.WriteStream = sinon.stub() + @db.docHistory = {} + @db.docHistory.pipe = sinon.stub() + @db.docHistory.find = sinon.stub().returns @db.docHistory + @db.docHistory.pipe.returns + pipe:-> + on: (type, cb)-> + if type == "finish" + cb() + @JSONStream.stringify = sinon.stub() + + @MongoAWS.archiveDocHistory @project_id, @doc_id, @callback + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + + describe "unArchiveDocHistory", -> + + beforeEach -> + @awssdk.config = { update: sinon.stub() } + @awssdk.S3 = sinon.stub() + @s3streams.ReadStream = sinon.stub() + + @s3streams.ReadStream.returns + #describe on 'open' behavior + on: (type, cb)-> + pipe:-> + #describe on 'data' behavior + on: (type, cb)-> + cb([]) + #describe on 'end' behavior + on: (type, cb)-> + cb() + #describe on 'error' behavior + on: sinon.stub() + + @MongoAWS.handleBulk = sinon.stub() + @MongoAWS.unArchiveDocHistory @project_id, @doc_id, @callback + + it "should call handleBulk", -> + @MongoAWS.handleBulk.calledWith([],@callback).should.equal true diff --git a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee index 7c559c0937..e9533664e3 100644 --- a/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee +++ b/services/track-changes/test/unit/coffee/HttpController/HttpControllerTests.coffee @@ -13,6 +13,7 @@ describe "HttpController", -> "./DiffManager": @DiffManager = {} "./RestoreManager": @RestoreManager = {} "./PackManager": @PackManager = {} + "./DocArchiveManager": @DocArchiveManager = {} @doc_id = "doc-id-123" @project_id = "project-id-123" @next = sinon.stub() @@ -130,3 +131,39 @@ describe "HttpController", -> it "should return a success code", -> @res.send.calledWith(204).should.equal true + + describe "archiveProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + @res = + send: sinon.stub() + @DocArchiveManager.archiveAllDocsChanges = sinon.stub().callsArg(1) + @HttpController.archiveProject @req, @res, @next + + it "should process archive doc changes", -> + @DocArchiveManager.archiveAllDocsChanges + .calledWith(@project_id) + .should.equal true + + it "should return a success code", -> + @res.send.calledWith(204).should.equal true + + describe "unArchiveProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + @res = + send: sinon.stub() + @DocArchiveManager.unArchiveAllDocsChanges = sinon.stub().callsArg(1) + @HttpController.unArchiveProject @req, @res, @next + + it "should process unarchive doc changes", -> + @DocArchiveManager.unArchiveAllDocsChanges + .calledWith(@project_id) + .should.equal true + + it "should return a success code", -> + @res.send.calledWith(204).should.equal true diff --git a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee index 24588de6ea..8db054b40f 100644 --- a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee @@ -398,3 +398,89 @@ describe "MongoManager", -> it "should call the callback", -> @callback.called.should.equal true + + describe "getDocChangesCount", -> + beforeEach -> + @db.docHistory = + count: sinon.stub().callsArg(2) + @MongoManager.getDocChangesCount @doc_id, @callback + + it "should return if there is any doc changes", -> + @db.docHistory.count + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : { $exists : false } + }, { + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "getArchivedDocChanges", -> + beforeEach -> + @db.docHistory = + count: sinon.stub().callsArg(2) + @MongoManager.getArchivedDocChanges @doc_id, @callback + + it "should return if there is any archived doc changes", -> + @db.docHistory.count + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : true + }, { + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "markDocHistoryAsArchived", -> + beforeEach -> + @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} + @db.docHistory = + update: sinon.stub().callsArg(2) + remove: sinon.stub().callsArg(1) + @MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback + + it "should update last doc change with inS3 flag", -> + @db.docHistory.update + .calledWith({ + _id: ObjectId(@update._id) + },{ + $set : { inS3 : true } + }) + .should.equal true + + it "should remove any other doc changes before last update", -> + @db.docHistory.remove + .calledWith({ + doc_id: ObjectId(@doc_id) + inS3 : { $exists : false } + v: { $lt : @update.v } + expiresAt: {$exists : false} + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "markDocHistoryAsUnarchived", -> + beforeEach -> + @db.docHistory = + update: sinon.stub().callsArg(3) + @MongoManager.markDocHistoryAsUnarchived @doc_id, @callback + + it "should remove any doc changes inS3 flag", -> + @db.docHistory.update + .calledWith({ + doc_id: ObjectId(@doc_id) + },{ + $unset : { inS3 : true } + },{ + multi: true + }) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true \ No newline at end of file From 0c16fbed8808c5e74f6207d298aa6c697472b0cb Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Sep 2015 15:39:19 -0300 Subject: [PATCH 18/39] add more unit tests --- .../DocArchive/DocArchiveManager.coffee | 64 ++++++++++++++++++- .../unit/coffee/DocArchive/MongoAWS.coffee | 33 ++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee index c76afab988..ac266132c3 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -30,9 +30,7 @@ describe "DocArchiveManager", -> @callback = sinon.stub() describe "archiveAllDocsChanges", -> - it "should archive all project docs change", (done)-> - @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) @DocArchiveManager.archiveDocChangesWithLock = sinon.stub().callsArgWith(2, null) @@ -42,3 +40,65 @@ describe "DocArchiveManager", -> @DocArchiveManager.archiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true should.not.exist err done() + + describe "archiveDocChangesWithLock", -> + beforeEach -> + @DocArchiveManager.archiveDocChanges = sinon.stub().callsArg(2) + @LockManager.runWithLock = sinon.stub().callsArg(2) + @DocArchiveManager.archiveDocChangesWithLock @project_id, @doc_id, @callback + + it "should run archiveDocChangesWithLock with the lock", -> + @LockManager.runWithLock + .calledWith( + "HistoryArchiveLock:#{@doc_id}" + ) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "archiveDocChanges", -> + beforeEach -> + @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} + @MongoManager.getDocChangesCount = sinon.stub().callsArg(1) + @MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update) + @MongoAWS.archiveDocHistory = sinon.stub().callsArg(2) + @MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2) + @DocArchiveManager.archiveDocChanges @project_id, @doc_id, @callback + + it "should run markDocHistoryAsArchived with doc_id and update", -> + @MongoManager.markDocHistoryAsArchived + .calledWith( + @doc_id, @update + ) + .should.equal true + it "should call the callback", -> + @callback.called.should.equal true + + describe "unArchiveAllDocsChanges", -> + it "should unarchive all project docs change", (done)-> + @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) + @DocArchiveManager.unArchiveDocChanges = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.unArchiveAllDocsChanges @project_id, (err)=> + @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[0]._id).should.equal true + @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[1]._id).should.equal true + @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[2]._id).should.equal true + should.not.exist err + done() + + describe "unArchiveDocChanges", -> + beforeEach -> + @MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1) + @MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2) + @MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1) + @DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback + + it "should run markDocHistoryAsUnarchived with doc_id", -> + @MongoManager.markDocHistoryAsUnarchived + .calledWith( + @doc_id + ) + .should.equal true + it "should call the callback", -> + @callback.called.should.equal true diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index 88c9f4b7da..180562dea7 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -76,3 +76,36 @@ describe "MongoAWS", -> it "should call handleBulk", -> @MongoAWS.handleBulk.calledWith([],@callback).should.equal true + + describe "handleBulk", -> + beforeEach -> + @ops = [{ + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }, { + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }, { + _id: ObjectId() + doc_id: ObjectId() + project_id: ObjectId() + }] + @bulk = + find: sinon.stub().returns + upsert: sinon.stub().returns + updateOne: sinon.stub() + execute: sinon.stub().callsArgWith(0, null, {}) + @db.docHistory = {} + @db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk + @MongoAWS.handleBulk @ops, @callback + + it "should call updateOne for each operation", -> + @bulk.find.calledWith({_id:@ops[0]._id}).should.equal true + @bulk.find.calledWith({_id:@ops[1]._id}).should.equal true + @bulk.find.calledWith({_id:@ops[2]._id}).should.equal true + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + From d2b1243701af2aeaa89f89fbf00807c25ff5b405 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Sep 2015 15:45:29 -0300 Subject: [PATCH 19/39] split MongoAWS files --- .../track-changes/app/coffee/MongoAWS.coffee | 118 ----------------- .../app/coffee/MongoAWSexternal.coffee | 123 ++++++++++++++++++ 2 files changed, 123 insertions(+), 118 deletions(-) create mode 100644 services/track-changes/app/coffee/MongoAWSexternal.coffee diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 9b4271c167..aedd474c48 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -1,9 +1,6 @@ settings = require "settings-sharelatex" -child_process = require "child_process" -mongoUri = require "mongo-uri"; logger = require "logger-sharelatex" AWS = require 'aws-sdk' -fs = require 'fs' S3S = require 's3-streams' {db, ObjectId} = require "./mongojs" JSONStream = require "JSONStream" @@ -82,118 +79,3 @@ module.exports = MongoAWS = else logger.log count:ops.length, result:result, "bulked ReadlineStream" cb(err) - - - archiveDocHistoryExternal: (project_id, doc_id, callback = (error) ->) -> - MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> - MongoAWS.s3upStream project_id, doc_id, filepath, callback - #delete temp file? - - - unArchiveDocHistoryExternal: (project_id, doc_id, callback = (error) ->) -> - MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> - if error == null - MongoAWS.mongoImportDocHistory filepath, callback - #delete temp file? - else - callback - - mongoExportDocHistory: (doc_id, callback = (error, filepath) ->) -> - uriData = mongoUri.parse(settings.mongo.url); - filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonUp' - - args = [] - args.push '-h' - args.push uriData.hosts[0] - args.push '-d' - args.push uriData.database - args.push '-c' - args.push 'docHistory' - args.push '-q' - args.push "{doc_id: ObjectId('#{doc_id}') , expiresAt: {$exists : false} }" - args.push '-o' - args.push filepath - - proc = child_process.spawn "mongoexport", args - - proc.on "error", callback - - stderr = "" - proc.stderr.on "data", (chunk) -> stderr += chunk.toString() - - proc.on "close", (code) -> - if code == 0 - return callback(null,filepath) - else - return callback(new Error("mongodump failed: #{stderr}"),null) - - mongoImportDocHistory: (filepath, callback = (error) ->) -> - - uriData = mongoUri.parse(settings.mongo.url); - - args = [] - args.push '-h' - args.push uriData.hosts[0] - args.push '-d' - args.push uriData.database - args.push '-c' - args.push 'docHistory' - args.push '--file' - args.push filepath - - proc = child_process.spawn "mongoimport", args - - proc.on "error", callback - - stderr = "" - proc.stderr.on "data", (chunk) -> stderr += chunk.toString() - - proc.on "close", (code) -> - if code == 0 - return callback(null,filepath) - else - return callback(new Error("mongodump failed: #{stderr}"),null) - - s3upStream: (project_id, doc_id, filepath, callback = (error) ->) -> - - AWS.config.update { - accessKeyId: settings.filestore.s3.key - secretAccessKey: settings.filestore.s3.secret - } - - upload = S3S.WriteStream new AWS.S3(), { - "Bucket": settings.filestore.stores.user_files, - "Key": project_id+"/changes-"+doc_id - } - - fs.createReadStream(filepath) - .on 'open', (obj) -> - return 1 - .pipe(upload) - .on 'finish', () -> - return callback(null) - .on 'error', (err) -> - return callback(err) - - s3downStream: (project_id, doc_id, callback = (error, filepath) ->) -> - - filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' - - AWS.config.update { - accessKeyId: settings.filestore.s3.key - secretAccessKey: settings.filestore.s3.secret - } - - download = S3S.ReadStream new AWS.S3(), { - "Bucket": settings.filestore.stores.user_files, - "Key": project_id+"/changes-"+doc_id - } - - download - .on 'open', (obj) -> - return 1 - .pipe(fs.createWriteStream(filepath)) - .on 'finish', () -> - return callback(null, filepath) - .on 'error', (err) -> - return callback(err, null) diff --git a/services/track-changes/app/coffee/MongoAWSexternal.coffee b/services/track-changes/app/coffee/MongoAWSexternal.coffee new file mode 100644 index 0000000000..f422a583b5 --- /dev/null +++ b/services/track-changes/app/coffee/MongoAWSexternal.coffee @@ -0,0 +1,123 @@ +settings = require "settings-sharelatex" +child_process = require "child_process" +mongoUri = require "mongo-uri"; +logger = require "logger-sharelatex" +AWS = require 'aws-sdk' +fs = require 'fs' +S3S = require 's3-streams' + +module.exports = MongoAWSexternal = + + archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.mongoExportDocHistory doc_id, (error, filepath) -> + MongoAWS.s3upStream project_id, doc_id, filepath, callback + #delete temp file? + + + unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + MongoAWS.s3downStream project_id, doc_id, (error, filepath) -> + if error == null + MongoAWS.mongoImportDocHistory filepath, callback + #delete temp file? + else + callback + + mongoExportDocHistory: (doc_id, callback = (error, filepath) ->) -> + uriData = mongoUri.parse(settings.mongo.url); + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonUp' + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '-q' + args.push "{doc_id: ObjectId('#{doc_id}') , expiresAt: {$exists : false} }" + args.push '-o' + args.push filepath + + proc = child_process.spawn "mongoexport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + + mongoImportDocHistory: (filepath, callback = (error) ->) -> + + uriData = mongoUri.parse(settings.mongo.url); + + args = [] + args.push '-h' + args.push uriData.hosts[0] + args.push '-d' + args.push uriData.database + args.push '-c' + args.push 'docHistory' + args.push '--file' + args.push filepath + + proc = child_process.spawn "mongoimport", args + + proc.on "error", callback + + stderr = "" + proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + + proc.on "close", (code) -> + if code == 0 + return callback(null,filepath) + else + return callback(new Error("mongodump failed: #{stderr}"),null) + + s3upStream: (project_id, doc_id, filepath, callback = (error) ->) -> + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + upload = S3S.WriteStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + fs.createReadStream(filepath) + .on 'open', (obj) -> + return 1 + .pipe(upload) + .on 'finish', () -> + return callback(null) + .on 'error', (err) -> + return callback(err) + + s3downStream: (project_id, doc_id, callback = (error, filepath) ->) -> + + filepath = settings.path.dumpFolder + '/' + doc_id + '.jsonDown' + + AWS.config.update { + accessKeyId: settings.filestore.s3.key + secretAccessKey: settings.filestore.s3.secret + } + + download = S3S.ReadStream new AWS.S3(), { + "Bucket": settings.filestore.stores.user_files, + "Key": project_id+"/changes-"+doc_id + } + + download + .on 'open', (obj) -> + return 1 + .pipe(fs.createWriteStream(filepath)) + .on 'finish', () -> + return callback(null, filepath) + .on 'error', (err) -> + return callback(err, null) From 7de103af689e91e8f4baa9e547bd6d3059c7a695 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Sep 2015 17:00:32 -0300 Subject: [PATCH 20/39] fix unit scope error --- .../test/unit/coffee/DocArchive/MongoAWS.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index 180562dea7..4279ec0ab6 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -79,7 +79,7 @@ describe "MongoAWS", -> describe "handleBulk", -> beforeEach -> - @ops = [{ + @bulkOps = [{ _id: ObjectId() doc_id: ObjectId() project_id: ObjectId() @@ -99,12 +99,12 @@ describe "MongoAWS", -> execute: sinon.stub().callsArgWith(0, null, {}) @db.docHistory = {} @db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk - @MongoAWS.handleBulk @ops, @callback + @MongoAWS.handleBulk @bulkOps, @callback it "should call updateOne for each operation", -> - @bulk.find.calledWith({_id:@ops[0]._id}).should.equal true - @bulk.find.calledWith({_id:@ops[1]._id}).should.equal true - @bulk.find.calledWith({_id:@ops[2]._id}).should.equal true + @bulk.find.calledWith({_id:@bulkOps[0]._id}).should.equal true + @bulk.find.calledWith({_id:@bulkOps[1]._id}).should.equal true + @bulk.find.calledWith({_id:@bulkOps[2]._id}).should.equal true it "should call the callback", -> @callback.calledWith(null).should.equal true From da9e7dc7e1db69ac9023f09081edf98f45ee1d02 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Sep 2015 18:47:34 -0300 Subject: [PATCH 21/39] init archive acceptance tests --- .../app/coffee/DocArchiveManager.coffee | 4 +- .../coffee/ArchivingUpdatesTests.coffee | 88 +++++++++++++++++++ .../coffee/helpers/MockDocStoreApi.coffee | 24 +++++ .../coffee/helpers/TrackChangesClient.coffee | 16 +++- 4 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee create mode 100644 services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index e0d0d1bf1a..ab54a83039 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -36,7 +36,7 @@ module.exports = DocArchiveManager = else MongoManager.getLastCompressedUpdate doc_id, (error, update) -> MongoAWS.archiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "mongoexport" + logger.log doc_id:doc_id, error: error, "export to S3" MongoManager.markDocHistoryAsArchived doc_id, update, (error) -> return callback(error) if error? callback() @@ -59,7 +59,7 @@ module.exports = DocArchiveManager = return callback() else MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "mongoimport" + logger.log doc_id:doc_id, error: error, "import from S3" MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> return callback(error) if error? callback() diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee new file mode 100644 index 0000000000..4859d28761 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -0,0 +1,88 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +expect = chai.expect +mongojs = require "../../../app/js/mongojs" +db = mongojs.db +ObjectId = mongojs.ObjectId +Settings = require "settings-sharelatex" +request = require "request" +rclient = require("redis").createClient() # Only works locally for now + +TrackChangesClient = require "./helpers/TrackChangesClient" +MockDocStoreApi = require "./helpers/MockDocStoreApi" +MockWebApi = require "./helpers/MockWebApi" + +describe "Archiving updates", -> + before (done) -> + @now = Date.now() + @to = @now + @user_id = ObjectId().toString() + @doc_id = ObjectId().toString() + @project_id = ObjectId().toString() + + @minutes = 60 * 1000 + @hours = 60 * @minutes + + MockWebApi.projects[@project_id] = + features: + versioning: true + + MockWebApi.users[@user_id] = @user = + email: "user@sharelatex.com" + first_name: "Leo" + last_name: "Lion" + id: @user_id + sinon.spy MockWebApi, "getUser" + + MockDocStoreApi.docs[@doc_id] = @doc = + _id: @doc_id + project_id: @project_id + sinon.spy MockDocStoreApi, "getAllDoc" + + @updates = [] + for i in [0..9] + @updates.push { + op: [{ i: "a", p: 0 }] + meta: { ts: @now - (9 - i) * @hours - 2 * @minutes, user_id: @user_id } + v: 2 * i + 1 + } + @updates.push { + op: [{ i: "b", p: 0 }] + meta: { ts: @now - (9 - i) * @hours, user_id: @user_id } + v: 2 * i + 2 + } + + TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) => + throw error if error? + TrackChangesClient.flushDoc @project_id, @doc_id, (error) -> + throw error if error? + done() + + after: () -> + MockWebApi.getUser.restore() + + describe "archiving a doc's updates", -> + before (done) -> + + TrackChangesClient.archiveProject @project_id, (error) -> + throw error if error? + done() + + it "should remain one doc", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + throw error if error? + count.should.equal 1 + done() + + it "should remained doc marked as inS3", (done) -> + db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + throw error if error? + doc.inS3.should.equal true + done() + + it "should remained doc have last version", (done) -> + db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + throw error if error? + doc.v.should.equal 20 + done() \ No newline at end of file diff --git a/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee b/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee new file mode 100644 index 0000000000..29864479a4 --- /dev/null +++ b/services/track-changes/test/acceptance/coffee/helpers/MockDocStoreApi.coffee @@ -0,0 +1,24 @@ +express = require("express") +app = express() + +module.exports = MockDocUpdaterApi = + docs: {} + + getAllDoc: (project_id, callback = (error) ->) -> + callback null, @docs + + run: () -> + app.get "/project/:project_id/doc", (req, res, next) => + @getAllDoc req.params.project_id, (error, docs) -> + if error? + res.send 500 + if !docs? + res.send 404 + else + res.send JSON.stringify docs + + app.listen 3016, (error) -> + throw error if error? + +MockDocUpdaterApi.run() + diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee index 661b0eafb8..552d455d31 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -71,4 +71,18 @@ module.exports = TrackChangesClient = "X-User-Id": user_id }, (error, response, body) => response.statusCode.should.equal 204 - callback null \ No newline at end of file + callback null + + archiveProject: (project_id, callback = (error) ->) -> + request.get { + url: "http://localhost:3015/project/#{project_id}/archive" + }, (error, response, body) => + response.statusCode.should.equal 204 + callback(error) + + unarchiveProject: (project_id, callback = (error) ->) -> + request.get { + url: "http://localhost:3015/project/#{project_id}/unarchive" + }, (error, response, body) => + response.statusCode.should.equal 204 + callback(error) \ No newline at end of file From c5a8a249c687fa74404556d3f7440de5c5ea7e65 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 3 Sep 2015 08:36:32 -0300 Subject: [PATCH 22/39] add unarchive acceptance tests --- .../track-changes/app/coffee/MongoAWS.coffee | 17 ++++++---- .../coffee/ArchivingUpdatesTests.coffee | 32 ++++++++++++++++--- .../coffee/helpers/TrackChangesClient.coffee | 22 ++++++++++++- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index aedd474c48..3fba799d22 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -72,10 +72,13 @@ module.exports = MongoAWS = op.doc_id = ObjectId(op.doc_id) op.project_id = ObjectId(op.project_id) bulk.find({_id:op._id}).upsert().updateOne(op) - - bulk.execute (err, result) -> - if err? - logger.error err:err, "error bulking ReadlineStream" - else - logger.log count:ops.length, result:result, "bulked ReadlineStream" - cb(err) + + if ops.length > 0 + bulk.execute (err, result) -> + if err? + logger.error err:err, "error bulking ReadlineStream" + else + logger.log count:ops.length, result:result, "bulked ReadlineStream" + cb(err) + else + cb() diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index 4859d28761..973d4bde06 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -59,17 +59,18 @@ describe "Archiving updates", -> throw error if error? done() - after: () -> + after (done) -> MockWebApi.getUser.restore() + db.docHistory.remove {project_id: ObjectId(@project_id)} + TrackChangesClient.removeS3Doc @project_id, @doc_id, done describe "archiving a doc's updates", -> before (done) -> - TrackChangesClient.archiveProject @project_id, (error) -> throw error if error? done() - it "should remain one doc", (done) -> + it "should remain one doc change", (done) -> db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> throw error if error? count.should.equal 1 @@ -85,4 +86,27 @@ describe "Archiving updates", -> db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> throw error if error? doc.v.should.equal 20 - done() \ No newline at end of file + done() + + it "should store twenty doc changes in S3", (done) -> + TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) => + doc.length.should.equal 20 + done() + + describe "unarchiving a doc's updates", -> + before (done) -> + TrackChangesClient.unarchiveProject @project_id, (error) -> + throw error if error? + done() + + it "should restore doc changes", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + throw error if error? + count.should.equal 20 + done() + + it "should remove doc marked as inS3", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) -> + throw error if error? + count.should.equal 0 + done() diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee index 552d455d31..1e409908d9 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -1,6 +1,7 @@ request = require "request" rclient = require("redis").createClient() # Only works locally for now {db, ObjectId} = require "../../../../app/js/mongojs" +Settings = require "settings-sharelatex" module.exports = TrackChangesClient = flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) -> @@ -85,4 +86,23 @@ module.exports = TrackChangesClient = url: "http://localhost:3015/project/#{project_id}/unarchive" }, (error, response, body) => response.statusCode.should.equal 204 - callback(error) \ No newline at end of file + callback(error) + + buildS3Options: (content, key)-> + return { + aws: + key: Settings.filestore.s3.key + secret: Settings.filestore.s3.secret + bucket: Settings.filestore.stores.user_files + timeout: 30 * 1000 + json: content + uri:"https://#{Settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } + + getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> + options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id) + request.get options, callback + + removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> + options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id) + request.del options, callback \ No newline at end of file From 0b3ebcff065f2428909f241444e94745511465b7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 8 Sep 2015 16:23:15 +0100 Subject: [PATCH 23/39] remove if statments checking if s3 is a backend if its not enable then it can crash. In prod it should always be there or not used at all --- services/track-changes/app.coffee | 5 ++--- services/track-changes/app/coffee/DocArchiveManager.coffee | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index f7af275d62..f5f12078ac 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -27,9 +27,8 @@ app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpContro app.post "/doc/:doc_id/pack", HttpController.packDoc -if Settings.filestore?.backend == "s3" - app.get '/project/:project_id/archive', HttpController.archiveProject - app.get '/project/:project_id/unarchive', HttpController.unArchiveProject +app.get '/project/:project_id/archive', HttpController.archiveProject +app.get '/project/:project_id/unarchive', HttpController.unArchiveProject packWorker = null # use a single packing worker diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index ab54a83039..cf69f69b8a 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -10,8 +10,6 @@ settings = require("settings-sharelatex") module.exports = DocArchiveManager = archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> - if settings.filestore?.backend != "s3" - return callback(null) DocstoreHandler.getAllDocs project_id, (error, docs) -> if error? return callback(error) @@ -42,8 +40,6 @@ module.exports = DocArchiveManager = callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> - if settings.filestore?.backend != "s3" - return callback(null) DocstoreHandler.getAllDocs project_id, (error, docs) -> if error? return callback(error) From 17b0d99a65c6ac00ac14d308cbbf0500c45931fc Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 8 Sep 2015 16:26:01 +0100 Subject: [PATCH 24/39] rework the archiveDocChangesWithLock function make it a bit more readable for me, struggle to trust indentation based calls in coffeescript --- .../track-changes/app/coffee/DocArchiveManager.coffee | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index cf69f69b8a..b316863d0b 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -20,12 +20,9 @@ module.exports = DocArchiveManager = async.series jobs, callback archiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> - LockManager.runWithLock( - "HistoryArchiveLock:#{doc_id}", - (releaseLock) -> - DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock - callback - ) + job = (releaseLock) -> + DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock + LockManager.runWithLock("HistoryArchiveLock:#{doc_id}", job, callback) archiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getDocChangesCount doc_id, (error, count) -> From 18d817ee0a6da3f32d1aef3670dcbcaca01e474e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 8 Sep 2015 16:33:45 +0100 Subject: [PATCH 25/39] added some missing error handling --- .../track-changes/app/coffee/DocArchiveManager.coffee | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index b316863d0b..0e3eeba13e 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -26,12 +26,15 @@ module.exports = DocArchiveManager = archiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getDocChangesCount doc_id, (error, count) -> + return callback(error) if error? if count == 0 return callback() else MongoManager.getLastCompressedUpdate doc_id, (error, update) -> + return callback(error) if error? MongoAWS.archiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "export to S3" + return callback(error) if error? + logger.log doc_id:doc_id, project_id:project_id, "exported document to S3" MongoManager.markDocHistoryAsArchived doc_id, update, (error) -> return callback(error) if error? callback() @@ -48,11 +51,13 @@ module.exports = DocArchiveManager = unArchiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getArchivedDocChanges doc_id, (error, count) -> + return callback(error) if error? if count == 0 return callback() else MongoAWS.unArchiveDocHistory project_id, doc_id, (error) -> - logger.log doc_id:doc_id, error: error, "import from S3" + return callback(error) if error? + logger.log doc_id:doc_id, project_id:project_id, "imported document from S3" MongoManager.markDocHistoryAsUnarchived doc_id, (error) -> return callback(error) if error? callback() From 70200a9cf1987593373f1ea754afed3a6921a322 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 15:31:43 +0100 Subject: [PATCH 26/39] only log document ids, not document content avoid filling the log with large documents --- services/track-changes/app/coffee/DocstoreHandler.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/track-changes/app/coffee/DocstoreHandler.coffee b/services/track-changes/app/coffee/DocstoreHandler.coffee index 7ca270fb0f..9e3b10d258 100644 --- a/services/track-changes/app/coffee/DocstoreHandler.coffee +++ b/services/track-changes/app/coffee/DocstoreHandler.coffee @@ -12,6 +12,7 @@ module.exports = DocstoreHandler = json: true }, (error, res, docs) -> return callback(error) if error? + logger.log {error, res, docs: if docs?.length then docs.map (d) -> d._id else []}, "docstore response" if 200 <= res.statusCode < 300 callback(null, docs) else From dfa0036507dcab5dd6dadd74a9f4c491b5edebd4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 15:32:36 +0100 Subject: [PATCH 27/39] pause stream while writing to mongo --- services/track-changes/app/coffee/MongoAWS.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 3fba799d22..583244bbae 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -57,7 +57,9 @@ module.exports = MongoAWS = if line.length > 2 ops.push(JSON.parse(line)) if ops.length == MongoAWS.bulkLimit + download.pause() MongoAWS.handleBulk ops.slice(0), () -> + download.resume() ops.splice(0,ops.length) .on 'end', () -> MongoAWS.handleBulk ops, callback From 82d0f4fce823128c652fe7a281b067dba74eada7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 15:33:59 +0100 Subject: [PATCH 28/39] make unarchive more responsive by downloading documents in parallel unarchive is triggered interactively so we should try to make it reasonably fast --- services/track-changes/app/coffee/DocArchiveManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 0e3eeba13e..210f6d9f9c 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -47,7 +47,7 @@ module.exports = DocArchiveManager = return callback new Error("No docs for project #{project_id}") jobs = _.map docs, (doc) -> (cb)-> DocArchiveManager.unArchiveDocChanges project_id, doc._id, cb - async.series jobs, callback + async.parallelLimit jobs, 4, callback unArchiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getArchivedDocChanges doc_id, (error, count) -> From 1c1b1d95956f899dabd853085e3d6647bb5c785d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 15:34:30 +0100 Subject: [PATCH 29/39] log the case where there are no entries in the document history --- services/track-changes/app/coffee/DocArchiveManager.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 210f6d9f9c..4da954b6b3 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -28,6 +28,7 @@ module.exports = DocArchiveManager = MongoManager.getDocChangesCount doc_id, (error, count) -> return callback(error) if error? if count == 0 + logger.log {project_id, doc_id}, "document history is empty, not archiving" return callback() else MongoManager.getLastCompressedUpdate doc_id, (error, update) -> From d9085a5e5e5ab0a5737da77ca42a1f07e03246f0 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:00:25 +0100 Subject: [PATCH 30/39] add error handler to each stage of upload pipeline --- .../track-changes/app/coffee/MongoAWS.coffee | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 583244bbae..7fe90378c6 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -10,7 +10,12 @@ module.exports = MongoAWS = bulkLimit: 10 - archiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + archiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> + + callback = (args...) -> + _callback(args...) + _callback = () -> + query = { doc_id: ObjectId(doc_id) expiresAt: {$exists : false} @@ -27,10 +32,14 @@ module.exports = MongoAWS = } db.docHistory.find(query) + .on 'error', (err) -> + callback(err) .pipe JSONStream.stringify() - .pipe upload - .on 'finish', () -> - return callback(null) + .pipe upload + .on 'error', (err) -> + callback(err) + .on 'finish', () -> + return callback(null) unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> From 9d39012b49b0518ab6a0a17ac12f7707d7b8c7ae Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:00:37 +0100 Subject: [PATCH 31/39] add error handler to each stage of download pipeline --- services/track-changes/app/coffee/MongoAWS.coffee | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 7fe90378c6..ca4914c733 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -41,7 +41,11 @@ module.exports = MongoAWS = .on 'finish', () -> return callback(null) - unArchiveDocHistory: (project_id, doc_id, callback = (error) ->) -> + unArchiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> + + callback = (args...) -> + _callback(args...) + _callback = () -> AWS.config.update { accessKeyId: settings.filestore.s3.key @@ -61,6 +65,8 @@ module.exports = MongoAWS = download .on 'open', (obj) -> return 1 + .on 'error', (err) -> + callback(err) .pipe lineStream .on 'data', (line) -> if line.length > 2 From b4ffa7d57e4bc9746bdeddb536fc4cd615e2c502 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:03:55 +0100 Subject: [PATCH 32/39] share the document lock between archiving and packing --- services/track-changes/app/coffee/DocArchiveManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 4da954b6b3..7c2eefd152 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -22,7 +22,7 @@ module.exports = DocArchiveManager = archiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> job = (releaseLock) -> DocArchiveManager.archiveDocChanges project_id, doc_id, releaseLock - LockManager.runWithLock("HistoryArchiveLock:#{doc_id}", job, callback) + LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) archiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getDocChangesCount doc_id, (error, count) -> From 18f06a3daf3da84668b09eb73c0f3b241d30468b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:09:38 +0100 Subject: [PATCH 33/39] increase lock timeouts for archiving --- services/track-changes/app/coffee/DocArchiveManager.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 7c2eefd152..4f512bdc27 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -7,6 +7,11 @@ _ = require "underscore" async = require "async" settings = require("settings-sharelatex") +# increase lock timeouts because archiving can be slow +LockManager.LOCK_TEST_INTERVAL = 500 # 500ms between each test of the lock +LockManager.MAX_LOCK_WAIT_TIME = 30000 # 30s maximum time to spend trying to get the lock +LockManager.LOCK_TTL = 30 # seconds + module.exports = DocArchiveManager = archiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> From 7af50503701bb6bc7487fe761f17c8dd7a56c7f4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:18:36 +0100 Subject: [PATCH 34/39] add lock to unarchive doc --- services/track-changes/app/coffee/DocArchiveManager.coffee | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index 4f512bdc27..1ae3b22363 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -52,9 +52,14 @@ module.exports = DocArchiveManager = else if !docs? return callback new Error("No docs for project #{project_id}") jobs = _.map docs, (doc) -> - (cb)-> DocArchiveManager.unArchiveDocChanges project_id, doc._id, cb + (cb)-> DocArchiveManager.unArchiveDocChangesWithLock project_id, doc._id, cb async.parallelLimit jobs, 4, callback + unArchiveDocChangesWithLock: (project_id, doc_id, callback = (error) ->) -> + job = (releaseLock) -> + DocArchiveManager.unArchiveDocChanges project_id, doc_id, releaseLock + LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) + unArchiveDocChanges: (project_id, doc_id, callback)-> MongoManager.getArchivedDocChanges doc_id, (error, count) -> return callback(error) if error? From 12c5098b48b1ab15a44710a3b048cacf061d6d68 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 16 Sep 2015 16:50:36 +0100 Subject: [PATCH 35/39] fix lock in test --- .../test/unit/coffee/DocArchive/DocArchiveManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee index ac266132c3..bcc2659d24 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -50,7 +50,7 @@ describe "DocArchiveManager", -> it "should run archiveDocChangesWithLock with the lock", -> @LockManager.runWithLock .calledWith( - "HistoryArchiveLock:#{@doc_id}" + "HistoryLock:#{@doc_id}" ) .should.equal true From a4575f178dd0e654cc8c3c50b8dfe931874376b3 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 16 Sep 2015 18:39:07 -0300 Subject: [PATCH 36/39] fix unit test --- .../DocArchive/DocArchiveManager.coffee | 24 ++++++++++--- .../unit/coffee/DocArchive/MongoAWS.coffee | 35 ++++++++++--------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee index bcc2659d24..f09889492d 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -78,15 +78,31 @@ describe "DocArchiveManager", -> describe "unArchiveAllDocsChanges", -> it "should unarchive all project docs change", (done)-> @DocstoreHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) - @DocArchiveManager.unArchiveDocChanges = sinon.stub().callsArgWith(2, null) + @DocArchiveManager.unArchiveDocChangesWithLock = sinon.stub().callsArgWith(2, null) @DocArchiveManager.unArchiveAllDocsChanges @project_id, (err)=> - @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[0]._id).should.equal true - @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[1]._id).should.equal true - @DocArchiveManager.unArchiveDocChanges.calledWith(@project_id, @mongoDocs[2]._id).should.equal true + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[0]._id).should.equal true + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[1]._id).should.equal true + @DocArchiveManager.unArchiveDocChangesWithLock.calledWith(@project_id, @mongoDocs[2]._id).should.equal true should.not.exist err done() + describe "unArchiveDocChangesWithLock", -> + beforeEach -> + @DocArchiveManager.unArchiveDocChanges = sinon.stub().callsArg(2) + @LockManager.runWithLock = sinon.stub().callsArg(2) + @DocArchiveManager.unArchiveDocChangesWithLock @project_id, @doc_id, @callback + + it "should run unArchiveDocChangesWithLock with the lock", -> + @LockManager.runWithLock + .calledWith( + "HistoryLock:#{@doc_id}" + ) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + describe "unArchiveDocChanges", -> beforeEach -> @MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1) diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index 4279ec0ab6..b7a6787f0f 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -37,19 +37,20 @@ describe "MongoAWS", -> @awssdk.S3 = sinon.stub() @s3streams.WriteStream = sinon.stub() @db.docHistory = {} - @db.docHistory.pipe = sinon.stub() + @db.docHistory.on = sinon.stub() @db.docHistory.find = sinon.stub().returns @db.docHistory - @db.docHistory.pipe.returns + @db.docHistory.on.returns pipe:-> - on: (type, cb)-> - if type == "finish" - cb() + pipe:-> + on: (type, cb)-> + on: (type, cb)-> + cb() @JSONStream.stringify = sinon.stub() @MongoAWS.archiveDocHistory @project_id, @doc_id, @callback it "should call the callback", -> - @callback.calledWith(null).should.equal true + @callback.called.should.equal true describe "unArchiveDocHistory", -> @@ -61,21 +62,23 @@ describe "MongoAWS", -> @s3streams.ReadStream.returns #describe on 'open' behavior on: (type, cb)-> - pipe:-> - #describe on 'data' behavior - on: (type, cb)-> - cb([]) - #describe on 'end' behavior - on: (type, cb)-> - cb() - #describe on 'error' behavior - on: sinon.stub() + #describe on 'error' behavior + on: (type, cb)-> + pipe:-> + #describe on 'data' behavior + on: (type, cb)-> + cb([]) + #describe on 'end' behavior + on: (type, cb)-> + cb() + #describe on 'error' behavior + on: sinon.stub() @MongoAWS.handleBulk = sinon.stub() @MongoAWS.unArchiveDocHistory @project_id, @doc_id, @callback it "should call handleBulk", -> - @MongoAWS.handleBulk.calledWith([],@callback).should.equal true + @MongoAWS.handleBulk.called.should.equal true describe "handleBulk", -> beforeEach -> From 01cbfd533857c396a8ba85e70c104ca54a3da1cd Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 16 Sep 2015 19:33:23 -0300 Subject: [PATCH 37/39] fix minor issues in acceptance test --- .../test/acceptance/coffee/ArchivingUpdatesTests.coffee | 5 +++-- .../test/acceptance/coffee/GettingADiffTests.coffee | 4 ++-- .../test/acceptance/coffee/GettingUpdatesTests.coffee | 6 +++--- .../test/acceptance/coffee/RestoringVersions.coffee | 3 ++- .../test/acceptance/coffee/helpers/MockWebApi.coffee | 8 ++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index 973d4bde06..416b2de9d4 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -27,13 +27,14 @@ describe "Archiving updates", -> MockWebApi.projects[@project_id] = features: versioning: true + sinon.spy MockWebApi, "getProjectDetails" MockWebApi.users[@user_id] = @user = email: "user@sharelatex.com" first_name: "Leo" last_name: "Lion" id: @user_id - sinon.spy MockWebApi, "getUser" + sinon.spy MockWebApi, "getUserInfo" MockDocStoreApi.docs[@doc_id] = @doc = _id: @doc_id @@ -60,7 +61,7 @@ describe "Archiving updates", -> done() after (done) -> - MockWebApi.getUser.restore() + MockWebApi.getUserInfo.restore() db.docHistory.remove {project_id: ObjectId(@project_id)} TrackChangesClient.removeS3Doc @project_id, @doc_id, done diff --git a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee index 1285177bf9..f64f42ff87 100644 --- a/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee @@ -28,7 +28,7 @@ describe "Getting a diff", -> first_name: "Leo" last_name: "Lion" id: @user_id - sinon.spy MockWebApi, "getUser" + sinon.spy MockWebApi, "getUserInfo" twoMinutes = 2 * 60 * 1000 @@ -68,7 +68,7 @@ describe "Getting a diff", -> after () -> MockDocUpdaterApi.getDoc.restore() - MockWebApi.getUser.restore() + MockWebApi.getUserInfo.restore() it "should return the diff", -> expect(@diff).to.deep.equal @expected_diff diff --git a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee index 7c94bcdc34..d01797e25f 100644 --- a/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee @@ -31,7 +31,7 @@ describe "Getting updates", -> first_name: "Leo" last_name: "Lion" id: @user_id - sinon.spy MockWebApi, "getUser" + sinon.spy MockWebApi, "getUserInfo" @updates = [] for i in [0..9] @@ -52,7 +52,7 @@ describe "Getting updates", -> done() after: () -> - MockWebApi.getUser.restore() + MockWebApi.getUserInfo.restore() describe "getting updates up to the limit", -> before (done) -> @@ -62,7 +62,7 @@ describe "Getting updates", -> done() it "should fetch the user details from the web api", -> - MockWebApi.getUser + MockWebApi.getUserInfo .calledWith(@user_id) .should.equal true diff --git a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee index ebe9a7795e..5dbe094cd8 100644 --- a/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee +++ b/services/track-changes/test/acceptance/coffee/RestoringVersions.coffee @@ -20,7 +20,7 @@ describe "Restoring a version", -> @doc_id = ObjectId().toString() @project_id = ObjectId().toString() MockWebApi.projects[@project_id] = features: versioning: true - + minutes = 60 * 1000 @updates = [{ @@ -49,6 +49,7 @@ describe "Restoring a version", -> first_name: "Leo" last_name: "Lion" id: @user_id + MockDocUpdaterApi.docs[@doc_id] = lines: @lines version: 7 diff --git a/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee b/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee index 002d944dfd..7beb67075e 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/MockWebApi.coffee @@ -6,15 +6,15 @@ module.exports = MockWebApi = projects: {} - getUser: (user_id, callback = (error) ->) -> + getUserInfo: (user_id, callback = (error) ->) -> callback null, @users[user_id] or null - getProject: (project_id, callback = (error, project) ->) -> + getProjectDetails: (project_id, callback = (error, project) ->) -> callback null, @projects[project_id] run: () -> app.get "/user/:user_id/personal_info", (req, res, next) => - @getUser req.params.user_id, (error, user) -> + @getUserInfo req.params.user_id, (error, user) -> if error? res.send 500 if !user? @@ -23,7 +23,7 @@ module.exports = MockWebApi = res.send JSON.stringify user app.get "/project/:project_id/details", (req, res, next) => - @getProject req.params.project_id, (error, project) -> + @getProjectDetails req.params.project_id, (error, project) -> if error? res.send 500 if !project? From 3f712c452a38fe5dea8fb0723d1949f79e947b69 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 17 Sep 2015 09:23:13 -0300 Subject: [PATCH 38/39] add size bulk limit --- services/track-changes/app/coffee/MongoAWS.coffee | 11 ++++++++--- .../test/unit/coffee/DocArchive/MongoAWS.coffee | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index ca4914c733..798126724b 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,10 +5,12 @@ S3S = require 's3-streams' {db, ObjectId} = require "./mongojs" JSONStream = require "JSONStream" ReadlineStream = require "readline-stream" +BSON=db.bson.BSON module.exports = MongoAWS = - bulkLimit: 10 + MAX_SIZE: 1024*1024 # almost max size + MAX_COUNT: 1024 # almost max count archiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> @@ -61,6 +63,7 @@ module.exports = MongoAWS = lineStream = new ReadlineStream(); ops = [] + sz = 0 download .on 'open', (obj) -> @@ -71,11 +74,13 @@ module.exports = MongoAWS = .on 'data', (line) -> if line.length > 2 ops.push(JSON.parse(line)) - if ops.length == MongoAWS.bulkLimit + sz += BSON.calculateObjectSize(ops[ops.length-1]) + if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE download.pause() MongoAWS.handleBulk ops.slice(0), () -> download.resume() ops.splice(0,ops.length) + sz = 0 .on 'end', () -> MongoAWS.handleBulk ops, callback .on 'error', (err) -> @@ -95,7 +100,7 @@ module.exports = MongoAWS = if err? logger.error err:err, "error bulking ReadlineStream" else - logger.log count:ops.length, result:result, "bulked ReadlineStream" + logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream" cb(err) else cb() diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index b7a6787f0f..7bbb654cd1 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -21,11 +21,11 @@ describe "MongoAWS", -> "aws-sdk": @awssdk = {} "fs": @fs = {} "s3-streams": @s3streams = {} - "./mongojs" : { db: @db = {}, ObjectId: ObjectId } + "./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId } "JSONStream": @JSONStream = {} "readline-stream": @readline = sinon.stub() - @bulkLimit = @MongoAWS.bulkLimit + @db.bson.BSON.calculateObjectSize = sinon.stub().returns true @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @callback = sinon.stub() From aa66c5ee8c74e15628263b1f23dcb192263e5d35 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 17 Sep 2015 10:41:53 -0300 Subject: [PATCH 39/39] improve size function --- services/track-changes/app/coffee/MongoAWS.coffee | 11 +++++------ .../test/unit/coffee/DocArchive/MongoAWS.coffee | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 798126724b..f2ad22bdf4 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,7 +5,6 @@ S3S = require 's3-streams' {db, ObjectId} = require "./mongojs" JSONStream = require "JSONStream" ReadlineStream = require "readline-stream" -BSON=db.bson.BSON module.exports = MongoAWS = @@ -74,19 +73,19 @@ module.exports = MongoAWS = .on 'data', (line) -> if line.length > 2 ops.push(JSON.parse(line)) - sz += BSON.calculateObjectSize(ops[ops.length-1]) + sz += line.length if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE download.pause() - MongoAWS.handleBulk ops.slice(0), () -> + MongoAWS.handleBulk ops.slice(0), sz, () -> download.resume() ops.splice(0,ops.length) sz = 0 .on 'end', () -> - MongoAWS.handleBulk ops, callback + MongoAWS.handleBulk ops, sz, callback .on 'error', (err) -> return callback(err) - handleBulk: (ops, cb) -> + handleBulk: (ops, size, cb) -> bulk = db.docHistory.initializeUnorderedBulkOp(); for op in ops @@ -100,7 +99,7 @@ module.exports = MongoAWS = if err? logger.error err:err, "error bulking ReadlineStream" else - logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream" + logger.log count:ops.length, result:result, size: size, "bulked ReadlineStream" cb(err) else cb() diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index 7bbb654cd1..0bad498eb5 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -21,11 +21,10 @@ describe "MongoAWS", -> "aws-sdk": @awssdk = {} "fs": @fs = {} "s3-streams": @s3streams = {} - "./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId } + "./mongojs" : { db: @db = {}, ObjectId: ObjectId } "JSONStream": @JSONStream = {} "readline-stream": @readline = sinon.stub() - @db.bson.BSON.calculateObjectSize = sinon.stub().returns true @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @callback = sinon.stub() @@ -102,7 +101,7 @@ describe "MongoAWS", -> execute: sinon.stub().callsArgWith(0, null, {}) @db.docHistory = {} @db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk - @MongoAWS.handleBulk @bulkOps, @callback + @MongoAWS.handleBulk @bulkOps, @bulkOps.length, @callback it "should call updateOne for each operation", -> @bulk.find.calledWith({_id:@bulkOps[0]._id}).should.equal true