From 314c7350042848a8a721199899fa5f5a2119e382 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 1 Jun 2015 18:22:02 -0300 Subject: [PATCH 01/21] initial version for archive docs in S3 --- services/docstore/app.coffee | 1 + .../docstore/app/coffee/DocManager.coffee | 35 ++++++++++++++++++ .../docstore/app/coffee/HttpController.coffee | 7 ++++ .../docstore/config/settings.defaults.coffee | 8 +++++ .../acceptance/coffee/ArchiveDocsTests.coffee | 36 +++++++++++++++++++ .../coffee/helpers/DocstoreClient.coffee | 14 +++++--- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index ec9b250851..8609d79e8f 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -20,6 +20,7 @@ app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc +app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> res.send('docstore is alive') diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index ef03969a30..57184d8844 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -3,6 +3,10 @@ Errors = require "./Errors" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 module.exports = DocManager = @@ -58,3 +62,34 @@ module.exports = DocManager = return callback(error) if error? callback() + archiveAllDocs: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + + jobs = for doc in docs + do (doc) => + (cb) => + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving file in aws" + cb(err) + + async.series jobs, callback + +buildS3Options = (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 2a974794b2..1c790ac9ba 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -78,3 +78,10 @@ module.exports = HttpController = _buildRawDocView: (doc)-> return (doc?.lines or []).join("\n") + + archiveAllDocs: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "archiving all docs" + DocManager.archiveAllDocs project_id, (error) -> + return next(error) if error? + res.send 204 diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index 8a3fc9197f..d708b56ce2 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -9,3 +9,11 @@ module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' + + #filestore: + # backend: "s3" + # stores: + # user_files: "" + # s3: + # key: "" + # secret: "" \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee new file mode 100644 index 0000000000..f55e39099c --- /dev/null +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -0,0 +1,36 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +{ObjectId} = require "mongojs" +async = require "async" + +DocstoreClient = require "./helpers/DocstoreClient" + +describe "Archiving all docs", -> + beforeEach (done) -> + @project_id = ObjectId() + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + }] + jobs = for doc in @docs + do (doc) => + (callback) => + DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> + doc.lines[0] = doc.lines[0]+" added" + DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback + async.series jobs, done + + it "should archive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + res.statusCode.should.equal 204 + done() \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 1d2bcadc75..989a2ce0e7 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -1,5 +1,6 @@ request = require("request").defaults(jar: false) {db, ObjectId} = require("../../../../app/js/mongojs") +settings = require("settings-sharelatex") module.exports = DocstoreClient = @@ -16,28 +17,31 @@ module.exports = DocstoreClient = getDoc: (project_id, doc_id, qs, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: true qs:qs }, callback getAllDocs: (project_id, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc" json: true }, callback updateDoc: (project_id, doc_id, lines, callback = (error, res, body) ->) -> request.post { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: lines: lines }, callback deleteDoc: (project_id, doc_id, callback = (error, res, body) ->) -> request.del { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" }, callback - + archiveAllDoc: (project_id, callback = (error, res, body) ->) -> + request.get { + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive" + }, callback From 7f5204f0f79110852e856e2bd7458b62a05a4175 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 1 Jun 2015 18:24:40 -0300 Subject: [PATCH 02/21] initial version for archive docs in S3 --- services/docstore/app.coffee | 1 + .../docstore/app/coffee/DocManager.coffee | 35 ++++++++++++++++++ .../docstore/app/coffee/HttpController.coffee | 7 ++++ .../docstore/config/settings.defaults.coffee | 8 +++++ .../acceptance/coffee/ArchiveDocsTests.coffee | 36 +++++++++++++++++++ .../coffee/helpers/DocstoreClient.coffee | 14 +++++--- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index ec9b250851..8609d79e8f 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -20,6 +20,7 @@ app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc +app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> res.send('docstore is alive') diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index ef03969a30..57184d8844 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -3,6 +3,10 @@ Errors = require "./Errors" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 module.exports = DocManager = @@ -58,3 +62,34 @@ module.exports = DocManager = return callback(error) if error? callback() + archiveAllDocs: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + + jobs = for doc in docs + do (doc) => + (cb) => + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving file in aws" + cb(err) + + async.series jobs, callback + +buildS3Options = (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 2a974794b2..1c790ac9ba 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -78,3 +78,10 @@ module.exports = HttpController = _buildRawDocView: (doc)-> return (doc?.lines or []).join("\n") + + archiveAllDocs: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "archiving all docs" + DocManager.archiveAllDocs project_id, (error) -> + return next(error) if error? + res.send 204 diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index 8a3fc9197f..d708b56ce2 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -9,3 +9,11 @@ module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' + + #filestore: + # backend: "s3" + # stores: + # user_files: "" + # s3: + # key: "" + # secret: "" \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee new file mode 100644 index 0000000000..f55e39099c --- /dev/null +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -0,0 +1,36 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +{ObjectId} = require "mongojs" +async = require "async" + +DocstoreClient = require "./helpers/DocstoreClient" + +describe "Archiving all docs", -> + beforeEach (done) -> + @project_id = ObjectId() + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + }] + jobs = for doc in @docs + do (doc) => + (callback) => + DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> + doc.lines[0] = doc.lines[0]+" added" + DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback + async.series jobs, done + + it "should archive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + res.statusCode.should.equal 204 + done() \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 1d2bcadc75..989a2ce0e7 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -1,5 +1,6 @@ request = require("request").defaults(jar: false) {db, ObjectId} = require("../../../../app/js/mongojs") +settings = require("settings-sharelatex") module.exports = DocstoreClient = @@ -16,28 +17,31 @@ module.exports = DocstoreClient = getDoc: (project_id, doc_id, qs, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: true qs:qs }, callback getAllDocs: (project_id, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc" json: true }, callback updateDoc: (project_id, doc_id, lines, callback = (error, res, body) ->) -> request.post { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: lines: lines }, callback deleteDoc: (project_id, doc_id, callback = (error, res, body) ->) -> request.del { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" }, callback - + archiveAllDoc: (project_id, callback = (error, res, body) ->) -> + request.get { + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive" + }, callback From 7c7ff649043d5e26284afed108c8f5396b207c4a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 1 Jun 2015 19:36:26 -0300 Subject: [PATCH 03/21] retrieving all docs from s3 --- .../docstore/app/coffee/DocManager.coffee | 48 +++++++++++++++---- .../docstore/app/coffee/MongoManager.coffee | 11 +++++ .../docstore/config/settings.defaults.coffee | 3 +- .../acceptance/coffee/ArchiveDocsTests.coffee | 12 ++++- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 57184d8844..205128a72f 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -19,13 +19,14 @@ module.exports = DocManager = callback null, doc getAllDocs: (project_id, callback = (error, docs) ->) -> - MongoManager.getProjectsDocs project_id, (error, docs) -> - if err? - return callback(error) - else if !docs? - return callback new Errors.NotFoundError("No docs for project #{project_id}") - else - return callback(null, docs) + DocManager.unArchiveAllDocs project_id, (error) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + else + return callback(null, docs) updateDoc: (project_id, doc_id, lines, callback = (error, modified, rev) ->) -> MongoManager.findDoc doc_id, (err, doc)-> @@ -76,11 +77,40 @@ module.exports = DocManager = options = buildS3Options(doc.lines, project_id+"/"+doc._id) request.put options, (err, res)-> if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving file in aws" - cb(err) + logger.err err:err, res:res, "something went wrong archiving doc in aws" + cb(err) + MongoManager.markDocAsArchived doc._id, (error) -> + return cb(error) if error? + cb() async.series jobs, callback + unArchiveAllDocs: (project_id, callback = (error) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + + jobs = for doc in docs + do (doc) => + if !doc.inS3? + (cb) => cb() + else + (cb) => + logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" + options = buildS3Options(true, project_id+"/"+doc._id) + request.get options, (err, res, lines)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong unarchiving doc from aws" + cb(err) + MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> + return cb(error) if error? + cb() + + async.series jobs, callback + + buildS3Options = (content, key)-> return { aws: diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index 352483fbfc..ef73297b90 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -13,9 +13,11 @@ module.exports = MongoManager = update = $set:{} $inc:{} + $unset:{} update.$set["lines"] = lines update.$set["project_id"] = ObjectId(project_id) update.$inc["rev"] = 1 #on new docs being created this will set the rev to 1 + update.$unset["inS3"] = true db.docs.update _id: ObjectId(doc_id), update, {upsert: true}, callback @@ -25,3 +27,12 @@ module.exports = MongoManager = update.$set["deleted"] = true db.docs.update _id: ObjectId(doc_id), update, (err)-> callback(err) + + markDocAsArchived: (doc_id, callback)-> + update = + $set: {} + $unset: {} + update.$set["inS3"] = true + update.$unset["lines"] = true + db.docs.update _id: doc_id, update, (err)-> + callback(err) \ No newline at end of file diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index d708b56ce2..4a16b05c2a 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -16,4 +16,5 @@ module.exports = # user_files: "" # s3: # key: "" - # secret: "" \ No newline at end of file + # secret: "" + \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index f55e39099c..9569f97cda 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -33,4 +33,14 @@ describe "Archiving all docs", -> it "should archive all the docs", (done) -> DocstoreClient.archiveAllDoc @project_id, (error, res) => res.statusCode.should.equal 204 - done() \ No newline at end of file + done() + + it "should unarchive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + DocstoreClient.getAllDocs @project_id, (error, res, docs) => + throw error if error? + docs.length.should.equal @docs.length + for doc, i in docs + doc.lines.should.deep.equal @docs[i].lines + done() + From 19f1a9ad82457eea53936f3e49aee3dd1980f3af Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jun 2015 10:59:35 +0100 Subject: [PATCH 04/21] changed style of async jobs to map to use underscore which is like we use in the rest of the stack. Also removed a few => binds which are not needed --- .../docstore/app/coffee/DocManager.coffee | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 205128a72f..aae71bc2d3 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -70,18 +70,17 @@ module.exports = DocManager = else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = for doc in docs - do (doc) => - (cb) => - logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" - options = buildS3Options(doc.lines, project_id+"/"+doc._id) - request.put options, (err, res)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving doc in aws" - cb(err) - MongoManager.markDocAsArchived doc._id, (error) -> - return cb(error) if error? - cb() + jobs = _.map docs, (doc) -> + (cb)-> + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving doc in aws" + cb(err) + MongoManager.markDocAsArchived doc._id, (error) -> + return cb(error) if error? + cb() async.series jobs, callback @@ -92,21 +91,20 @@ module.exports = DocManager = else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = for doc in docs - do (doc) => + jobs = _.map docs, (doc) -> + (cb)-> if !doc.inS3? - (cb) => cb() + return cb() else - (cb) => - logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" - options = buildS3Options(true, project_id+"/"+doc._id) - request.get options, (err, res, lines)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong unarchiving doc from aws" - cb(err) - MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> - return cb(error) if error? - cb() + logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" + options = buildS3Options(true, project_id+"/"+doc._id) + request.get options, (err, res, lines)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong unarchiving doc from aws" + cb(err) + MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> + return cb(error) if error? + cb() async.series jobs, callback From 60bc7bf4e1ca77329faf5c9a5bbab4ca6e6c47b2 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jun 2015 12:36:38 +0100 Subject: [PATCH 05/21] pulled logic for arching docs arpart a bit, can be moved into its own file hacked out (needs tests) the unarchiving of individual docs & doc updates --- .../docstore/app/coffee/DocManager.coffee | 61 +++++++++++-------- .../docstore/app/coffee/MongoManager.coffee | 3 + 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index aae71bc2d3..a89d83e11c 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -16,7 +16,13 @@ module.exports = DocManager = return callback(err) else if !doc? return callback new Errors.NotFoundError("No such doc: #{doc_id} in project #{project_id}") - callback null, doc + else if doc?.inS3 + DocManager.unarchiveDoc project_id, doc_id, (err)-> + if err? + return callback(err) + MongoManager.findDoc doc_id, callback + else + callback err, doc getAllDocs: (project_id, callback = (error, docs) ->) -> DocManager.unArchiveAllDocs project_id, (error) -> @@ -29,8 +35,8 @@ module.exports = DocManager = return callback(null, docs) updateDoc: (project_id, doc_id, lines, callback = (error, modified, rev) ->) -> - MongoManager.findDoc doc_id, (err, doc)-> - if err? + DocManager.getDoc doc_id, (err, doc)-> + if err? and !(error instanceof Errors.NotFoundError) logger.err project_id: project_id, doc_id: doc_id, err:err, "error getting document for update" return callback(err) @@ -63,51 +69,54 @@ module.exports = DocManager = return callback(error) if error? callback() + + #DOC ARCHIVER archiveAllDocs: (project_id, callback = (error, docs) ->) -> MongoManager.getProjectsDocs project_id, (error, docs) -> if err? return callback(error) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = _.map docs, (doc) -> - (cb)-> - logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" - options = buildS3Options(doc.lines, project_id+"/"+doc._id) - request.put options, (err, res)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving doc in aws" - cb(err) - MongoManager.markDocAsArchived doc._id, (error) -> - return cb(error) if error? - cb() - + (cb)-> DocManager.archiveDoc project_id, doc, cb async.series jobs, callback + + archiveDoc: (project_id, doc, callback)-> + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving doc in aws" + callback(err) + MongoManager.markDocAsArchived doc._id, (error) -> + return callback(error) if error? + callback() + unArchiveAllDocs: (project_id, callback = (error) ->) -> MongoManager.getProjectsDocs project_id, (error, docs) -> if err? return callback(error) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = _.map docs, (doc) -> (cb)-> if !doc.inS3? return cb() else - logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" - options = buildS3Options(true, project_id+"/"+doc._id) - request.get options, (err, res, lines)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong unarchiving doc from aws" - cb(err) - MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> - return cb(error) if error? - cb() - + unarchiveDoc project_id, doc_id, cb async.series jobs, callback + unarchiveDoc: (project_id, doc_id, callback)-> + logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" + options = buildS3Options(true, project_id+"/"+doc._id) + request.get options, (err, res, lines)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong unarchiving doc from aws" + callback(err) + MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> + return callback(error) if error? + callback() buildS3Options = (content, key)-> return { diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index ef73297b90..1f8b9ca6d8 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -34,5 +34,8 @@ module.exports = MongoManager = $unset: {} update.$set["inS3"] = true update.$unset["lines"] = true + # to ensure that the lines have not changed during the archive process + # what if we passed the lines through into the query {_id: doc_id, lines:lines} + # or more performant would be todo the rev {_id:doc_id, rev:rev} db.docs.update _id: doc_id, update, (err)-> callback(err) \ No newline at end of file From ea92f57acc7a9c80636f44b550b4a6ff452ea694 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jun 2015 12:39:49 +0100 Subject: [PATCH 06/21] point where we unset doc lines also search on doc rev to make sure the document has not been updated --- services/docstore/app/coffee/DocManager.coffee | 2 +- services/docstore/app/coffee/MongoManager.coffee | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index a89d83e11c..61cd4a1be4 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -89,7 +89,7 @@ module.exports = DocManager = if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong archiving doc in aws" callback(err) - MongoManager.markDocAsArchived doc._id, (error) -> + MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> return callback(error) if error? callback() diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index 1f8b9ca6d8..f33e5995ef 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -28,14 +28,12 @@ module.exports = MongoManager = db.docs.update _id: ObjectId(doc_id), update, (err)-> callback(err) - markDocAsArchived: (doc_id, callback)-> + markDocAsArchived: (doc_id, rev, callback)-> update = $set: {} $unset: {} update.$set["inS3"] = true update.$unset["lines"] = true - # to ensure that the lines have not changed during the archive process - # what if we passed the lines through into the query {_id: doc_id, lines:lines} - # or more performant would be todo the rev {_id:doc_id, rev:rev} + # to ensure that the lines have not changed during the archive process we search via the rev db.docs.update _id: doc_id, update, (err)-> callback(err) \ No newline at end of file From 86f16caeefee4abf64214a9d4493149b58556f13 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 15:13:28 -0300 Subject: [PATCH 07/21] fix some issues --- services/docstore/app/coffee/DocManager.coffee | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 61cd4a1be4..df52a2a16b 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -35,8 +35,8 @@ module.exports = DocManager = return callback(null, docs) updateDoc: (project_id, doc_id, lines, callback = (error, modified, rev) ->) -> - DocManager.getDoc doc_id, (err, doc)-> - if err? and !(error instanceof Errors.NotFoundError) + DocManager.getDoc project_id, doc_id, (err, doc)-> + if err? and !(err instanceof Errors.NotFoundError) logger.err project_id: project_id, doc_id: doc_id, err:err, "error getting document for update" return callback(err) @@ -104,17 +104,17 @@ module.exports = DocManager = if !doc.inS3? return cb() else - unarchiveDoc project_id, doc_id, cb + DocManager.unarchiveDoc project_id, doc._id, cb async.series jobs, callback unarchiveDoc: (project_id, doc_id, callback)-> - logger.log project_id: project_id, doc_id: doc._id, "getting doc from s3" - options = buildS3Options(true, project_id+"/"+doc._id) + logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3" + options = buildS3Options(true, project_id+"/"+doc_id) request.get options, (err, res, lines)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong unarchiving doc from aws" callback(err) - MongoManager.upsertIntoDocCollection project_id, doc._id.toString(), lines, (error) -> + MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> return callback(error) if error? callback() From ddd26798f18616970c0475b829eef121da9f55aa Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 15:55:22 -0300 Subject: [PATCH 08/21] refactoring s3 filter in router --- services/docstore/app.coffee | 4 +- .../docstore/app/coffee/DocArchive.coffee | 71 +++++++++++++++++++ .../docstore/app/coffee/DocManager.coffee | 67 +---------------- .../docstore/app/coffee/HttpController.coffee | 3 +- .../docstore/app/coffee/MongoManager.coffee | 12 +++- 5 files changed, 89 insertions(+), 68 deletions(-) create mode 100644 services/docstore/app/coffee/DocArchive.coffee diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index 8609d79e8f..5f45f5a8c3 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -20,7 +20,9 @@ app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc -app.get '/project/:project_id/archive', HttpController.archiveAllDocs + +if Settings.filestore.backend == "s3" + app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> res.send('docstore is alive') diff --git a/services/docstore/app/coffee/DocArchive.coffee b/services/docstore/app/coffee/DocArchive.coffee new file mode 100644 index 0000000000..25f4d02506 --- /dev/null +++ b/services/docstore/app/coffee/DocArchive.coffee @@ -0,0 +1,71 @@ +MongoManager = require "./MongoManager" +Errors = require "./Errors" +logger = require "logger-sharelatex" +_ = require "underscore" +async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 + +module.exports = DocArchive = + + archiveAllDocs: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> DocArchive.archiveDoc project_id, doc, cb + async.series jobs, callback + + + archiveDoc: (project_id, doc, callback)-> + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving doc in aws" + callback(err) + MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> + return callback(error) if error? + callback() + + unArchiveAllDocs: (project_id, callback = (error) ->) -> + MongoManager.getArchivedProjectDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + jobs = _.map docs, (doc) -> + (cb)-> + if !doc.inS3? + return cb() + else + DocArchive.unarchiveDoc project_id, doc._id, cb + async.series jobs, callback + + unarchiveDoc: (project_id, doc_id, callback)-> + logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3" + options = buildS3Options(true, project_id+"/"+doc_id) + request.get options, (err, res, lines)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong unarchiving doc from aws" + callback(err) + MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> + return callback(error) if error? + callback() + +buildS3Options = (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } \ No newline at end of file diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index df52a2a16b..5ffad57b9d 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -7,6 +7,7 @@ settings = require("settings-sharelatex") request = require("request") crypto = require("crypto") thirtySeconds = 30 * 1000 +DocArchive = require "./DocArchive" module.exports = DocManager = @@ -17,7 +18,7 @@ module.exports = DocManager = else if !doc? return callback new Errors.NotFoundError("No such doc: #{doc_id} in project #{project_id}") else if doc?.inS3 - DocManager.unarchiveDoc project_id, doc_id, (err)-> + DocArchive.unarchiveDoc project_id, doc_id, (err)-> if err? return callback(err) MongoManager.findDoc doc_id, callback @@ -25,7 +26,7 @@ module.exports = DocManager = callback err, doc getAllDocs: (project_id, callback = (error, docs) ->) -> - DocManager.unArchiveAllDocs project_id, (error) -> + DocArchive.unArchiveAllDocs project_id, (error) -> MongoManager.getProjectsDocs project_id, (error, docs) -> if err? return callback(error) @@ -68,65 +69,3 @@ module.exports = DocManager = MongoManager.markDocAsDeleted doc_id, (error) -> return callback(error) if error? callback() - - - #DOC ARCHIVER - archiveAllDocs: (project_id, callback = (error, docs) ->) -> - MongoManager.getProjectsDocs project_id, (error, docs) -> - if err? - return callback(error) - else if !docs? - return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = _.map docs, (doc) -> - (cb)-> DocManager.archiveDoc project_id, doc, cb - async.series jobs, callback - - - archiveDoc: (project_id, doc, callback)-> - logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" - options = buildS3Options(doc.lines, project_id+"/"+doc._id) - request.put options, (err, res)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving doc in aws" - callback(err) - MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> - return callback(error) if error? - callback() - - unArchiveAllDocs: (project_id, callback = (error) ->) -> - MongoManager.getProjectsDocs project_id, (error, docs) -> - if err? - return callback(error) - else if !docs? - return callback new Errors.NotFoundError("No docs for project #{project_id}") - jobs = _.map docs, (doc) -> - (cb)-> - if !doc.inS3? - return cb() - else - DocManager.unarchiveDoc project_id, doc._id, cb - async.series jobs, callback - - unarchiveDoc: (project_id, doc_id, callback)-> - logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3" - options = buildS3Options(true, project_id+"/"+doc_id) - request.get options, (err, res, lines)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong unarchiving doc from aws" - callback(err) - MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> - return callback(error) if error? - callback() - -buildS3Options = (content, key)-> - return { - aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: settings.filestore.stores.user_files - timeout: thirtySeconds - json: content - #headers: - # 'content-md5': crypto.createHash("md5").update(content).digest("hex") - uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" - } diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 1c790ac9ba..86d2dec470 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -1,5 +1,6 @@ DocManager = require "./DocManager" logger = require "logger-sharelatex" +DocArchive = require "./DocArchive" module.exports = HttpController = getDoc: (req, res, next = (error) ->) -> @@ -82,6 +83,6 @@ module.exports = HttpController = archiveAllDocs: (req, res, next = (error) ->) -> project_id = req.params.project_id logger.log project_id: project_id, "archiving all docs" - DocManager.archiveAllDocs project_id, (error) -> + DocArchive.archiveAllDocs project_id, (error) -> return next(error) if error? res.send 204 diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index f33e5995ef..b454214ff1 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -9,6 +9,12 @@ module.exports = MongoManager = getProjectsDocs: (project_id, callback)-> db.docs.find project_id: ObjectId(project_id.toString()), {}, callback + getArchivedProjectDocs: (project_id, callback)-> + query = + project_id: ObjectId(project_id.toString()) + inS3: true + db.docs.find query, {}, callback + upsertIntoDocCollection: (project_id, doc_id, lines, callback)-> update = $set:{} @@ -34,6 +40,8 @@ module.exports = MongoManager = $unset: {} update.$set["inS3"] = true update.$unset["lines"] = true - # to ensure that the lines have not changed during the archive process we search via the rev - db.docs.update _id: doc_id, update, (err)-> + query = + _id: doc_id + rev: rev + db.docs.update query, update, (err)-> callback(err) \ No newline at end of file From 5c31e8006286a130c37cbec721166fa656299a9b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 16:29:32 -0300 Subject: [PATCH 09/21] more refactoring --- services/docstore/app.coffee | 2 +- ...rchive.coffee => DocArchiveManager.coffee} | 0 .../docstore/app/coffee/DocManager.coffee | 2 +- .../docstore/app/coffee/HttpController.coffee | 2 +- .../acceptance/coffee/ArchiveDocsTests.coffee | 71 ++++++++++--------- 5 files changed, 40 insertions(+), 37 deletions(-) rename services/docstore/app/coffee/{DocArchive.coffee => DocArchiveManager.coffee} (100%) diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index 5f45f5a8c3..3c3a35d213 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -21,7 +21,7 @@ app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc -if Settings.filestore.backend == "s3" +if Settings.filestore?.backend == "s3" app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> diff --git a/services/docstore/app/coffee/DocArchive.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee similarity index 100% rename from services/docstore/app/coffee/DocArchive.coffee rename to services/docstore/app/coffee/DocArchiveManager.coffee diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 5ffad57b9d..85beac7031 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -7,7 +7,7 @@ settings = require("settings-sharelatex") request = require("request") crypto = require("crypto") thirtySeconds = 30 * 1000 -DocArchive = require "./DocArchive" +DocArchive = require "./DocArchiveManager" module.exports = DocManager = diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 86d2dec470..77d129811d 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -1,6 +1,6 @@ DocManager = require "./DocManager" logger = require "logger-sharelatex" -DocArchive = require "./DocArchive" +DocArchive = require "./DocArchiveManager" module.exports = HttpController = getDoc: (req, res, next = (error) ->) -> diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 9569f97cda..7b886bec9c 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -3,44 +3,47 @@ chai = require("chai") chai.should() {ObjectId} = require "mongojs" async = require "async" +Settings = require("settings-sharelatex") DocstoreClient = require "./helpers/DocstoreClient" -describe "Archiving all docs", -> - beforeEach (done) -> - @project_id = ObjectId() - @docs = [{ - _id: ObjectId() - lines: ["one", "two", "three"] - rev: 2 - }, { - _id: ObjectId() - lines: ["aaa", "bbb", "ccc"] - rev: 4 - }, { - _id: ObjectId() - lines: ["111", "222", "333"] - rev: 6 - }] - jobs = for doc in @docs - do (doc) => - (callback) => - DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> - doc.lines[0] = doc.lines[0]+" added" - DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback - async.series jobs, done +if Settings.filestore?.backend == "s3" - it "should archive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - res.statusCode.should.equal 204 - done() + describe "Archiving all docs", -> + beforeEach (done) -> + @project_id = ObjectId() + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + }] + jobs = for doc in @docs + do (doc) => + (callback) => + DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> + doc.lines[0] = doc.lines[0]+" added" + DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback + async.series jobs, done - it "should unarchive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - DocstoreClient.getAllDocs @project_id, (error, res, docs) => - throw error if error? - docs.length.should.equal @docs.length - for doc, i in docs - doc.lines.should.deep.equal @docs[i].lines + it "should archive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + res.statusCode.should.equal 204 done() + it "should unarchive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + DocstoreClient.getAllDocs @project_id, (error, res, docs) => + throw error if error? + docs.length.should.equal @docs.length + for doc, i in docs + doc.lines.should.deep.equal @docs[i].lines + done() + From 6add9a0da75706d9c4e9da1deba60d40034be21d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 17:12:11 -0300 Subject: [PATCH 10/21] init unit tests --- .../test/unit/coffee/DocArchiveManager.coffee | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 services/docstore/test/unit/coffee/DocArchiveManager.coffee diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee new file mode 100644 index 0000000000..f00fb11c69 --- /dev/null +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -0,0 +1,70 @@ +assert = require("chai").assert +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../app/js/DocArchiveManager.js" +SandboxedModule = require('sandboxed-module') +ObjectId = require("mongojs").ObjectId +Errors = require "../../../app/js/Errors" + +describe "DocArchiveManager", -> + + beforeEach -> + + @settings = + filestore: + backend: "s3" + s3: + secret: "secret" + key: "this_key" + stores: + user_files:"sl_user_files" + + @request = + put: sinon.stub().callsArgWith(1, null, statusCode:200) + + @MongoManager = + markDocAsArchived: sinon.stub().callsArgWith(2, null) + upsertIntoDocCollection: sinon.stub() + + @DocArchiveManager = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings + "./MongoManager": @MongoManager + "request": @request + "logger-sharelatex": + log:-> + err:-> + + @key = "my/key" + @bucketName = "my-bucket" + @error = "my errror" + + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + }] + + @project_id = ObjectId().toString() + @callback = sinon.stub() + @stubbedError = new Error("blew up") + + describe "archiveDoc", -> + + it "should use correct options", (done)-> + @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + opts = @request.put.args[0][0] + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@settings.filestore.stores.user_files}) + opts.json.should.equal @docs[0].lines + opts.timeout.should.equal (30*1000) + opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" + done() \ No newline at end of file From 0cd21204300a71fb59c6f00710d4c5ea899eb06f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 18:13:16 -0300 Subject: [PATCH 11/21] more unit tests for archive feature --- .../app/coffee/DocArchiveManager.coffee | 8 +- .../test/unit/coffee/DocArchiveManager.coffee | 149 +++++++++++++++--- .../test/unit/coffee/DocManagerTests.coffee | 3 + 3 files changed, 133 insertions(+), 27 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 25f4d02506..8c24237983 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -12,7 +12,7 @@ module.exports = DocArchive = archiveAllDocs: (project_id, callback = (error, docs) ->) -> MongoManager.getProjectsDocs project_id, (error, docs) -> - if err? + if error? return callback(error) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") @@ -27,14 +27,14 @@ module.exports = DocArchive = request.put options, (err, res)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong archiving doc in aws" - callback(err) + return callback(err) MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> return callback(error) if error? callback() unArchiveAllDocs: (project_id, callback = (error) ->) -> MongoManager.getArchivedProjectDocs project_id, (error, docs) -> - if err? + if error? return callback(error) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") @@ -52,7 +52,7 @@ module.exports = DocArchive = request.get options, (err, res, lines)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong unarchiving doc from aws" - callback(err) + return callback(err) MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> return callback(error) if error? callback() diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee index f00fb11c69..a2c9a1d4f9 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManager.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -22,13 +22,33 @@ describe "DocArchiveManager", -> user_files:"sl_user_files" @request = - put: sinon.stub().callsArgWith(1, null, statusCode:200) + put: {} + get: {} + + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + inS3: true + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + inS3: true + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + inS3: true + }] @MongoManager = markDocAsArchived: sinon.stub().callsArgWith(2, null) - upsertIntoDocCollection: sinon.stub() + upsertIntoDocCollection: sinon.stub().callsArgWith(3, null) + getProjectsDocs: sinon.stub().callsArgWith(1, null, @docs) + getArchivedProjectDocs: sinon.stub().callsArgWith(1, null, @docs) - @DocArchiveManager = SandboxedModule.require modulePath, requires: + @requires = "settings-sharelatex": @settings "./MongoManager": @MongoManager "request": @request @@ -36,35 +56,118 @@ describe "DocArchiveManager", -> log:-> err:-> - @key = "my/key" - @bucketName = "my-bucket" @error = "my errror" - - @docs = [{ - _id: ObjectId() - lines: ["one", "two", "three"] - rev: 2 - }, { - _id: ObjectId() - lines: ["aaa", "bbb", "ccc"] - rev: 4 - }, { - _id: ObjectId() - lines: ["111", "222", "333"] - rev: 6 - }] - @project_id = ObjectId().toString() - @callback = sinon.stub() - @stubbedError = new Error("blew up") + @stubbedError = new Errors.NotFoundError("blew up") describe "archiveDoc", -> it "should use correct options", (done)-> + @request.put = sinon.stub().callsArgWith(1, null, statusCode:200) + @requires["request"] = @request + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> opts = @request.put.args[0][0] assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@settings.filestore.stores.user_files}) opts.json.should.equal @docs[0].lines opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" - done() \ No newline at end of file + done() + + it "should return the error", (done)-> + @request.put = sinon.stub().callsArgWith(1, @error, {}) + @requires["request"] = @request + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + err.should.equal @error + done() + + describe "unarchiveDoc", -> + + it "should use correct options", (done)-> + @request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @docs[0].lines) + @requires["request"] = @request + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.unarchiveDoc @project_id, @docs[0]._id, (err)=> + opts = @request.get.args[0][0] + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@settings.filestore.stores.user_files}) + opts.json.should.equal true + opts.timeout.should.equal (30*1000) + opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" + done() + + it "should return the error", (done)-> + @request.get = sinon.stub().callsArgWith(1, @error, {}, {}) + @requires["request"] = @request + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.unarchiveDoc @project_id, @docs[0], (err)=> + err.should.equal @error + done() + + describe "archiveAllDocs", -> + + it "should archive all project docs", (done)-> + @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, @docs) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + @DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.archiveAllDocs @project_id, (err)=> + for doc in @docs + @DocArchiveManager.archiveDoc.calledWith(@project_id, doc).should.equal true + should.not.exist err + done() + + it "should return error if have no docs", (done)-> + @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, null) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.archiveAllDocs @project_id, (err)=> + should.exist err + done() + + it "should return the error", (done)-> + @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, @error, null) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.archiveAllDocs @project_id, (err)=> + err.should.equal @error + done() + + describe "unArchiveAllDocs", -> + + it "should unarchive all inS3 docs", (done)-> + @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, @docs) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + @DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null) + + @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> + for doc in @docs + @DocArchiveManager.unarchiveDoc.calledWith(@project_id, doc._id).should.equal true + should.not.exist err + done() + + it "should return error if have no docs", (done)-> + @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, null) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> + should.exist err + done() + + it "should return the error", (done)-> + @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, @error, null) + @requires["./MongoManager"] = @MongoManager + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> + err.should.equal @error + done() diff --git a/services/docstore/test/unit/coffee/DocManagerTests.coffee b/services/docstore/test/unit/coffee/DocManagerTests.coffee index ab28058791..1aef029ddf 100644 --- a/services/docstore/test/unit/coffee/DocManagerTests.coffee +++ b/services/docstore/test/unit/coffee/DocManagerTests.coffee @@ -11,6 +11,7 @@ describe "DocManager", -> beforeEach -> @DocManager = SandboxedModule.require modulePath, requires: "./MongoManager": @MongoManager = {} + "./DocArchiveManager": @DocArchiveManager = {} "logger-sharelatex": @logger = log: sinon.stub() warn:-> @@ -74,6 +75,7 @@ describe "DocManager", -> beforeEach -> @docs = [{ _id: @doc_id, lines: ["mock-lines"] }] @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, @docs) + @DocArchiveManager.unArchiveAllDocs = sinon.stub().callsArgWith(1, null, @docs) @DocManager.getAllDocs @project_id, @callback it "should get the project from the database", -> @@ -87,6 +89,7 @@ describe "DocManager", -> describe "when there are no docs for the project", -> beforeEach -> @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, null) + @DocArchiveManager.unArchiveAllDocs = sinon.stub().callsArgWith(1, null, null) @DocManager.getAllDocs @project_id, @callback it "should return a NotFoundError", -> From a26320013deb4f5a0956250d7aad76ea8ab7f6f1 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 19:24:45 -0300 Subject: [PATCH 12/21] more acceptance tests --- .../app/coffee/DocArchiveManager.coffee | 28 ++++----- .../acceptance/coffee/ArchiveDocsTests.coffee | 59 +++++++++++++++---- .../coffee/helpers/DocstoreClient.coffee | 4 ++ .../unit/coffee/HttpControllerTests.coffee | 18 ++++++ 4 files changed, 82 insertions(+), 27 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 8c24237983..77cc78df67 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -23,7 +23,7 @@ module.exports = DocArchive = archiveDoc: (project_id, doc, callback)-> logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" - options = buildS3Options(doc.lines, project_id+"/"+doc._id) + options = DocArchive.buildS3Options(doc.lines, project_id+"/"+doc._id) request.put options, (err, res)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong archiving doc in aws" @@ -48,7 +48,7 @@ module.exports = DocArchive = unarchiveDoc: (project_id, doc_id, callback)-> logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3" - options = buildS3Options(true, project_id+"/"+doc_id) + options = DocArchive.buildS3Options(true, project_id+"/"+doc_id) request.get options, (err, res, lines)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong unarchiving doc from aws" @@ -57,15 +57,15 @@ module.exports = DocArchive = return callback(error) if error? callback() -buildS3Options = (content, key)-> - return { - aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: settings.filestore.stores.user_files - timeout: thirtySeconds - json: content - #headers: - # 'content-md5': crypto.createHash("md5").update(content).digest("hex") - uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" - } \ No newline at end of file + buildS3Options: (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 7b886bec9c..e961ecce95 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -1,7 +1,7 @@ sinon = require "sinon" chai = require("chai") -chai.should() -{ObjectId} = require "mongojs" +should = chai.should() +{db, ObjectId} = require "../../../app/js/mongojs" async = require "async" Settings = require("settings-sharelatex") @@ -11,6 +11,7 @@ if Settings.filestore?.backend == "s3" describe "Archiving all docs", -> beforeEach (done) -> + @callback = sinon.stub() @project_id = ObjectId() @docs = [{ _id: ObjectId() @@ -33,17 +34,49 @@ if Settings.filestore?.backend == "s3" DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback async.series jobs, done - it "should archive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - res.statusCode.should.equal 204 - done() + afterEach (done) -> + db.docs.remove({project_id: @project_id}, done) - it "should unarchive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - DocstoreClient.getAllDocs @project_id, (error, res, docs) => - throw error if error? - docs.length.should.equal @docs.length - for doc, i in docs - doc.lines.should.deep.equal @docs[i].lines + describe "Archiving all docs", -> + beforeEach (done) -> + + DocstoreClient.archiveAllDoc @project_id, (error, @res) => done() + it "should archive all the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines in each doc", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + db.docs.findOne _id: archiveDoc._id, (error, doc) => + should.not.exist doc.lines + doc.inS3.should.equal true + callback() + async.series jobs, done + + it "should be able get the same docs back", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => + doc.toString().should.equal archiveDoc.lines.toString() + callback() + async.series jobs, done + + + describe "Unarchiving all docs", -> + + it "should unarchive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + DocstoreClient.getAllDocs @project_id, (error, res, docs) => + throw error if error? + docs.length.should.equal @docs.length + for doc, i in docs + doc.lines.should.deep.equal @docs[i].lines + done() + diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 989a2ce0e7..74acf1134a 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -1,6 +1,7 @@ request = require("request").defaults(jar: false) {db, ObjectId} = require("../../../../app/js/mongojs") settings = require("settings-sharelatex") +DocArchiveManager = require("../../../../app/js/DocArchiveManager.js") module.exports = DocstoreClient = @@ -45,3 +46,6 @@ module.exports = DocstoreClient = url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive" }, callback + getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> + options = DocArchiveManager.buildS3Options(true, project_id+"/"+doc_id) + request.get options, callback diff --git a/services/docstore/test/unit/coffee/HttpControllerTests.coffee b/services/docstore/test/unit/coffee/HttpControllerTests.coffee index 66f82f5b7b..c76ced9696 100644 --- a/services/docstore/test/unit/coffee/HttpControllerTests.coffee +++ b/services/docstore/test/unit/coffee/HttpControllerTests.coffee @@ -11,6 +11,7 @@ describe "HttpController", -> beforeEach -> @HttpController = SandboxedModule.require modulePath, requires: "./DocManager": @DocManager = {} + "./DocArchiveManager": @DocArchiveManager = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } @res = { send: sinon.stub(), json: sinon.stub(), setHeader:sinon.stub() } @req = { query:{}} @@ -247,3 +248,20 @@ describe "HttpController", -> @res.send .calledWith(204) .should.equal true + + describe "archiveAllDocs", -> + beforeEach -> + @req.params = + project_id: @project_id + @DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1) + @HttpController.archiveAllDocs @req, @res, @next + + it "should archive the project", -> + @DocArchiveManager.archiveAllDocs + .calledWith(@project_id) + .should.equal true + + it "should return a 204 (No Content)", -> + @res.send + .calledWith(204) + .should.equal true \ No newline at end of file From 5ba14522da36ca0edf07f3a2adf8f2505bec7b6c Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 2 Jun 2015 20:08:50 -0300 Subject: [PATCH 13/21] handling error --- .../app/coffee/DocArchiveManager.coffee | 4 +- .../docstore/config/settings.defaults.coffee | 2 +- .../acceptance/coffee/ArchiveDocsTests.coffee | 90 +++++++++++-------- 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 77cc78df67..f04e1b125a 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -27,7 +27,7 @@ module.exports = DocArchive = request.put options, (err, res)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong archiving doc in aws" - return callback(err) + return callback new Errors.NotFoundError("Error in S3 request") MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> return callback(error) if error? callback() @@ -52,7 +52,7 @@ module.exports = DocArchive = request.get options, (err, res, lines)-> if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong unarchiving doc from aws" - return callback(err) + return callback new Errors.NotFoundError("Error in S3 request") MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> return callback(error) if error? callback() diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index 4a16b05c2a..1af857e4f7 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -17,4 +17,4 @@ module.exports = # s3: # key: "" # secret: "" - \ No newline at end of file + # fail: false \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index e961ecce95..4e27dc6302 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -37,46 +37,58 @@ if Settings.filestore?.backend == "s3" afterEach (done) -> db.docs.remove({project_id: @project_id}, done) - describe "Archiving all docs", -> - beforeEach (done) -> + if !Settings.filestore?.fail - DocstoreClient.archiveAllDoc @project_id, (error, @res) => - done() + describe "Archiving all docs", -> + beforeEach (done) -> - it "should archive all the docs", (done) -> - @res.statusCode.should.equal 204 - done() - - it "should set inS3 and unset lines in each doc", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - db.docs.findOne _id: archiveDoc._id, (error, doc) => - should.not.exist doc.lines - doc.inS3.should.equal true - callback() - async.series jobs, done - - it "should be able get the same docs back", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => - doc.toString().should.equal archiveDoc.lines.toString() - callback() - async.series jobs, done - - - describe "Unarchiving all docs", -> - - it "should unarchive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - DocstoreClient.getAllDocs @project_id, (error, res, docs) => - throw error if error? - docs.length.should.equal @docs.length - for doc, i in docs - doc.lines.should.deep.equal @docs[i].lines + DocstoreClient.archiveAllDoc @project_id, (error, @res) => done() + it "should archive all the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines in each doc", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + db.docs.findOne _id: archiveDoc._id, (error, doc) => + should.not.exist doc.lines + doc.inS3.should.equal true + callback() + async.series jobs, done + + it "should be able get the same docs back", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => + doc.toString().should.equal archiveDoc.lines.toString() + callback() + async.series jobs, done + + + describe "Unarchiving all docs", -> + + it "should unarchive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + DocstoreClient.getAllDocs @project_id, (error, res, docs) => + throw error if error? + docs.length.should.equal @docs.length + for doc, i in docs + doc.lines.should.deep.equal @docs[i].lines + done() + + # set fail to true and also run the docstore service with a denied config + if Settings.filestore?.fail + + describe "Test S3 fail request", -> + + it "should return a 404", (done) -> + + DocstoreClient.archiveAllDoc @project_id, (error, res) => + res.statusCode.should.equal 404 + done() From aa23c1da56f0c40591671af7a3ea5936c06620f4 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 Jun 2015 11:45:47 -0300 Subject: [PATCH 14/21] md5 check for doc lines --- services/docstore/app/coffee/DocArchiveManager.coffee | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index f04e1b125a..206dc929f5 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -25,9 +25,14 @@ module.exports = DocArchive = logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" options = DocArchive.buildS3Options(doc.lines, project_id+"/"+doc._id) request.put options, (err, res)-> + md5lines = crypto.createHash("md5").update(JSON.stringify(doc.lines)).digest("hex") + md5response = res.headers.etag.toString().replace(/\"/g, '') if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong archiving doc in aws" return callback new Errors.NotFoundError("Error in S3 request") + if md5lines != md5response + logger.err responseMD5:md5response, linesMD5:md5lines, "error in response md5 from s3" + return callback new Errors.NotFoundError("Error in S3 md5 response") MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> return callback(error) if error? callback() @@ -66,6 +71,6 @@ module.exports = DocArchive = timeout: thirtySeconds json: content #headers: - # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + # 'content-md5': crypto.createHash("md5").update(JSON.stringify(content)).digest("hex") uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" } \ No newline at end of file From 9a68bfb39328ff52b784fc29f291f8cdfdb3e7a7 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 Jun 2015 12:00:08 -0300 Subject: [PATCH 15/21] fix unit tests --- .../test/unit/coffee/DocArchiveManager.coffee | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee index a2c9a1d4f9..dcd425138b 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManager.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -7,6 +7,7 @@ modulePath = "../../../app/js/DocArchiveManager.js" SandboxedModule = require('sandboxed-module') ObjectId = require("mongojs").ObjectId Errors = require "../../../app/js/Errors" +crypto = require("crypto") describe "DocArchiveManager", -> @@ -58,12 +59,12 @@ describe "DocArchiveManager", -> @error = "my errror" @project_id = ObjectId().toString() - @stubbedError = new Errors.NotFoundError("blew up") + @stubbedError = new Errors.NotFoundError("Error in S3 request") describe "archiveDoc", -> it "should use correct options", (done)-> - @request.put = sinon.stub().callsArgWith(1, null, statusCode:200) + @request.put = sinon.stub().callsArgWith(1, null, {statusCode:200,headers:{etag:""}}) @requires["request"] = @request @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires @@ -75,13 +76,23 @@ describe "DocArchiveManager", -> opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" done() - it "should return the error", (done)-> - @request.put = sinon.stub().callsArgWith(1, @error, {}) + it "should return no md5 error", (done)-> + @md5 = crypto.createHash("md5").update(JSON.stringify(@docs[0].lines)).digest("hex") + @request.put = sinon.stub().callsArgWith(1, null, {statusCode:200,headers:{etag:@md5}}) @requires["request"] = @request @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> - err.should.equal @error + should.not.exist err + done() + + it "should return the error", (done)-> + @request.put = sinon.stub().callsArgWith(1, @stubbedError, {statusCode:400,headers:{etag:""}}) + @requires["request"] = @request + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + + @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + should.exist err done() describe "unarchiveDoc", -> @@ -100,12 +111,12 @@ describe "DocArchiveManager", -> done() it "should return the error", (done)-> - @request.get = sinon.stub().callsArgWith(1, @error, {}, {}) + @request.get = sinon.stub().callsArgWith(1, @stubbedError, {}, {}) @requires["request"] = @request @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires @DocArchiveManager.unarchiveDoc @project_id, @docs[0], (err)=> - err.should.equal @error + should.exist err done() describe "archiveAllDocs", -> From 42f2afc4dda8aa31d58cf6d61b6a8f24d8e99a29 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 Jun 2015 20:05:15 -0300 Subject: [PATCH 16/21] remove unarchived docs from s3 --- services/docstore/app/coffee/DocArchiveManager.coffee | 7 ++++++- .../docstore/test/unit/coffee/DocArchiveManager.coffee | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 206dc929f5..d82067737e 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -60,7 +60,12 @@ module.exports = DocArchive = return callback new Errors.NotFoundError("Error in S3 request") MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> return callback(error) if error? - callback() + logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" + request.del options, (err, res, body)-> + if err? || res.statusCode != 204 + logger.err err:err, res:res, "something went wrong deleting doc from aws" + return callback new Errors.NotFoundError("Error in S3 request") + callback() buildS3Options: (content, key)-> return { diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee index dcd425138b..33d5205e85 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManager.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -25,6 +25,7 @@ describe "DocArchiveManager", -> @request = put: {} get: {} + del: {} @docs = [{ _id: ObjectId() @@ -98,7 +99,8 @@ describe "DocArchiveManager", -> describe "unarchiveDoc", -> it "should use correct options", (done)-> - @request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @docs[0].lines) + @request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @docs[0].lines) + @request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {}) @requires["request"] = @request @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires From 9af2c07c3bd7f2fe1d961b7ea45a6377ffe1f8af Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 13 Aug 2015 12:59:04 +0100 Subject: [PATCH 17/21] always add archiving root in --- services/docstore/app.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index 3c3a35d213..be51abecd8 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -21,8 +21,7 @@ app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc -if Settings.filestore?.backend == "s3" - app.get '/project/:project_id/archive', HttpController.archiveAllDocs +app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> res.send('docstore is alive') From 859e8a0e570178d9aa5194d0f79d5f096810f3cb Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 13 Aug 2015 13:00:09 +0100 Subject: [PATCH 18/21] - deal with calling archive twice on same project - clean up unit tests - change s3 keys - use err not error --- .../app/coffee/DocArchiveManager.coffee | 49 ++++---- .../acceptance/coffee/ArchiveDocsTests.coffee | 112 +++++++++++------- .../test/unit/coffee/DocArchiveManager.coffee | 107 ++++++++--------- 3 files changed, 143 insertions(+), 125 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index d82067737e..9c11192e51 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -10,14 +10,18 @@ thirtySeconds = 30 * 1000 module.exports = DocArchive = - archiveAllDocs: (project_id, callback = (error, docs) ->) -> - MongoManager.getProjectsDocs project_id, (error, docs) -> - if error? - return callback(error) + archiveAllDocs: (project_id, callback = (err, docs) ->) -> + MongoManager.getProjectsDocs project_id, (err, docs) -> + if err? + return callback(err) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") jobs = _.map docs, (doc) -> - (cb)-> DocArchive.archiveDoc project_id, doc, cb + (cb)-> + if doc.inS3 + return cb() + else + DocArchive.archiveDoc project_id, doc, cb async.series jobs, callback @@ -25,22 +29,23 @@ module.exports = DocArchive = logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" options = DocArchive.buildS3Options(doc.lines, project_id+"/"+doc._id) request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, project_id:project_id, doc_id: doc._id, statusCode: res?.statusCode, "something went wrong archiving doc in aws" + return callback new Error("Error in S3 request") md5lines = crypto.createHash("md5").update(JSON.stringify(doc.lines)).digest("hex") md5response = res.headers.etag.toString().replace(/\"/g, '') - if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong archiving doc in aws" - return callback new Errors.NotFoundError("Error in S3 request") if md5lines != md5response - logger.err responseMD5:md5response, linesMD5:md5lines, "error in response md5 from s3" - return callback new Errors.NotFoundError("Error in S3 md5 response") - MongoManager.markDocAsArchived doc._id, doc.rev, (error) -> - return callback(error) if error? + logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc._id, "err in response md5 from s3" + return callback new Error("Error in S3 md5 response") + MongoManager.markDocAsArchived doc._id, doc.rev, (err) -> + return callback(err) if err? callback() - unArchiveAllDocs: (project_id, callback = (error) ->) -> - MongoManager.getArchivedProjectDocs project_id, (error, docs) -> - if error? - return callback(error) + unArchiveAllDocs: (project_id, callback = (err) ->) -> + MongoManager.getArchivedProjectDocs project_id, (err, docs) -> + if err? + logger.err err:err, project_id:project_id, "error unarchiving all docs" + return callback(err) else if !docs? return callback new Errors.NotFoundError("No docs for project #{project_id}") jobs = _.map docs, (doc) -> @@ -58,8 +63,8 @@ module.exports = DocArchive = if err? || res.statusCode != 200 logger.err err:err, res:res, "something went wrong unarchiving doc from aws" return callback new Errors.NotFoundError("Error in S3 request") - MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (error) -> - return callback(error) if error? + MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (err) -> + return callback(err) if err? logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" request.del options, (err, res, body)-> if err? || res.statusCode != 204 @@ -70,12 +75,12 @@ module.exports = DocArchive = buildS3Options: (content, key)-> return { aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret - bucket: settings.filestore.stores.user_files + key: settings.docstore.s3.key + secret: settings.docstore.s3.secret + bucket: settings.docstore.s3.bucket timeout: thirtySeconds json: content #headers: # 'content-md5': crypto.createHash("md5").update(JSON.stringify(content)).digest("hex") - uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + uri:"https://#{settings.docstore.s3.bucket}.s3.amazonaws.com/#{key}" } \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 4e27dc6302..545fff7a2c 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -28,7 +28,7 @@ if Settings.filestore?.backend == "s3" }] jobs = for doc in @docs do (doc) => - (callback) => + (callback) => DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> doc.lines[0] = doc.lines[0]+" added" DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback @@ -37,58 +37,78 @@ if Settings.filestore?.backend == "s3" afterEach (done) -> db.docs.remove({project_id: @project_id}, done) - if !Settings.filestore?.fail - describe "Archiving all docs", -> - beforeEach (done) -> + describe "Archiving all docs", -> + beforeEach (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, @res) => - done() - - it "should archive all the docs", (done) -> - @res.statusCode.should.equal 204 + DocstoreClient.archiveAllDoc @project_id, (error, @res) => done() - it "should set inS3 and unset lines in each doc", (done) -> + it "should archive all the docs", (done) -> + @res.statusCode.should.equal 204 + done() - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - db.docs.findOne _id: archiveDoc._id, (error, doc) => - should.not.exist doc.lines - doc.inS3.should.equal true - callback() - async.series jobs, done + it "should set inS3 and unset lines in each doc", (done) -> - it "should be able get the same docs back", (done) -> + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + db.docs.findOne _id: archiveDoc._id, (error, doc) => + should.not.exist doc.lines + doc.inS3.should.equal true + callback() + async.series jobs, done - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => - doc.toString().should.equal archiveDoc.lines.toString() - callback() - async.series jobs, done + it "should be able get the same docs back", (done) -> + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => + doc.toString().should.equal archiveDoc.lines.toString() + callback() + async.series jobs, done - describe "Unarchiving all docs", -> - - it "should unarchive all the docs", (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, res) => - DocstoreClient.getAllDocs @project_id, (error, res, docs) => - throw error if error? - docs.length.should.equal @docs.length - for doc, i in docs - doc.lines.should.deep.equal @docs[i].lines - done() - - # set fail to true and also run the docstore service with a denied config - if Settings.filestore?.fail - - describe "Test S3 fail request", -> - - it "should return a 404", (done) -> - - DocstoreClient.archiveAllDoc @project_id, (error, res) => - res.statusCode.should.equal 404 + describe "Arching all docs twice", -> + beforeEach (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + @res.statusCode.should.equal 204 + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + @res.statusCode.should.equal 204 + done() + + it "should archive all the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines in each doc", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + db.docs.findOne _id: archiveDoc._id, (error, doc) => + should.not.exist doc.lines + doc.inS3.should.equal true + callback() + async.series jobs, done + + it "should be able get the same docs back", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => + doc.toString().should.equal archiveDoc.lines.toString() + callback() + async.series jobs, done + + describe "Unarchiving all docs", -> + + it "should unarchive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + DocstoreClient.getAllDocs @project_id, (error, res, docs) => + throw error if error? + docs.length.should.equal @docs.length + for doc, i in docs + doc.lines.should.deep.equal @docs[i].lines done() diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee index 33d5205e85..2cf47b5a71 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManager.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -14,41 +14,58 @@ describe "DocArchiveManager", -> beforeEach -> @settings = - filestore: - backend: "s3" + docstore: s3: secret: "secret" key: "this_key" - stores: - user_files:"sl_user_files" + bucket:"doc-archive-unit-test" @request = put: {} get: {} del: {} - @docs = [{ + @archivedDocs = [{ + _id: ObjectId() + inS3:true + rev: 2 + }, { + _id: ObjectId() + inS3:true + rev: 4 + }, { + _id: ObjectId() + inS3:true + rev: 6 + }] + + @mongoDocs = [{ _id: ObjectId() lines: ["one", "two", "three"] rev: 2 - inS3: true }, { _id: ObjectId() lines: ["aaa", "bbb", "ccc"] rev: 4 + }, { + _id: ObjectId() inS3: true + rev: 6 + }, { + _id: ObjectId() + inS3: true + rev: 6 }, { _id: ObjectId() lines: ["111", "222", "333"] rev: 6 - inS3: true }] @MongoManager = markDocAsArchived: sinon.stub().callsArgWith(2, null) upsertIntoDocCollection: sinon.stub().callsArgWith(3, null) - getProjectsDocs: sinon.stub().callsArgWith(1, null, @docs) - getArchivedProjectDocs: sinon.stub().callsArgWith(1, null, @docs) + getProjectsDocs: sinon.stub().callsArgWith(1, null, @mongoDocs) + getArchivedProjectDocs: sinon.stub().callsArgWith(1, null, @mongoDocs) @requires = "settings-sharelatex": @settings @@ -61,84 +78,71 @@ describe "DocArchiveManager", -> @error = "my errror" @project_id = ObjectId().toString() @stubbedError = new Errors.NotFoundError("Error in S3 request") + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires describe "archiveDoc", -> it "should use correct options", (done)-> @request.put = sinon.stub().callsArgWith(1, null, {statusCode:200,headers:{etag:""}}) - @requires["request"] = @request - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - - @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + @DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=> opts = @request.put.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@settings.filestore.stores.user_files}) - opts.json.should.equal @docs[0].lines + assert.deepEqual(opts.aws, {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket}) + opts.json.should.equal @mongoDocs[0].lines opts.timeout.should.equal (30*1000) - opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" + opts.uri.should.equal "https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{@mongoDocs[0]._id}" done() it "should return no md5 error", (done)-> - @md5 = crypto.createHash("md5").update(JSON.stringify(@docs[0].lines)).digest("hex") + @md5 = crypto.createHash("md5").update(JSON.stringify(@mongoDocs[0].lines)).digest("hex") @request.put = sinon.stub().callsArgWith(1, null, {statusCode:200,headers:{etag:@md5}}) - @requires["request"] = @request - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - - @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + @DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=> should.not.exist err done() it "should return the error", (done)-> @request.put = sinon.stub().callsArgWith(1, @stubbedError, {statusCode:400,headers:{etag:""}}) - @requires["request"] = @request - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - - @DocArchiveManager.archiveDoc @project_id, @docs[0], (err)=> + @DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=> should.exist err done() describe "unarchiveDoc", -> it "should use correct options", (done)-> - @request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @docs[0].lines) + @request.get = sinon.stub().callsArgWith(1, null, statusCode:200, @mongoDocs[0].lines) @request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {}) - @requires["request"] = @request - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - - @DocArchiveManager.unarchiveDoc @project_id, @docs[0]._id, (err)=> + @DocArchiveManager.unarchiveDoc @project_id, @mongoDocs[0]._id, (err)=> opts = @request.get.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@settings.filestore.stores.user_files}) + assert.deepEqual(opts.aws, {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket}) opts.json.should.equal true opts.timeout.should.equal (30*1000) - opts.uri.should.equal "https://#{@settings.filestore.stores.user_files}.s3.amazonaws.com/#{@project_id}/#{@docs[0]._id}" + opts.uri.should.equal "https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{@mongoDocs[0]._id}" done() it "should return the error", (done)-> @request.get = sinon.stub().callsArgWith(1, @stubbedError, {}, {}) - @requires["request"] = @request - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - - @DocArchiveManager.unarchiveDoc @project_id, @docs[0], (err)=> + @DocArchiveManager.unarchiveDoc @project_id, @mongoDocs[0], (err)=> should.exist err done() describe "archiveAllDocs", -> - it "should archive all project docs", (done)-> - @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, @docs) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + it "should archive all project docs which are not in s3", (done)-> + @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, @mongoDocs) @DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null) @DocArchiveManager.archiveAllDocs @project_id, (err)=> - for doc in @docs - @DocArchiveManager.archiveDoc.calledWith(@project_id, doc).should.equal true + @DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[0]).should.equal true + @DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[1]).should.equal true + @DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[4]).should.equal true + + @DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[2]).should.equal false + @DocArchiveManager.archiveDoc.calledWith(@project_id, @mongoDocs[3]).should.equal false + should.not.exist err done() it "should return error if have no docs", (done)-> @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, null, null) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires @DocArchiveManager.archiveAllDocs @project_id, (err)=> should.exist err @@ -146,8 +150,6 @@ describe "DocArchiveManager", -> it "should return the error", (done)-> @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(1, @error, null) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires @DocArchiveManager.archiveAllDocs @project_id, (err)=> err.should.equal @error @@ -156,31 +158,22 @@ describe "DocArchiveManager", -> describe "unArchiveAllDocs", -> it "should unarchive all inS3 docs", (done)-> - @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, @docs) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, @archivedDocs) @DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null) - @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> - for doc in @docs + for doc in @archivedDocs @DocArchiveManager.unarchiveDoc.calledWith(@project_id, doc._id).should.equal true should.not.exist err done() it "should return error if have no docs", (done)-> @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, null, null) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> should.exist err done() it "should return the error", (done)-> @MongoManager.getArchivedProjectDocs = sinon.stub().callsArgWith(1, @error, null) - @requires["./MongoManager"] = @MongoManager - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires - @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> err.should.equal @error done() From b8884352f6ab2289c462623a8a323250dd575f86 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 13 Aug 2015 13:59:12 +0100 Subject: [PATCH 19/21] added some better error logging --- services/docstore/app/coffee/DocArchiveManager.coffee | 8 +++----- services/docstore/app/coffee/DocManager.coffee | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 9c11192e51..23bd733d3f 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -35,7 +35,7 @@ module.exports = DocArchive = md5lines = crypto.createHash("md5").update(JSON.stringify(doc.lines)).digest("hex") md5response = res.headers.etag.toString().replace(/\"/g, '') if md5lines != md5response - logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc._id, "err in response md5 from s3" + logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc?._id, "err in response md5 from s3" return callback new Error("Error in S3 md5 response") MongoManager.markDocAsArchived doc._id, doc.rev, (err) -> return callback(err) if err? @@ -61,14 +61,14 @@ module.exports = DocArchive = options = DocArchive.buildS3Options(true, project_id+"/"+doc_id) request.get options, (err, res, lines)-> if err? || res.statusCode != 200 - logger.err err:err, res:res, "something went wrong unarchiving doc from aws" + logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong unarchiving doc from aws" return callback new Errors.NotFoundError("Error in S3 request") MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), lines, (err) -> return callback(err) if err? logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" request.del options, (err, res, body)-> if err? || res.statusCode != 204 - logger.err err:err, res:res, "something went wrong deleting doc from aws" + logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws" return callback new Errors.NotFoundError("Error in S3 request") callback() @@ -80,7 +80,5 @@ module.exports = DocArchive = bucket: settings.docstore.s3.bucket timeout: thirtySeconds json: content - #headers: - # 'content-md5': crypto.createHash("md5").update(JSON.stringify(content)).digest("hex") uri:"https://#{settings.docstore.s3.bucket}.s3.amazonaws.com/#{key}" } \ No newline at end of file diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 85beac7031..2e215567fb 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -20,6 +20,7 @@ module.exports = DocManager = else if doc?.inS3 DocArchive.unarchiveDoc project_id, doc_id, (err)-> if err? + logger.err err:err, project_id:project_id, doc_id:doc_id, "error unarchiving doc" return callback(err) MongoManager.findDoc doc_id, callback else From ddf4692f0de7ce601779a26bf8595cb93ead52c7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 13 Aug 2015 13:59:40 +0100 Subject: [PATCH 20/21] added acceptance test for large doc ~1mb --- .../acceptance/coffee/ArchiveDocsTests.coffee | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 545fff7a2c..fa07f1abae 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -102,6 +102,44 @@ if Settings.filestore?.backend == "s3" callback() async.series jobs, done + + + describe "archiving massive document", (done)-> + beforeEach (done)-> + @timeout 1000 * 30 + quarterMegInBytes = 250000 + lines = require("crypto").randomBytes(quarterMegInBytes).toString("hex") + console.log @project_id, @docs[1]._id, "helllllo" + @docs[1].lines = [lines,lines,lines,lines] + DocstoreClient.updateDoc @project_id, @docs[1]._id, @docs[1].lines, => + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + done() + + + it "should archive all the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines in each doc", (done) -> + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + db.docs.findOne _id: archiveDoc._id, (error, doc) => + should.not.exist doc.lines + doc.inS3.should.equal true + callback() + async.series jobs, done + + it "should be able get the same docs back", (done) -> + + jobs = for archiveDoc in @docs + do (archiveDoc) => + (callback) => + DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => + doc.toString().should.equal archiveDoc.lines.toString() + callback() + async.series jobs, done + describe "Unarchiving all docs", -> it "should unarchive all the docs", (done) -> From 32bf64236bc6ba82922bc01c3b0b9093e800a82e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 13 Aug 2015 14:16:40 +0100 Subject: [PATCH 21/21] fixed config to work with s3 correctly --- services/docstore/config/settings.defaults.coffee | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index 1af857e4f7..e7b4046b64 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -10,11 +10,8 @@ module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' - #filestore: - # backend: "s3" - # stores: - # user_files: "" + #docstore: # s3: # key: "" # secret: "" - # fail: false \ No newline at end of file + # bucket: "something"