diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index 67fc7a14c9..e75cb1547f 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -42,6 +42,7 @@ app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc app.post '/project/:project_id/archive', HttpController.archiveAllDocs app.post '/project/:project_id/unarchive', HttpController.unArchiveAllDocs +app.post '/project/:project_id/destroy', HttpController.destroyAllDocs app.get "/health_check", HttpController.healthCheck diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 12f4b47c82..1fb6fb2657 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -79,11 +79,43 @@ module.exports = DocArchive = MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), mongo_doc, (err) -> return callback(err) if err? logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" - request.del options, (err, res, body)-> - if err? || res.statusCode != 204 - logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws" - return callback new Errors.NotFoundError("Error in S3 request") - callback() + DocArchive._deleteDocFromS3 project_id, doc_id, callback + + destroyAllDocs: (project_id, callback = (err) ->) -> + MongoManager.getProjectsDocs project_id, {include_deleted: true}, {_id: 1}, (err, docs) -> + if err? + logger.err err:err, project_id:project_id, "error getting project's docs" + return callback(err) + else if !docs? + return callback() + jobs = _.map docs, (doc) -> + (cb)-> + DocArchive.destroyDoc(project_id, doc._id, cb) + async.parallelLimit jobs, 5, callback + + destroyDoc: (project_id, doc_id, callback)-> + logger.log project_id: project_id, doc_id: doc_id, "removing doc from mongo and s3" + MongoManager.findDoc project_id, doc_id, {inS3: 1}, (error, doc) -> + return callback error if error? + return callback new Errors.NotFoundError("Doc not found in Mongo") unless doc? + if doc.inS3 == true + DocArchive._deleteDocFromS3 project_id, doc_id, (err) -> + return err if err? + MongoManager.destroyDoc doc_id, callback + else + MongoManager.destroyDoc doc_id, callback + + _deleteDocFromS3: (project_id, doc_id, callback) -> + try + options = DocArchive.buildS3Options(project_id+"/"+doc_id) + catch e + return callback e + options.json = true + request.del options, (err, res, body)-> + if err? || res.statusCode != 204 + logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws" + return callback new Error("Error in S3 request") + callback() _s3DocToMongoDoc: (doc, callback = (error, mongo_doc) ->) -> mongo_doc = {} diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 7ebcc4fd44..ae2ce1670f 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -117,6 +117,13 @@ module.exports = HttpController = return next(error) if error? res.send 200 + destroyAllDocs: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "destroying all docs" + DocArchive.destroyAllDocs project_id, (error) -> + return next(error) if error? + res.send 204 + healthCheck: (req, res)-> HealthChecker.check (err)-> if err? diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index 1d3d4b4a3c..3bd9eb34c5 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -72,6 +72,14 @@ module.exports = MongoManager = upsert: true }, callback + destroyDoc: (doc_id, callback) -> + db.docs.remove { + _id: ObjectId(doc_id) + }, (err) -> + return callback(err) if err? + db.docOps.remove { + doc_id: ObjectId(doc_id) + }, callback [ 'findDoc', diff --git a/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee b/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee index f97a352c14..4aac25902e 100644 --- a/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee @@ -2,6 +2,7 @@ sinon = require "sinon" chai = require("chai") chai.should() {db, ObjectId} = require "../../../app/js/mongojs" +expect = chai.expect DocstoreApp = require "./helpers/DocstoreApp" DocstoreClient = require "./helpers/DocstoreClient" @@ -40,3 +41,45 @@ describe "Deleting a doc", -> res.statusCode.should.equal 404 done() +describe "Destroying a project's documents", -> + describe "when the doc exists", -> + beforeEach (done) -> + db.docOps.insert {doc_id: ObjectId(@doc_id), version: 1}, (err) -> + return done(err) if err? + DocstoreClient.destroyAllDoc @project_id, done + + it "should remove the doc from the docs collection", (done) -> + db.docs.find _id: @doc_id, (err, docs) -> + expect(err).not.to.exist + expect(docs).to.deep.equal [] + done() + + it "should remove the docOps from the docOps collection", (done) -> + db.docOps.find doc_id: @doc_id, (err, docOps) -> + expect(err).not.to.exist + expect(docOps).to.deep.equal [] + done() + + describe "when the doc is archived", -> + beforeEach (done) -> + DocstoreClient.archiveAllDoc @project_id, (err) -> + return done(err) if err? + DocstoreClient.destroyAllDoc @project_id, done + + it "should remove the doc from the docs collection", (done) -> + db.docs.find _id: @doc_id, (err, docs) -> + expect(err).not.to.exist + expect(docs).to.deep.equal [] + done() + + it "should remove the docOps from the docOps collection", (done) -> + db.docOps.find doc_id: @doc_id, (err, docOps) -> + expect(err).not.to.exist + expect(docOps).to.deep.equal [] + done() + + it "should remove the doc contents from s3", (done) -> + DocstoreClient.getS3Doc @project_id, @doc_id, (error, res, s3_doc) => + throw error if error? + expect(res.statusCode).to.equal 404 + done() diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 755f12d9b0..446f99ad58 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -44,8 +44,12 @@ module.exports = DocstoreClient = archiveAllDoc: (project_id, callback = (error, res, body) ->) -> request.post { url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive" - }, callback + }, callback + destroyAllDoc: (project_id, callback = (error, res, body) ->) -> + request.post { + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/destroy" + }, callback getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> options = DocArchiveManager.buildS3Options(project_id+"/"+doc_id) diff --git a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee index 2b9287d912..70196a4b86 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee @@ -61,6 +61,22 @@ describe "DocArchiveManager", -> rev: 6 }] + @unarchivedDocs = [{ + _id: ObjectId() + lines: ["wombat", "potato", "banana"] + rev: 2 + }, { + _id: ObjectId() + lines: ["llama", "turnip", "apple"] + rev: 4 + }, { + _id: ObjectId() + lines: ["elephant", "swede", "nectarine"] + rev: 6 + }] + + @mixedDocs = @archivedDocs.concat(@unarchivedDocs) + @MongoManager = markDocAsArchived: sinon.stub().callsArgWith(2, null) upsertIntoDocCollection: sinon.stub().callsArgWith(3, null) @@ -214,6 +230,51 @@ describe "DocArchiveManager", -> @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> err.should.equal @error done() + + describe "destroyAllDocs", -> + beforeEach -> + @request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {}) + @MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @mixedDocs) + @MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null) + @MongoManager.destroyDoc = sinon.stub().yields() + for doc in @mixedDocs + @MongoManager.findDoc.withArgs(@project_id, doc._id).callsArgWith(3, null, doc) + + it "should destroy all the docs", (done)-> + @DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null) + @DocArchiveManager.destroyAllDocs @project_id, (err)=> + for doc in @mixedDocs + @DocArchiveManager.destroyDoc.calledWith(@project_id, doc._id).should.equal true + should.not.exist err + done() + + it "should only the s3 docs from s3", (done)-> + docOpts = (doc) => + JSON.parse(JSON.stringify({ + aws: {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket}, + json: true, + timeout: 30 * 1000 + uri:"https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{doc._id}" + })) + + @DocArchiveManager.destroyAllDocs @project_id, (err)=> + expect(err).not.to.exist + + for doc in @archivedDocs + sinon.assert.calledWith(@request.del, docOpts(doc)) + for doc in @unarchivedDocs + expect(@request.del.calledWith(docOpts(doc))).to.equal false # no notCalledWith + + done() + + it "should remove the docs from mongo", (done)-> + @DocArchiveManager.destroyAllDocs @project_id, (err)=> + expect(err).not.to.exist + + for doc in @mixedDocs + sinon.assert.calledWith(@MongoManager.destroyDoc, doc._id) + + done() describe "_s3DocToMongoDoc", -> describe "with the old schema", -> diff --git a/services/docstore/test/unit/coffee/HttpControllerTests.coffee b/services/docstore/test/unit/coffee/HttpControllerTests.coffee index 80ec0c64fe..362325845d 100644 --- a/services/docstore/test/unit/coffee/HttpControllerTests.coffee +++ b/services/docstore/test/unit/coffee/HttpControllerTests.coffee @@ -326,3 +326,17 @@ describe "HttpController", -> @res.send .calledWith(204) .should.equal true + + describe "destroyAllDocs", -> + beforeEach -> + @req.params = + project_id: @project_id + @DocArchiveManager.destroyAllDocs = sinon.stub().callsArg(1) + @HttpController.destroyAllDocs @req, @res, @next + + it "should destroy the docs", -> + sinon.assert.calledWith(@DocArchiveManager.destroyAllDocs, @project_id) + + it "should return 204", -> + sinon.assert.calledWith(@res.send, 204) + diff --git a/services/docstore/test/unit/coffee/MongoManagerTests.coffee b/services/docstore/test/unit/coffee/MongoManagerTests.coffee index 1a264bf0d2..842a3c8884 100644 --- a/services/docstore/test/unit/coffee/MongoManagerTests.coffee +++ b/services/docstore/test/unit/coffee/MongoManagerTests.coffee @@ -110,6 +110,18 @@ describe "MongoManager", -> err.should.equal @stubbedErr done() + describe "destroyDoc", -> + beforeEach (done) -> + @db.docs.remove = sinon.stub().yields() + @db.docOps.remove = sinon.stub().yields() + @MongoManager.destroyDoc '123456789012', done + + it "should destroy the doc", -> + sinon.assert.calledWith(@db.docs.remove, {_id: ObjectId('123456789012')}) + + it "should destroy the docOps", -> + sinon.assert.calledWith(@db.docOps.remove, {doc_id: ObjectId('123456789012')}) + describe "getDocVersion", -> describe "when the doc exists", -> beforeEach ->