From a9ab7b739d9ef3708b735279406fe82ee3feccab Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 30 Mar 2017 17:13:43 +0100 Subject: [PATCH 1/3] Archive ranges as well as doc lines --- .../app/coffee/DocArchiveManager.coffee | 84 +++- .../docstore/app/coffee/MongoManager.coffee | 1 + .../docstore/app/coffee/RangeManager.coffee | 19 +- .../acceptance/coffee/ArchiveDocsTests.coffee | 466 +++++++++++++----- .../coffee/helpers/DocstoreClient.coffee | 5 +- .../test/unit/coffee/DocArchiveManager.coffee | 14 +- 6 files changed, 436 insertions(+), 153 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 925ad3bfc9..a4ae7c2344 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -6,12 +6,13 @@ async = require "async" settings = require("settings-sharelatex") request = require("request") crypto = require("crypto") +RangeManager = require("./RangeManager") thirtySeconds = 30 * 1000 module.exports = DocArchive = archiveAllDocs: (project_id, callback = (err, docs) ->) -> - MongoManager.getProjectsDocs project_id, {include_deleted: true}, {lines: true, rev: true, inS3: true}, (err, docs) -> + MongoManager.getProjectsDocs project_id, {include_deleted: true}, {lines: true, ranges: true, rev: true, inS3: true}, (err, docs) -> if err? return callback(err) else if !docs? @@ -26,21 +27,26 @@ module.exports = DocArchive = archiveDoc: (project_id, doc, callback)-> logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" try - options = DocArchive.buildS3Options(doc.lines, project_id+"/"+doc._id) + options = DocArchive.buildS3Options(project_id+"/"+doc._id) catch e return callback e - request.put options, (err, res)-> - if err? || res.statusCode != 200 - logger.err err:err, res:res, project_id:project_id, doc_id: doc._id, statusCode: res?.statusCode, "something went wrong archiving doc in aws" - return callback new Error("Error in S3 request") - md5lines = crypto.createHash("md5").update(JSON.stringify(doc.lines), "utf8").digest("hex") - md5response = res.headers.etag.toString().replace(/\"/g, '') - if md5lines != md5response - logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc?._id, "err in response md5 from s3" - return callback new Error("Error in S3 md5 response") - MongoManager.markDocAsArchived doc._id, doc.rev, (err) -> - return callback(err) if err? - callback() + DocArchive._mongoDocToS3Doc doc, (error, json_doc) -> + return callback(error) if error? + options.body = json_doc + options.headers = + 'Content-Type': "application/json" + request.put options, (err, res) -> + if err? || res.statusCode != 200 + logger.err err:err, res:res, project_id:project_id, doc_id: doc._id, statusCode: res?.statusCode, "something went wrong archiving doc in aws" + return callback new Error("Error in S3 request") + md5lines = crypto.createHash("md5").update(json_doc, "utf8").digest("hex") + md5response = res.headers.etag.toString().replace(/\"/g, '') + if md5lines != md5response + logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc?._id, "err in response md5 from s3" + return callback new Error("Error in S3 md5 response") + MongoManager.markDocAsArchived doc._id, doc.rev, (err) -> + return callback(err) if err? + callback() unArchiveAllDocs: (project_id, callback = (err) ->) -> MongoManager.getArchivedProjectDocs project_id, (err, docs) -> @@ -60,23 +66,50 @@ module.exports = DocArchive = unarchiveDoc: (project_id, doc_id, callback)-> logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3" try - options = DocArchive.buildS3Options(true, project_id+"/"+doc_id) + options = DocArchive.buildS3Options(project_id+"/"+doc_id) catch e return callback e - request.get options, (err, res, lines)-> + options.json = true + request.get options, (err, res, doc)-> if err? || res.statusCode != 200 logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong unarchiving doc from aws" return callback new Errors.NotFoundError("Error in S3 request") - MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), {lines}, (err) -> - return callback(err) if err? - logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" - request.del options, (err, res, body)-> - if err? || res.statusCode != 204 - logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws" - return callback new Errors.NotFoundError("Error in S3 request") - callback() + DocArchive._s3DocToMongoDoc doc, (error, mongo_doc) -> + return callback(error) if error? + MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), mongo_doc, (err) -> + return callback(err) if err? + logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3" + request.del options, (err, res, body)-> + if err? || res.statusCode != 204 + logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws" + return callback new Errors.NotFoundError("Error in S3 request") + callback() + + _s3DocToMongoDoc: (doc, callback = (error, mongo_doc) ->) -> + mongo_doc = {} + if doc.schema_v == 1 and doc.lines? + mongo_doc.lines = doc.lines + if doc.ranges? + mongo_doc.ranges = RangeManager.jsonRangesToMongo(doc.ranges) + else if doc instanceof Array + mongo_doc.lines = doc + else + return callback(new Error("I don't understand the doc format in s3")) + return callback null, mongo_doc - buildS3Options: (content, key)-> + _mongoDocToS3Doc: (doc, callback = (error, s3_doc) ->) -> + json = JSON.stringify({ + lines: doc.lines + ranges: doc.ranges + schema_v: 1 + }) + if json.indexOf("\u0000") != -1 + error = new Error("null bytes detected") + logger.error {err: error, project_id, doc_id}, error.message + return callback(error) + return callback null, json + + buildS3Options: (key)-> if !settings.docstore.s3? throw new Error("S3 settings are not configured") return { @@ -85,6 +118,5 @@ module.exports = DocArchive = secret: settings.docstore.s3.secret bucket: settings.docstore.s3.bucket timeout: thirtySeconds - json: content uri:"https://#{settings.docstore.s3.bucket}.s3.amazonaws.com/#{key}" } \ No newline at end of file diff --git a/services/docstore/app/coffee/MongoManager.coffee b/services/docstore/app/coffee/MongoManager.coffee index fe6f06e8cb..0f713e6064 100644 --- a/services/docstore/app/coffee/MongoManager.coffee +++ b/services/docstore/app/coffee/MongoManager.coffee @@ -42,6 +42,7 @@ module.exports = MongoManager = $unset: {} update.$set["inS3"] = true update.$unset["lines"] = true + update.$unset["ranges"] = true query = _id: doc_id rev: rev diff --git a/services/docstore/app/coffee/RangeManager.coffee b/services/docstore/app/coffee/RangeManager.coffee index 661d6e4b28..61b0a62246 100644 --- a/services/docstore/app/coffee/RangeManager.coffee +++ b/services/docstore/app/coffee/RangeManager.coffee @@ -16,16 +16,21 @@ module.exports = RangeManager = jsonRangesToMongo: (ranges) -> return null if !ranges? + + updateMetadata = (metadata) -> + if metadata?.ts? + metadata.ts = new Date(metadata.ts) + if metadata?.user_id? + metadata.user_id = RangeManager._safeObjectId(metadata.user_id) + for change in ranges.changes or [] - change.id = @_safeObjectId(change.id) - if change.metadata?.ts? - change.metadata.ts = new Date(change.metadata.ts) - if change.metadata?.user_id? - change.metadata.user_id = @_safeObjectId(change.metadata.user_id) + change.id = RangeManager._safeObjectId(change.id) + updateMetadata(change.metadata) for comment in ranges.comments or [] - comment.id = @_safeObjectId(comment.id) + comment.id = RangeManager._safeObjectId(comment.id) if comment.op?.t? - comment.op.t = @_safeObjectId(comment.op.t) + comment.op.t = RangeManager._safeObjectId(comment.op.t) + updateMetadata(comment.metadata) return ranges _safeObjectId: (data) -> diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index 7bec662ab2..00d6ade7a4 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -1,152 +1,386 @@ sinon = require "sinon" chai = require("chai") should = chai.should() -{db, ObjectId} = require "../../../app/js/mongojs" +{db, ObjectId, ISODate} = require "../../../app/js/mongojs" async = require "async" Settings = require("settings-sharelatex") +DocArchiveManager = require("../../../app/js/DocArchiveManager.js") +request = require "request" DocstoreClient = require "./helpers/DocstoreClient" describe "Archiving", -> - beforeEach (done) -> - @callback = sinon.stub() - @project_id = ObjectId() - @docs = [{ - _id: ObjectId() - lines: ["one", "two", "three"] - rev: 2 - }, { - _id: ObjectId() - lines: ["aaa", "bbb", "ccc"] - rev: 4 - }, { - _id: ObjectId() - lines: [ "", "undefined", "undef", "null", "NULL", "(null)", "nil", "NIL", "true", "false", "True", "False", "None", "\\", "\\\\", "0", "1", "1.00", "$1.00", "1/2", "1E2", "1E02", "1E+02", "-1", "-1.00", "-$1.00", "-1/2", "-1E2", "-1E02", "-1E+02", "1/0", "0/0", "-2147483648/-1", "-9223372036854775808/-1", "0.00", "0..0", ".", "0.0.0", "0,00", "0,,0", ",", "0,0,0", "0.0/0", "1.0/0.0", "0.0/0.0", "1,0/0,0", "0,0/0,0", "--1", "-", "-.", "-,", "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", "NaN", "Infinity", "-Infinity", "0x0", "0xffffffff", "0xffffffffffffffff", "0xabad1dea", "123456789012345678901234567890123456789", "1,000.00", "1 000.00", "1'000.00", "1,000,000.00", "1 000 000.00", "1'000'000.00", "1.000,00", "1 000,00", "1'000,00", "1.000.000,00", "1 000i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟", "̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕", "Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮", "˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥", "00˙Ɩ$-", "Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ", "𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠", "𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌", "𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈", "𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰", "𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘", "𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐", "⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", "", "<script>alert('123');</script>", "", " ", "\">", "'>", ">", "", "< / script >< script >alert(123)< / script >", " onfocus=JaVaSCript:alert(123) autofocus ", "\" onfocus=JaVaSCript:alert(123) autofocus ", "' onfocus=JaVaSCript:alert(123) autofocus ", "＜script＞alert(123)＜/script＞", "ript>alert(123)ript>", "-->", "\";alert(123);t=\"", "';alert(123);t='", "JavaSCript:alert(123)", ";alert(123);", "src=JaVaSCript:prompt(132)", "\"><\\x3Cscript>javascript:alert(1) ", "'`\"><\\x00script>javascript:alert(1)", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "XXX", "javascript:alert(1)\"` `>", "", "", "<a href=http://foo.bar/#x=`y></a><img alt=\"`><img src=x:x onerror=javascript:alert(1)></a>\">", "<!--[if]><script>javascript:alert(1)</script -->", "<!--[if<img src=x onerror=javascript:alert(1)//]> -->", "<script src=\"/\\%(jscript)s\"></script>", "<script src=\"\\\\%(jscript)s\"></script>", "<IMG \"\"\"><SCRIPT>alert(\"XSS\")</SCRIPT>\">", "<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))>", "<IMG SRC=# onmouseover=\"alert('xxs')\">", "<IMG SRC= onmouseover=\"alert('xxs')\">", "<IMG onmouseover=\"alert('xxs')\">", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "perl -e 'print \"<IMG SRC=java\\0script:alert(\\\"XSS\\\")>\";' > out", "<IMG SRC=\"  javascript:alert('XSS');\">", "<SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<BODY onload!#$%&()*~+-_.,:;?@[/|\\]^`=alert(\"XSS\")>", "<SCRIPT/SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<<SCRIPT>alert(\"XSS\");//<</SCRIPT>", "<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >", "<SCRIPT SRC=//ha.ckers.org/.j>", "<IMG SRC=\"javascript:alert('XSS')\"", "<iframe src=http://ha.ckers.org/scriptlet.html <", "\\\";alert('XSS');//", "<plaintext>", "1;DROP TABLE users", "1'; DROP TABLE users-- 1", "' OR 1=1 -- 1", "' OR '1'='1", "-", "--", "--version", "--help", "$USER", "/dev/null; touch /tmp/blns.fail ; echo", "`touch /tmp/blns.fail`", "$(touch /tmp/blns.fail)", "@{[system \"touch /tmp/blns.fail\"]}", "eval(\"puts 'hello world'\")", "System(\"ls -al /\")", "`ls -al /`", "Kernel.exec(\"ls -al /\")", "Kernel.exit(1)", "%x('ls -al /')", "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM \"file:///etc/passwd\" >]><foo>&xxe;</foo>", "$HOME", "$ENV{'HOME'}", "%d", "%s", "%*.*s", "../../../../../../../../../../../etc/passwd%00", "../../../../../../../../../../../etc/hosts", "() { 0; }; touch /tmp/blns.shellshock1.fail;", "() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }", "CON", "PRN", "AUX", "CLOCK$", "NUL", "A:", "ZZ:", "COM1", "LPT1", "LPT2", "LPT3", "COM2", "COM3", "COM4", "Scunthorpe General Hospital", "Penistone Community Church", "Lightwater Country Park", "Jimmy Clitheroe", "Horniman Museum", "shitake mushrooms", "RomansInSussex.co.uk", "http://www.cum.qc.ca/", "Craig Cockburn, Software Specialist", "Linda Callahan", "Dr. Herman I. Libshitz", "magna cum laude", "Super Bowl XXX", "medieval erection of parapets", "evaluate", "mocha", "expression", "Arsenal canal", "classic", "Tyson Gay", "If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.", "Roses are \u001b[0;31mred\u001b[0m, violets are \u001b[0;34mblue. Hope you enjoy terminal hue", "But now...\u001b[20Cfor my greatest trick...\u001b[8m", "The quic\b\b\b\b\b\bk brown fo\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007x... [Beeeep]", "Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗" ] - rev: 6 - }] - @version = 42 - @ranges = [] - jobs = for doc in @docs - do (doc) => - (callback) => - DocstoreClient.createDoc @project_id, doc._id, doc.lines, @version, @ranges, (err)=> - doc.lines[0] = doc.lines[0]+" added" - DocstoreClient.updateDoc @project_id, doc._id, doc.lines, @version, @ranges, callback - # Make sure archiving works on deleted docs too - jobs.push (cb) => - DocstoreClient.deleteDoc @project_id, @docs[2]._id, cb - async.series jobs, done - - afterEach (done) -> - db.docs.remove({project_id: @project_id}, done) - - describe "Archiving all docs", -> - beforeEach (done) -> - - DocstoreClient.archiveAllDoc @project_id, (error, @res) => - done() - - it "should archive all the docs", (done) -> - @res.statusCode.should.equal 204 - done() - - it "should set inS3 and unset lines in each doc", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - db.docs.findOne _id: archiveDoc._id, (error, doc) => - should.not.exist doc.lines - doc.inS3.should.equal true - callback() - async.series jobs, done - - it "should be able get the same docs back", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => - doc.toString().should.equal archiveDoc.lines.toString() - callback() - async.series jobs, done - - describe "Arching all docs twice", -> - beforeEach (done) -> - DocstoreClient.archiveAllDoc @project_id, (error, @res) => - @res.statusCode.should.equal 204 + describe "multiple docs in a project", -> + before (done) -> + @project_id = ObjectId() + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + ranges: {} + version: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + ranges: {} + version: 4 + }] + jobs = for doc in @docs + do (doc) => + (callback) => + DocstoreClient.createDoc @project_id, doc._id, doc.lines, doc.version, doc.ranges, callback + async.series jobs, (error) => + throw error if error? DocstoreClient.archiveAllDoc @project_id, (error, @res) => - @res.statusCode.should.equal 204 done() - + it "should archive all the docs", (done) -> @res.statusCode.should.equal 204 done() - it "should set inS3 and unset lines in each doc", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => + it "should set inS3 and unset lines and ranges in each doc", (done) -> + jobs = for doc in @docs + do (doc) => (callback) => - db.docs.findOne _id: archiveDoc._id, (error, doc) => + db.docs.findOne _id: doc._id, (error, doc) => should.not.exist doc.lines + should.not.exist doc.ranges doc.inS3.should.equal true callback() async.series jobs, done - it "should be able get the same docs back", (done) -> - - jobs = for archiveDoc in @docs - do (archiveDoc) => + it "should set the docs in s3 correctly", (done) -> + jobs = for doc in @docs + do (doc) => (callback) => - DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => - doc.toString().should.equal archiveDoc.lines.toString() + DocstoreClient.getS3Doc @project_id, doc._id, (error, res, s3_doc) => + s3_doc.lines.should.deep.equal doc.lines + s3_doc.ranges.should.deep.equal doc.ranges callback() async.series jobs, done + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should return the docs", (done) -> + for doc, i in @fetched_docs + doc.lines.should.deep.equal @docs[i].lines + done() + + it "should restore the docs to mongo", (done) -> + jobs = for doc, i in @docs + do (doc, i) => + (callback) => + db.docs.findOne _id: doc._id, (error, doc) => + doc.lines.should.deep.equal @docs[i].lines + doc.ranges.should.deep.equal @docs[i].ranges + should.not.exist doc.inS3 + callback() + async.series jobs, done + + describe "a deleted doc", -> + before (done) -> + @project_id = ObjectId() + @doc = { + _id: ObjectId() + lines: ["one", "two", "three"] + ranges: {} + version: 2 + } + DocstoreClient.createDoc @project_id, @doc._id, @doc.lines, @doc.version, @doc.ranges, (error) => + throw error if error? + DocstoreClient.deleteDoc @project_id, @doc._id, (error) => + throw error if error? + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? + done() + + it "should successully archive the docs", (done) -> + @res.statusCode.should.equal 204 + done() - describe "Archiving a large document", (done)-> - beforeEach (done)-> + it "should set inS3 and unset lines and ranges in each doc", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + should.not.exist doc.lines + should.not.exist doc.ranges + doc.inS3.should.equal true + doc.deleted.should.equal true + done() + + it "should set the doc in s3 correctly", (done) -> + DocstoreClient.getS3Doc @project_id, @doc._id, (error, res, s3_doc) => + throw error if error? + s3_doc.lines.should.deep.equal @doc.lines + s3_doc.ranges.should.deep.equal @doc.ranges + done() + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should not included the deleted", (done) -> + @fetched_docs.length.should.equal 0 + done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + doc.ranges.should.deep.equal @doc.ranges + should.not.exist doc.inS3 + doc.deleted.should.equal true + done() + + describe "a doc with large lines", -> + before (done) -> + @project_id = ObjectId() @timeout 1000 * 30 quarterMegInBytes = 250000 - lines = require("crypto").randomBytes(quarterMegInBytes).toString("hex") - @docs[1].lines = [lines,lines,lines,lines] - @version = 42 - @ranges = [] - DocstoreClient.updateDoc @project_id, @docs[1]._id, @docs[1].lines, @version, @ranges, () => + big_line = require("crypto").randomBytes(quarterMegInBytes).toString("hex") + @doc = { + _id: ObjectId() + lines: [big_line, big_line, big_line, big_line] + ranges: {} + version: 2 + } + DocstoreClient.createDoc @project_id, @doc._id, @doc.lines, @doc.version, @doc.ranges, (error) => + throw error if error? DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? done() - - it "should archive all the docs", (done) -> + + it "should successully archive the docs", (done) -> @res.statusCode.should.equal 204 done() - it "should set inS3 and unset lines in each doc", (done) -> - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - db.docs.findOne _id: archiveDoc._id, (error, doc) => - should.not.exist doc.lines - doc.inS3.should.equal true - callback() - async.series jobs, done + it "should set inS3 and unset lines and ranges in each doc", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + should.not.exist doc.lines + should.not.exist doc.ranges + doc.inS3.should.equal true + done() - it "should be able get the same docs back", (done) -> - jobs = for archiveDoc in @docs - do (archiveDoc) => - (callback) => - DocstoreClient.getS3Doc @project_id, archiveDoc._id, (error, res, doc) => - doc.toString().should.equal archiveDoc.lines.toString() - callback() - async.series jobs, done - - describe "Unarchiving", -> - it "should unarchive all the docs", (done) -> - non_deleted_docs = @docs.slice(0,2) - DocstoreClient.archiveAllDoc @project_id, (error, res) => - DocstoreClient.getAllDocs @project_id, (error, res, docs) => + it "should set the doc in s3 correctly", (done) -> + DocstoreClient.getS3Doc @project_id, @doc._id, (error, res, s3_doc) => + throw error if error? + s3_doc.lines.should.deep.equal @doc.lines + s3_doc.ranges.should.deep.equal @doc.ranges + done() + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => throw error if error? - docs.length.should.equal non_deleted_docs.length - for doc, i in non_deleted_docs - doc.lines.should.deep.equal @docs[i].lines done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + doc.ranges.should.deep.equal @doc.ranges + should.not.exist doc.inS3 + done() + + describe "a doc with naughty strings", -> + before (done) -> + @project_id = ObjectId() + @doc = { + _id: ObjectId() + lines: [ "", "undefined", "undef", "null", "NULL", "(null)", "nil", "NIL", "true", "false", "True", "False", "None", "\\", "\\\\", "0", "1", "1.00", "$1.00", "1/2", "1E2", "1E02", "1E+02", "-1", "-1.00", "-$1.00", "-1/2", "-1E2", "-1E02", "-1E+02", "1/0", "0/0", "-2147483648/-1", "-9223372036854775808/-1", "0.00", "0..0", ".", "0.0.0", "0,00", "0,,0", ",", "0,0,0", "0.0/0", "1.0/0.0", "0.0/0.0", "1,0/0,0", "0,0/0,0", "--1", "-", "-.", "-,", "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", "NaN", "Infinity", "-Infinity", "0x0", "0xffffffff", "0xffffffffffffffff", "0xabad1dea", "123456789012345678901234567890123456789", "1,000.00", "1 000.00", "1'000.00", "1,000,000.00", "1 000 000.00", "1'000'000.00", "1.000,00", "1 000,00", "1'000,00", "1.000.000,00", "1 000i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟", "̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕", "Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮", "˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥", "00˙Ɩ$-", "Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ", "𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠", "𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌", "𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈", "𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰", "𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘", "𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐", "⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", "<script>alert(123)</script>", "<script>alert('123');</script>", "<img src=x onerror=alert(123) />", "<svg><script>123<1>alert(123)</script> ", "\"><script>alert(123)</script>", "'><script>alert(123)</script>", "><script>alert(123)</script>", "</script><script>alert(123)</script>", "< / script >< script >alert(123)< / script >", " onfocus=JaVaSCript:alert(123) autofocus ", "\" onfocus=JaVaSCript:alert(123) autofocus ", "' onfocus=JaVaSCript:alert(123) autofocus ", "＜script＞alert(123)＜/script＞", "<sc<script>ript>alert(123)</sc</script>ript>", "--><script>alert(123)</script>", "\";alert(123);t=\"", "';alert(123);t='", "JavaSCript:alert(123)", ";alert(123);", "src=JaVaSCript:prompt(132)", "\"><script>alert(123);</script x=\"", "'><script>alert(123);</script x='", "><script>alert(123);</script x=", "\" autofocus onkeyup=\"javascript:alert(123)", "' autofocus onkeyup='javascript:alert(123)", "<script\\x20type=\"text/javascript\">javascript:alert(1);</script>", "<script\\x3Etype=\"text/javascript\">javascript:alert(1);</script>", "<script\\x0Dtype=\"text/javascript\">javascript:alert(1);</script>", "<script\\x09type=\"text/javascript\">javascript:alert(1);</script>", "<script\\x0Ctype=\"text/javascript\">javascript:alert(1);</script>", "<script\\x2Ftype=\"text/javascript\">javascript:alert(1);</script>", "<script\\x0Atype=\"text/javascript\">javascript:alert(1);</script>", "'`\"><\\x3Cscript>javascript:alert(1)</script> ", "'`\"><\\x00script>javascript:alert(1)</script>", "ABC<div style=\"x\\x3Aexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:expression\\x5C(javascript:alert(1)\">DEF", "ABC<div style=\"x:expression\\x00(javascript:alert(1)\">DEF", "ABC<div style=\"x:exp\\x00ression(javascript:alert(1)\">DEF", "ABC<div style=\"x:exp\\x5Cression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x0Aexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x09expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE3\\x80\\x80expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x84expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xC2\\xA0expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x80expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x8Aexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x0Dexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x0Cexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x87expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xEF\\xBB\\xBFexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x20expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x88expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x00expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x8Bexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x86expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x85expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x82expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\x0Bexpression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x81expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x83expression(javascript:alert(1)\">DEF", "ABC<div style=\"x:\\xE2\\x80\\x89expression(javascript:alert(1)\">DEF", "<a href=\"\\x0Bjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x0Fjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xC2\\xA0javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x05javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE1\\xA0\\x8Ejavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x18javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x11javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x88javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x89javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x80javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x17javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x03javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x0Ejavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Ajavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x00javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x10javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x82javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x20javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x13javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x09javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x8Ajavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x14javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x19javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\xAFjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Fjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x81javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Djavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x87javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x07javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE1\\x9A\\x80javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x83javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x04javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x01javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x08javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x84javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x86javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE3\\x80\\x80javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x12javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x0Djavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x0Ajavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x0Cjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x15javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\xA8javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x16javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x02javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Bjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x06javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\xA9javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x80\\x85javascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Ejavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\xE2\\x81\\x9Fjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"\\x1Cjavascript:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"javascript\\x00:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"javascript\\x3A:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"javascript\\x09:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"javascript\\x0D:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "<a href=\"javascript\\x0A:javascript:alert(1)\" id=\"fuzzelement1\">test</a>", "`\"'><img src=xxx:x \\x0Aonerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x22onerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x0Bonerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x0Donerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x2Fonerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x09onerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x0Conerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x00onerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x27onerror=javascript:alert(1)>", "`\"'><img src=xxx:x \\x20onerror=javascript:alert(1)>", "\"`'><script>\\x3Bjavascript:alert(1)</script>", "\"`'><script>\\x0Djavascript:alert(1)</script>", "\"`'><script>\\xEF\\xBB\\xBFjavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x81javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x84javascript:alert(1)</script>", "\"`'><script>\\xE3\\x80\\x80javascript:alert(1)</script>", "\"`'><script>\\x09javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x89javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x85javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x88javascript:alert(1)</script>", "\"`'><script>\\x00javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\xA8javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x8Ajavascript:alert(1)</script>", "\"`'><script>\\xE1\\x9A\\x80javascript:alert(1)</script>", "\"`'><script>\\x0Cjavascript:alert(1)</script>", "\"`'><script>\\x2Bjavascript:alert(1)</script>", "\"`'><script>\\xF0\\x90\\x96\\x9Ajavascript:alert(1)</script>", "\"`'><script>-javascript:alert(1)</script>", "\"`'><script>\\x0Ajavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\xAFjavascript:alert(1)</script>", "\"`'><script>\\x7Ejavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x87javascript:alert(1)</script>", "\"`'><script>\\xE2\\x81\\x9Fjavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\xA9javascript:alert(1)</script>", "\"`'><script>\\xC2\\x85javascript:alert(1)</script>", "\"`'><script>\\xEF\\xBF\\xAEjavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x83javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x8Bjavascript:alert(1)</script>", "\"`'><script>\\xEF\\xBF\\xBEjavascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x80javascript:alert(1)</script>", "\"`'><script>\\x21javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x82javascript:alert(1)</script>", "\"`'><script>\\xE2\\x80\\x86javascript:alert(1)</script>", "\"`'><script>\\xE1\\xA0\\x8Ejavascript:alert(1)</script>", "\"`'><script>\\x0Bjavascript:alert(1)</script>", "\"`'><script>\\x20javascript:alert(1)</script>", "\"`'><script>\\xC2\\xA0javascript:alert(1)</script>", "<img \\x00src=x onerror=\"alert(1)\">", "<img \\x47src=x onerror=\"javascript:alert(1)\">", "<img \\x11src=x onerror=\"javascript:alert(1)\">", "<img \\x12src=x onerror=\"javascript:alert(1)\">", "<img\\x47src=x onerror=\"javascript:alert(1)\">", "<img\\x10src=x onerror=\"javascript:alert(1)\">", "<img\\x13src=x onerror=\"javascript:alert(1)\">", "<img\\x32src=x onerror=\"javascript:alert(1)\">", "<img\\x47src=x onerror=\"javascript:alert(1)\">", "<img\\x11src=x onerror=\"javascript:alert(1)\">", "<img \\x47src=x onerror=\"javascript:alert(1)\">", "<img \\x34src=x onerror=\"javascript:alert(1)\">", "<img \\x39src=x onerror=\"javascript:alert(1)\">", "<img \\x00src=x onerror=\"javascript:alert(1)\">", "<img src\\x09=x onerror=\"javascript:alert(1)\">", "<img src\\x10=x onerror=\"javascript:alert(1)\">", "<img src\\x13=x onerror=\"javascript:alert(1)\">", "<img src\\x32=x onerror=\"javascript:alert(1)\">", "<img src\\x12=x onerror=\"javascript:alert(1)\">", "<img src\\x11=x onerror=\"javascript:alert(1)\">", "<img src\\x00=x onerror=\"javascript:alert(1)\">", "<img src\\x47=x onerror=\"javascript:alert(1)\">", "<img src=x\\x09onerror=\"javascript:alert(1)\">", "<img src=x\\x10onerror=\"javascript:alert(1)\">", "<img src=x\\x11onerror=\"javascript:alert(1)\">", "<img src=x\\x12onerror=\"javascript:alert(1)\">", "<img src=x\\x13onerror=\"javascript:alert(1)\">", "<img[a][b][c]src[d]=x[e]onerror=[f]\"alert(1)\">", "<img src=x onerror=\\x09\"javascript:alert(1)\">", "<img src=x onerror=\\x10\"javascript:alert(1)\">", "<img src=x onerror=\\x11\"javascript:alert(1)\">", "<img src=x onerror=\\x12\"javascript:alert(1)\">", "<img src=x onerror=\\x32\"javascript:alert(1)\">", "<img src=x onerror=\\x00\"javascript:alert(1)\">", "<a href=java script:javascript:alert(1)>XXX</a>", "<img src=\"x` `<script>javascript:alert(1)</script>\"` `>", "<img src onerror /\" '\"= alt=javascript:alert(1)//\">", "<title onpropertychange=javascript:alert(1)>", "<a href=http://foo.bar/#x=`y></a><img alt=\"`><img src=x:x onerror=javascript:alert(1)></a>\">", "<!--[if]><script>javascript:alert(1)</script -->", "<!--[if<img src=x onerror=javascript:alert(1)//]> -->", "<script src=\"/\\%(jscript)s\"></script>", "<script src=\"\\\\%(jscript)s\"></script>", "<IMG \"\"\"><SCRIPT>alert(\"XSS\")</SCRIPT>\">", "<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))>", "<IMG SRC=# onmouseover=\"alert('xxs')\">", "<IMG SRC= onmouseover=\"alert('xxs')\">", "<IMG onmouseover=\"alert('xxs')\">", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "perl -e 'print \"<IMG SRC=java\\0script:alert(\\\"XSS\\\")>\";' > out", "<IMG SRC=\"  javascript:alert('XSS');\">", "<SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<BODY onload!#$%&()*~+-_.,:;?@[/|\\]^`=alert(\"XSS\")>", "<SCRIPT/SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<<SCRIPT>alert(\"XSS\");//<</SCRIPT>", "<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >", "<SCRIPT SRC=//ha.ckers.org/.j>", "<IMG SRC=\"javascript:alert('XSS')\"", "<iframe src=http://ha.ckers.org/scriptlet.html <", "\\\";alert('XSS');//", "<plaintext>", "1;DROP TABLE users", "1'; DROP TABLE users-- 1", "' OR 1=1 -- 1", "' OR '1'='1", "-", "--", "--version", "--help", "$USER", "/dev/null; touch /tmp/blns.fail ; echo", "`touch /tmp/blns.fail`", "$(touch /tmp/blns.fail)", "@{[system \"touch /tmp/blns.fail\"]}", "eval(\"puts 'hello world'\")", "System(\"ls -al /\")", "`ls -al /`", "Kernel.exec(\"ls -al /\")", "Kernel.exit(1)", "%x('ls -al /')", "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM \"file:///etc/passwd\" >]><foo>&xxe;</foo>", "$HOME", "$ENV{'HOME'}", "%d", "%s", "%*.*s", "../../../../../../../../../../../etc/passwd%00", "../../../../../../../../../../../etc/hosts", "() { 0; }; touch /tmp/blns.shellshock1.fail;", "() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }", "CON", "PRN", "AUX", "CLOCK$", "NUL", "A:", "ZZ:", "COM1", "LPT1", "LPT2", "LPT3", "COM2", "COM3", "COM4", "Scunthorpe General Hospital", "Penistone Community Church", "Lightwater Country Park", "Jimmy Clitheroe", "Horniman Museum", "shitake mushrooms", "RomansInSussex.co.uk", "http://www.cum.qc.ca/", "Craig Cockburn, Software Specialist", "Linda Callahan", "Dr. Herman I. Libshitz", "magna cum laude", "Super Bowl XXX", "medieval erection of parapets", "evaluate", "mocha", "expression", "Arsenal canal", "classic", "Tyson Gay", "If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.", "Roses are \u001b[0;31mred\u001b[0m, violets are \u001b[0;34mblue. Hope you enjoy terminal hue", "But now...\u001b[20Cfor my greatest trick...\u001b[8m", "The quic\b\b\b\b\b\bk brown fo\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007x... [Beeeep]", "Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗" ] + ranges: {} + version: 2 + } + DocstoreClient.createDoc @project_id, @doc._id, @doc.lines, @doc.version, @doc.ranges, (error) => + throw error if error? + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? + done() + + it "should successully archive the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines and ranges in each doc", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + should.not.exist doc.lines + should.not.exist doc.ranges + doc.inS3.should.equal true + done() + + it "should set the doc in s3 correctly", (done) -> + DocstoreClient.getS3Doc @project_id, @doc._id, (error, res, s3_doc) => + throw error if error? + s3_doc.lines.should.deep.equal @doc.lines + s3_doc.ranges.should.deep.equal @doc.ranges + done() + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + doc.ranges.should.deep.equal @doc.ranges + should.not.exist doc.inS3 + done() + + describe "a doc with ranges", -> + before (done) -> + @project_id = ObjectId() + @doc = { + _id: ObjectId() + lines: ["one", "two", "three"] + ranges: { + changes: + [{ + id : ObjectId(), + op : { "i" : "foo", "p" : 24 } + metadata : { "user_id" : ObjectId(), "ts" : new Date("2017-01-27T16:10:44.194Z") } + }, { + id : ObjectId(), + op : { "d" : "bar", "p" : 50 } + metadata : { "user_id" : ObjectId(), "ts" : new Date("2017-01-27T18:10:44.194Z") } + }] + comments: [{ + id: ObjectId(), + op: { "c" : "comment", "p" : 284, "t" : ObjectId() }, + metadata: { "user_id" : ObjectId(), "ts" : new Date("2017-01-26T14:22:04.869Z") } + }] + } + version: 2 + } + DocstoreClient.createDoc @project_id, @doc._id, @doc.lines, @doc.version, @doc.ranges, (error) => + throw error if error? + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? + done() + + it "should successully archive the docs", (done) -> + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines and ranges in each doc", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + should.not.exist doc.lines + should.not.exist doc.ranges + doc.inS3.should.equal true + done() + + it "should set the doc in s3 correctly", (done) -> + DocstoreClient.getS3Doc @project_id, @doc._id, (error, res, s3_doc) => + throw error if error? + s3_doc.lines.should.deep.equal @doc.lines + ranges = JSON.parse(JSON.stringify(@doc.ranges)) # ObjectId -> String + s3_doc.ranges.should.deep.equal ranges + done() + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + doc.ranges.should.deep.equal @doc.ranges + should.not.exist doc.inS3 + done() + + describe "a doc that is archived twice", -> + before (done) -> + @project_id = ObjectId() + @doc = { + _id: ObjectId() + lines: ["abc", "def", "ghi"] + ranges: {} + version: 2 + } + DocstoreClient.createDoc @project_id, @doc._id, @doc.lines, @doc.version, @doc.ranges, (error) => + throw error if error? + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? + @res.statusCode.should.equal 204 + DocstoreClient.archiveAllDoc @project_id, (error, @res) => + throw error if error? + @res.statusCode.should.equal 204 + done() + + it "should set inS3 and unset lines and ranges in each doc", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + should.not.exist doc.lines + should.not.exist doc.ranges + doc.inS3.should.equal true + done() + + it "should set the doc in s3 correctly", (done) -> + DocstoreClient.getS3Doc @project_id, @doc._id, (error, res, s3_doc) => + throw error if error? + s3_doc.lines.should.deep.equal @doc.lines + s3_doc.ranges.should.deep.equal @doc.ranges + done() + + describe "after unarchiving from a request for the project", -> + before (done) -> + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + doc.ranges.should.deep.equal @doc.ranges + should.not.exist doc.inS3 + done() + + describe "a doc with the old schema (just an array of lines)", -> + before (done) -> + @project_id = ObjectId() + @doc = { + _id: ObjectId() + lines: ["abc", "def", "ghi"] + ranges: {} + version: 2 + } + options = DocArchiveManager.buildS3Options("#{@project_id}/#{@doc._id}") + options.json = @doc.lines + request.put options, (error, res, body) => + throw error if error? + res.statusCode.should.equal 200 + db.docs.insert { + project_id: @project_id + _id: @doc._id + rev: @doc.version + inS3: true + }, (error) => + throw error if error? + DocstoreClient.getAllDocs @project_id, (error, res, @fetched_docs) => + throw error if error? + done() + + it "should restore the doc to mongo", (done) -> + db.docs.findOne _id: @doc._id, (error, doc) => + throw error if error? + doc.lines.should.deep.equal @doc.lines + should.not.exist doc.inS3 + done() + + it "should return the doc", (done) -> + @fetched_docs[0].lines.should.deep.equal @doc.lines + done() diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 05d8069d69..755f12d9b0 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -48,5 +48,6 @@ module.exports = DocstoreClient = getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> - options = DocArchiveManager.buildS3Options(true, project_id+"/"+doc_id) - request.get options, callback + options = DocArchiveManager.buildS3Options(project_id+"/"+doc_id) + options.json = true + request.get options, callback diff --git a/services/docstore/test/unit/coffee/DocArchiveManager.coffee b/services/docstore/test/unit/coffee/DocArchiveManager.coffee index cac4594dd2..baa5e57bb1 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManager.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManager.coffee @@ -71,6 +71,7 @@ describe "DocArchiveManager", -> "settings-sharelatex": @settings "./MongoManager": @MongoManager "request": @request + "./RangeManager": @RangeManager = {} "logger-sharelatex": log:-> err:-> @@ -87,13 +88,22 @@ describe "DocArchiveManager", -> @DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=> opts = @request.put.args[0][0] assert.deepEqual(opts.aws, {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket}) - opts.json.should.equal @mongoDocs[0].lines + opts.body.should.equal JSON.stringify( + lines: @mongoDocs[0].lines + ranges: @mongoDocs[0].ranges + schema_v: 1 + ) opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{@mongoDocs[0]._id}" done() it "should return no md5 error", (done)-> - @md5 = crypto.createHash("md5").update(JSON.stringify(@mongoDocs[0].lines)).digest("hex") + data = JSON.stringify( + lines: @mongoDocs[0].lines + ranges: @mongoDocs[0].ranges + schema_v: 1 + ) + @md5 = crypto.createHash("md5").update(data).digest("hex") @request.put = sinon.stub().callsArgWith(1, null, {statusCode:200,headers:{etag:@md5}}) @DocArchiveManager.archiveDoc @project_id, @mongoDocs[0], (err)=> should.not.exist err From bf187f30ecf6b82b1d4275c6cde66a06dcda3473 Mon Sep 17 00:00:00 2001 From: James Allen <james@sharelatex.com> Date: Mon, 3 Apr 2017 15:50:33 +0100 Subject: [PATCH 2/3] Add tests for _s3DocToMongo --- .../unit/coffee/DocArchiveManagerTests.coffee | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee index c6c9eacea2..81ed38d177 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee @@ -212,3 +212,45 @@ describe "DocArchiveManager", -> @DocArchiveManager.unArchiveAllDocs @project_id, (err)=> err.should.equal @error done() + + describe "_s3DocToMongoDoc", -> + describe "with the old schema", -> + it "should return the docs lines", (done) -> + @DocArchiveManager._s3DocToMongoDoc ["doc", "lines"], (error, doc) -> + expect(doc).to.deep.equal { + lines: ["doc", "lines"] + } + done() + + describe "with the new schema", -> + it "should return the doc lines and ranges", (done) -> + @RangeManager.jsonRangesToMongo = sinon.stub().returns {"mongo": "ranges"} + @DocArchiveManager._s3DocToMongoDoc { + lines: ["doc", "lines"] + ranges: {"json": "ranges"} + schema_v: 1 + }, (error, doc) -> + expect(doc).to.deep.equal { + lines: ["doc", "lines"] + ranges: {"mongo": "ranges"} + } + done() + + it "should return just the doc lines when there are no ranges", (done) -> + @DocArchiveManager._s3DocToMongoDoc { + lines: ["doc", "lines"] + schema_v: 1 + }, (error, doc) -> + expect(doc).to.deep.equal { + lines: ["doc", "lines"] + } + done() + + describe "with an unrecognised schema", -> + it "should return an error", (done) -> + @DocArchiveManager._s3DocToMongoDoc { + schema_v: 2 + }, (error, doc) -> + expect(error).to.exist + done() + From 7a64f040f196814bd39f3e3e5a7e6df3c52638c8 Mon Sep 17 00:00:00 2001 From: James Allen <james@sharelatex.com> Date: Mon, 3 Apr 2017 16:04:54 +0100 Subject: [PATCH 3/3] Test and fix _mongoDocToS3Doc --- .../app/coffee/DocArchiveManager.coffee | 4 +- .../unit/coffee/DocArchiveManagerTests.coffee | 41 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/services/docstore/app/coffee/DocArchiveManager.coffee b/services/docstore/app/coffee/DocArchiveManager.coffee index 9c16d13d67..12f4b47c82 100644 --- a/services/docstore/app/coffee/DocArchiveManager.coffee +++ b/services/docstore/app/coffee/DocArchiveManager.coffee @@ -98,6 +98,8 @@ module.exports = DocArchive = return callback null, mongo_doc _mongoDocToS3Doc: (doc, callback = (error, s3_doc) ->) -> + if !doc.lines? + return callback(new Error("doc has no lines")) json = JSON.stringify({ lines: doc.lines ranges: doc.ranges @@ -105,7 +107,7 @@ module.exports = DocArchive = }) if json.indexOf("\u0000") != -1 error = new Error("null bytes detected") - logger.error {err: error, project_id, doc_id}, error.message + logger.err {err: error, doc, json}, error.message return callback(error) return callback null, json diff --git a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee index 81ed38d177..2b9287d912 100644 --- a/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee +++ b/services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee @@ -75,11 +75,13 @@ describe "DocArchiveManager", -> "logger-sharelatex": log:-> err:-> + @globals = + JSON: JSON @error = "my errror" @project_id = ObjectId().toString() @stubbedError = new Errors.NotFoundError("Error in S3 request") - @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires + @DocArchiveManager = SandboxedModule.require modulePath, requires: @requires, globals: @globals describe "archiveDoc", -> @@ -253,4 +255,41 @@ describe "DocArchiveManager", -> }, (error, doc) -> expect(error).to.exist done() + + describe "_mongoDocToS3Doc", -> + describe "with a valid doc", -> + it "should return the json version", (done) -> + @DocArchiveManager._mongoDocToS3Doc doc = { + lines: ["doc", "lines"] + ranges: { "mock": "ranges" } + }, (err, s3_doc) -> + expect(s3_doc).to.equal JSON.stringify({ + lines: ["doc", "lines"] + ranges: { "mock": "ranges" } + schema_v: 1 + }) + done() + + describe "with null bytes in the result", -> + beforeEach -> + @_stringify = JSON.stringify + JSON.stringify = sinon.stub().returns '{"bad": "\u0000"}' + + afterEach -> + JSON.stringify = @_stringify + + it "should return an error", (done) -> + @DocArchiveManager._mongoDocToS3Doc { + lines: ["doc", "lines"] + ranges: { "mock": "ranges" } + }, (err, s3_doc) -> + expect(err).to.exist + done() + + describe "without doc lines", -> + it "should return an error", (done) -> + @DocArchiveManager._mongoDocToS3Doc {}, (err, s3_doc) -> + expect(err).to.exist + done() +