DEF',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '
test',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '`"\'>
',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '"`\'>',
+ '
![]()
',
+ '
![]()
',
+ '
![]()
',
+ '
![]()
',
+ '
',
+ '',
+ '',
+ '',
+ '',
+ '',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![]()
',
+ '![](x\\x09onerror="javascript:alert(1)")
',
+ '![](x\\x10onerror="javascript:alert(1)")
',
+ '![](x\\x11onerror="javascript:alert(1)")
',
+ '![](x\\x12onerror="javascript:alert(1)")
',
+ '![](x\\x13onerror="javascript:alert(1)")
',
+ '',
+ '
',
+ '',
+ '',
+ '',
+ '',
+ '',
+ 'XXX',
+ '</script>)
',
+ '![]()
',
+ '',
+ '<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(1)></a>">',
+ '<!--[if]><script>javascript:alert(1)</script -->',
+ '<!--[if<img src=x onerror=javascript:alert(1)//]> -->',
+ '<script src="/\\%(jscript)s"></script>',
+ '<script src="\\\\%(jscript)s"></script>',
+ '<IMG """><SCRIPT>alert("XSS")</SCRIPT>">',
+ '<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))>',
+ '<IMG SRC=# onmouseover="alert(\'xxs\')">',
+ '<IMG SRC= onmouseover="alert(\'xxs\')">',
+ '<IMG onmouseover="alert(\'xxs\')">',
+ '<IMG SRC=javascript:alert('XSS')>',
+ '<IMG SRC=javascript:alert('XSS')>',
+ '<IMG SRC=javascript:alert('XSS')>',
+ '<IMG SRC="jav ascript:alert(\'XSS\');">',
+ '<IMG SRC="jav ascript:alert(\'XSS\');">',
+ '<IMG SRC="jav
ascript:alert(\'XSS\');">',
+ '<IMG SRC="jav
ascript:alert(\'XSS\');">',
+ 'perl -e \'print "<IMG SRC=java\\0script:alert(\\"XSS\\")>";\' > out',
+ '<IMG SRC=" javascript:alert(\'XSS\');">',
+ '<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT>',
+ '<BODY onload!#$%&()*~+-_.,:;?@[/|\\]^`=alert("XSS")>',
+ '<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT>',
+ '<<SCRIPT>alert("XSS");//<</SCRIPT>',
+ '<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >',
+ '<SCRIPT SRC=//ha.ckers.org/.j>',
+ '<IMG SRC="javascript:alert(\'XSS\')"',
+ '<iframe src=http://ha.ckers.org/scriptlet.html <',
+ "\\\";alert('XSS');//",
+ '<plaintext>',
+ '1;DROP TABLE users',
+ "1'; DROP TABLE users-- 1",
+ "' OR 1=1 -- 1",
+ "' OR '1'='1",
+ '-',
+ '--',
+ '--version',
+ '--help',
+ '$USER',
+ '/dev/null; touch /tmp/blns.fail ; echo',
+ '`touch /tmp/blns.fail`',
+ '$(touch /tmp/blns.fail)',
+ '@{[system "touch /tmp/blns.fail"]}',
+ 'eval("puts \'hello world\'")',
+ 'System("ls -al /")',
+ '`ls -al /`',
+ 'Kernel.exec("ls -al /")',
+ 'Kernel.exit(1)',
+ "%x('ls -al /')",
+ '<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo>',
+ '$HOME',
+ "$ENV{'HOME'}",
+ '%d',
+ '%s',
+ '%*.*s',
+ '../../../../../../../../../../../etc/passwd%00',
+ '../../../../../../../../../../../etc/hosts',
+ '() { 0; }; touch /tmp/blns.shellshock1.fail;',
+ '() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }',
+ 'CON',
+ 'PRN',
+ 'AUX',
+ 'CLOCK$',
+ 'NUL',
+ 'A:',
+ 'ZZ:',
+ 'COM1',
+ 'LPT1',
+ 'LPT2',
+ 'LPT3',
+ 'COM2',
+ 'COM3',
+ 'COM4',
+ 'Scunthorpe General Hospital',
+ 'Penistone Community Church',
+ 'Lightwater Country Park',
+ 'Jimmy Clitheroe',
+ 'Horniman Museum',
+ 'shitake mushrooms',
+ 'RomansInSussex.co.uk',
+ 'http://www.cum.qc.ca/',
+ 'Craig Cockburn, Software Specialist',
+ 'Linda Callahan',
+ 'Dr. Herman I. Libshitz',
+ 'magna cum laude',
+ 'Super Bowl XXX',
+ 'medieval erection of parapets',
+ 'evaluate',
+ 'mocha',
+ 'expression',
+ 'Arsenal canal',
+ 'classic',
+ 'Tyson Gay',
+ "If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.",
+ 'Roses are \u001b[0;31mred\u001b[0m, violets are \u001b[0;34mblue. Hope you enjoy terminal hue',
+ 'But now...\u001b[20Cfor my greatest trick...\u001b[8m',
+ 'The quic\b\b\b\b\b\bk brown fo\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007x... [Beeeep]',
+ 'Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗',
+ ],
+ ranges: {},
+ version: 2,
+ }
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc._id,
+ this.doc.lines,
+ this.doc.version,
+ this.doc.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+ this.res = res
+ if (error != null) {
+ throw error
+ }
+ return done()
+ })
+ }
+ )
+ })
+
+ it('should successully archive the docs', function (done) {
+ this.res.statusCode.should.equal(204)
+ return done()
+ })
+
+ it('should set inS3 and unset lines and ranges in each doc', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ expect(doc.lines).not.to.exist
+ expect(doc.ranges).not.to.exist
+ doc.inS3.should.equal(true)
+ return done()
+ })
+ })
+
+ it('should set the doc in s3 correctly', function (done) {
+ return DocstoreClient.getS3Doc(
+ this.project_id,
+ this.doc._id,
+ (error, s3_doc) => {
+ if (error != null) {
+ throw error
+ }
+ s3_doc.lines.should.deep.equal(this.doc.lines)
+ s3_doc.ranges.should.deep.equal(this.doc.ranges)
+ return done()
+ }
+ )
+ })
+
+ return describe('after unarchiving from a request for the project', function () {
+ before(function (done) {
+ return DocstoreClient.getAllDocs(
+ this.project_id,
+ (error, res, fetched_docs) => {
+ this.fetched_docs = fetched_docs
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+
+ return it('should restore the doc to mongo', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ expect(doc.inS3).not.to.exist
+ return done()
+ })
+ })
+ })
+ })
+
+ describe('a doc with ranges', function () {
+ before(function (done) {
+ this.project_id = ObjectId()
+ this.doc = {
+ _id: ObjectId(),
+ lines: ['one', 'two', 'three'],
+ ranges: {
+ changes: [
+ {
+ id: ObjectId(),
+ op: { i: 'foo', p: 24 },
+ metadata: {
+ user_id: ObjectId(),
+ ts: new Date('2017-01-27T16:10:44.194Z'),
+ },
+ },
+ {
+ id: ObjectId(),
+ op: { d: 'bar', p: 50 },
+ metadata: {
+ user_id: ObjectId(),
+ ts: new Date('2017-01-27T18:10:44.194Z'),
+ },
+ },
+ ],
+ comments: [
+ {
+ id: ObjectId(),
+ op: { c: 'comment', p: 284, t: ObjectId() },
+ metadata: {
+ user_id: ObjectId(),
+ ts: new Date('2017-01-26T14:22:04.869Z'),
+ },
+ },
+ ],
+ },
+ version: 2,
+ }
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc._id,
+ this.doc.lines,
+ this.doc.version,
+ this.doc.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+ this.res = res
+ if (error != null) {
+ throw error
+ }
+ return done()
+ })
+ }
+ )
+ })
+
+ it('should successully archive the docs', function (done) {
+ this.res.statusCode.should.equal(204)
+ return done()
+ })
+
+ it('should set inS3 and unset lines and ranges in each doc', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ expect(doc.lines).not.to.exist
+ expect(doc.ranges).not.to.exist
+ doc.inS3.should.equal(true)
+ return done()
+ })
+ })
+
+ it('should set the doc in s3 correctly', function (done) {
+ return DocstoreClient.getS3Doc(
+ this.project_id,
+ this.doc._id,
+ (error, s3_doc) => {
+ if (error != null) {
+ throw error
+ }
+ s3_doc.lines.should.deep.equal(this.doc.lines)
+ const ranges = JSON.parse(JSON.stringify(this.doc.ranges)) // ObjectId -> String
+ s3_doc.ranges.should.deep.equal(ranges)
+ return done()
+ }
+ )
+ })
+
+ return describe('after unarchiving from a request for the project', function () {
+ before(function (done) {
+ return DocstoreClient.getAllDocs(
+ this.project_id,
+ (error, res, fetched_docs) => {
+ this.fetched_docs = fetched_docs
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+
+ return it('should restore the doc to mongo', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ expect(doc.inS3).not.to.exist
+ return done()
+ })
+ })
+ })
+ })
+
+ describe('a doc that is archived twice', function () {
+ before(function (done) {
+ this.project_id = ObjectId()
+ this.doc = {
+ _id: ObjectId(),
+ lines: ['abc', 'def', 'ghi'],
+ ranges: {},
+ version: 2,
+ }
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc._id,
+ this.doc.lines,
+ this.doc.version,
+ this.doc.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return DocstoreClient.archiveAllDoc(this.project_id, (error, res) => {
+ this.res = res
+ if (error != null) {
+ throw error
+ }
+ this.res.statusCode.should.equal(204)
+ return DocstoreClient.archiveAllDoc(
+ this.project_id,
+ (error, res1) => {
+ this.res = res1
+ if (error != null) {
+ throw error
+ }
+ this.res.statusCode.should.equal(204)
+ return done()
+ }
+ )
+ })
+ }
+ )
+ })
+
+ it('should set inS3 and unset lines and ranges in each doc', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ expect(doc.lines).not.to.exist
+ expect(doc.ranges).not.to.exist
+ doc.inS3.should.equal(true)
+ return done()
+ })
+ })
+
+ it('should set the doc in s3 correctly', function (done) {
+ return DocstoreClient.getS3Doc(
+ this.project_id,
+ this.doc._id,
+ (error, s3_doc) => {
+ if (error != null) {
+ throw error
+ }
+ s3_doc.lines.should.deep.equal(this.doc.lines)
+ s3_doc.ranges.should.deep.equal(this.doc.ranges)
+ return done()
+ }
+ )
+ })
+
+ return describe('after unarchiving from a request for the project', function () {
+ before(function (done) {
+ return DocstoreClient.getAllDocs(
+ this.project_id,
+ (error, res, fetched_docs) => {
+ this.fetched_docs = fetched_docs
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+
+ return it('should restore the doc to mongo', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ expect(doc.inS3).not.to.exist
+ return done()
+ })
+ })
+ })
+ })
+
+ return describe('a doc with the old schema (just an array of lines)', function () {
+ before(function (done) {
+ this.project_id = ObjectId()
+ this.doc = {
+ _id: ObjectId(),
+ lines: ['abc', 'def', 'ghi'],
+ ranges: {},
+ version: 2,
+ }
+ uploadContent(
+ `${this.project_id}/${this.doc._id}`,
+ this.doc.lines,
+ error => {
+ expect(error).not.to.exist
+ db.docs.insert(
+ {
+ project_id: this.project_id,
+ _id: this.doc._id,
+ rev: this.doc.version,
+ inS3: true,
+ },
+ error => {
+ if (error != null) {
+ throw error
+ }
+ DocstoreClient.getAllDocs(
+ this.project_id,
+ (error, res, fetched_docs) => {
+ this.fetched_docs = fetched_docs
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ }
+ )
+ }
+ )
+ })
+
+ it('should restore the doc to mongo', function (done) {
+ return db.docs.findOne({ _id: this.doc._id }, (error, doc) => {
+ if (error != null) {
+ throw error
+ }
+ doc.lines.should.deep.equal(this.doc.lines)
+ expect(doc.inS3).not.to.exist
+ return done()
+ })
+ })
+
+ return it('should return the doc', function (done) {
+ this.fetched_docs[0].lines.should.deep.equal(this.doc.lines)
+ return done()
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/DeletingDocsTests.js b/services/docstore/test/acceptance/js/DeletingDocsTests.js
new file mode 100644
index 0000000000..4e9e987958
--- /dev/null
+++ b/services/docstore/test/acceptance/js/DeletingDocsTests.js
@@ -0,0 +1,470 @@
+/* eslint-disable
+ camelcase,
+ handle-callback-err,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const { db, ObjectId } = require('../../../app/js/mongodb')
+const { expect } = require('chai')
+const DocstoreApp = require('./helpers/DocstoreApp')
+const Errors = require('../../../app/js/Errors')
+const Settings = require('@overleaf/settings')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+function deleteTestSuite(deleteDoc) {
+ beforeEach(function (done) {
+ this.project_id = ObjectId()
+ this.doc_id = ObjectId()
+ this.lines = ['original', 'lines']
+ this.version = 42
+ this.ranges = []
+ return DocstoreApp.ensureRunning(() => {
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc_id,
+ this.lines,
+ this.version,
+ this.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+ })
+
+ it('should show as not deleted on /deleted', function (done) {
+ DocstoreClient.isDocDeleted(
+ this.project_id,
+ this.doc_id,
+ (error, res, body) => {
+ if (error) return done(error)
+ expect(res.statusCode).to.equal(200)
+ expect(body).to.have.property('deleted').to.equal(false)
+ done()
+ }
+ )
+ })
+
+ describe('when the doc exists', function () {
+ beforeEach(function (done) {
+ deleteDoc(this.project_id, this.doc_id, (error, res, doc) => {
+ this.res = res
+ return done()
+ })
+ })
+
+ afterEach(function (done) {
+ return db.docs.remove({ _id: this.doc_id }, done)
+ })
+
+ it('should mark the doc as deleted on /deleted', function (done) {
+ DocstoreClient.isDocDeleted(
+ this.project_id,
+ this.doc_id,
+ (error, res, body) => {
+ if (error) return done(error)
+ expect(res.statusCode).to.equal(200)
+ expect(body).to.have.property('deleted').to.equal(true)
+ done()
+ }
+ )
+ })
+
+ it('should insert a deleted doc into the docs collection', function (done) {
+ return db.docs.find({ _id: this.doc_id }).toArray((error, docs) => {
+ docs[0]._id.should.deep.equal(this.doc_id)
+ docs[0].lines.should.deep.equal(this.lines)
+ docs[0].deleted.should.equal(true)
+ return done()
+ })
+ })
+
+ it('should not export the doc to s3', function (done) {
+ setTimeout(() => {
+ DocstoreClient.getS3Doc(this.project_id, this.doc_id, error => {
+ expect(error).to.be.instanceOf(Errors.NotFoundError)
+ done()
+ })
+ }, 1000)
+ })
+ })
+
+ describe('when archiveOnSoftDelete is enabled', function () {
+ let archiveOnSoftDelete
+ beforeEach('overwrite settings', function () {
+ archiveOnSoftDelete = Settings.docstore.archiveOnSoftDelete
+ Settings.docstore.archiveOnSoftDelete = true
+ })
+ afterEach('restore settings', function () {
+ Settings.docstore.archiveOnSoftDelete = archiveOnSoftDelete
+ })
+
+ beforeEach('delete Doc', function (done) {
+ deleteDoc(this.project_id, this.doc_id, (error, res) => {
+ this.res = res
+ done()
+ })
+ })
+
+ beforeEach(function waitForBackgroundFlush(done) {
+ setTimeout(done, 500)
+ })
+
+ afterEach(function cleanupDoc(done) {
+ db.docs.remove({ _id: this.doc_id }, done)
+ })
+
+ it('should set the deleted flag in the doc', function (done) {
+ db.docs.findOne({ _id: this.doc_id }, (error, doc) => {
+ if (error) {
+ return done(error)
+ }
+ expect(doc.deleted).to.equal(true)
+ done()
+ })
+ })
+
+ it('should set inS3 and unset lines and ranges in the doc', function (done) {
+ db.docs.findOne({ _id: this.doc_id }, (error, doc) => {
+ if (error) {
+ return done(error)
+ }
+ expect(doc.lines).to.not.exist
+ expect(doc.ranges).to.not.exist
+ expect(doc.inS3).to.equal(true)
+ done()
+ })
+ })
+
+ it('should set the doc in s3 correctly', function (done) {
+ DocstoreClient.getS3Doc(this.project_id, this.doc_id, (error, s3_doc) => {
+ if (error) {
+ return done(error)
+ }
+ expect(s3_doc.lines).to.deep.equal(this.lines)
+ expect(s3_doc.ranges).to.deep.equal(this.ranges)
+ done()
+ })
+ })
+ })
+
+ describe('when the doc exists in another project', function () {
+ const otherProjectId = ObjectId()
+
+ it('should show as not existing on /deleted', function (done) {
+ DocstoreClient.isDocDeleted(otherProjectId, this.doc_id, (error, res) => {
+ if (error) return done(error)
+ expect(res.statusCode).to.equal(404)
+ done()
+ })
+ })
+
+ it('should return a 404 when trying to delete', function (done) {
+ deleteDoc(otherProjectId, this.doc_id, (error, res) => {
+ if (error) return done(error)
+ expect(res.statusCode).to.equal(404)
+ done()
+ })
+ })
+ })
+
+ return describe('when the doc does not exist', function () {
+ it('should show as not existing on /deleted', function (done) {
+ const missing_doc_id = ObjectId()
+ DocstoreClient.isDocDeleted(
+ this.project_id,
+ missing_doc_id,
+ (error, res) => {
+ if (error) return done(error)
+ expect(res.statusCode).to.equal(404)
+ done()
+ }
+ )
+ })
+
+ return it('should return a 404', function (done) {
+ const missing_doc_id = ObjectId()
+ deleteDoc(this.project_id, missing_doc_id, (error, res, doc) => {
+ res.statusCode.should.equal(404)
+ return done()
+ })
+ })
+ })
+}
+
+describe('Delete via PATCH', function () {
+ deleteTestSuite(DocstoreClient.deleteDoc)
+
+ describe('when providing a custom doc name in the delete request', function () {
+ beforeEach(function (done) {
+ DocstoreClient.deleteDocWithName(
+ this.project_id,
+ this.doc_id,
+ 'wombat.tex',
+ done
+ )
+ })
+
+ it('should insert the doc name into the docs collection', function (done) {
+ db.docs.find({ _id: this.doc_id }).toArray((error, docs) => {
+ if (error) return done(error)
+ expect(docs[0].name).to.equal('wombat.tex')
+ done()
+ })
+ })
+ })
+
+ describe('when providing a custom deletedAt date in the delete request', function () {
+ beforeEach('record date and delay', function (done) {
+ this.deletedAt = new Date()
+ setTimeout(done, 5)
+ })
+
+ beforeEach('perform deletion with past date', function (done) {
+ DocstoreClient.deleteDocWithDate(
+ this.project_id,
+ this.doc_id,
+ this.deletedAt,
+ done
+ )
+ })
+
+ it('should insert the date into the docs collection', function (done) {
+ db.docs.find({ _id: this.doc_id }).toArray((error, docs) => {
+ if (error) return done(error)
+ expect(docs[0].deletedAt.toISOString()).to.equal(
+ this.deletedAt.toISOString()
+ )
+ done()
+ })
+ })
+ })
+
+ describe('when providing no doc name in the delete request', function () {
+ beforeEach(function (done) {
+ DocstoreClient.deleteDocWithName(
+ this.project_id,
+ this.doc_id,
+ '',
+ (error, res) => {
+ this.res = res
+ done(error)
+ }
+ )
+ })
+
+ it('should reject the request', function () {
+ expect(this.res.statusCode).to.equal(400)
+ })
+ })
+
+ describe('when providing no date in the delete request', function () {
+ beforeEach(function (done) {
+ DocstoreClient.deleteDocWithDate(
+ this.project_id,
+ this.doc_id,
+ '',
+ (error, res) => {
+ this.res = res
+ done(error)
+ }
+ )
+ })
+
+ it('should reject the request', function () {
+ expect(this.res.statusCode).to.equal(400)
+ })
+ })
+
+ describe('before deleting anything', function () {
+ it('should show nothing in deleted docs response', function (done) {
+ DocstoreClient.getAllDeletedDocs(
+ this.project_id,
+ (error, deletedDocs) => {
+ if (error) return done(error)
+ expect(deletedDocs).to.deep.equal([])
+ done()
+ }
+ )
+ })
+ })
+
+ describe('when the doc gets a name on delete', function () {
+ beforeEach(function (done) {
+ DocstoreClient.deleteDoc(this.project_id, this.doc_id, done)
+ })
+
+ it('should show the doc in deleted docs response', function (done) {
+ DocstoreClient.getAllDeletedDocs(
+ this.project_id,
+ (error, deletedDocs) => {
+ if (error) return done(error)
+ expect(deletedDocs).to.deep.equal([
+ { _id: this.doc_id.toString(), name: 'main.tex' },
+ ])
+ done()
+ }
+ )
+ })
+
+ describe('after deleting multiple docs', function () {
+ beforeEach('create doc2', function (done) {
+ this.doc_id2 = ObjectId()
+ DocstoreClient.createDoc(
+ this.project_id,
+ this.doc_id2,
+ this.lines,
+ this.version,
+ this.ranges,
+ done
+ )
+ })
+ beforeEach('delete doc2', function (done) {
+ DocstoreClient.deleteDocWithName(
+ this.project_id,
+ this.doc_id2,
+ 'two.tex',
+ done
+ )
+ })
+ beforeEach('create doc3', function (done) {
+ this.doc_id3 = ObjectId()
+ DocstoreClient.createDoc(
+ this.project_id,
+ this.doc_id3,
+ this.lines,
+ this.version,
+ this.ranges,
+ done
+ )
+ })
+ beforeEach('delete doc3', function (done) {
+ DocstoreClient.deleteDocWithName(
+ this.project_id,
+ this.doc_id3,
+ 'three.tex',
+ done
+ )
+ })
+ it('should show all the docs as deleted', function (done) {
+ DocstoreClient.getAllDeletedDocs(
+ this.project_id,
+ (error, deletedDocs) => {
+ if (error) return done(error)
+
+ expect(deletedDocs).to.deep.equal([
+ { _id: this.doc_id3.toString(), name: 'three.tex' },
+ { _id: this.doc_id2.toString(), name: 'two.tex' },
+ { _id: this.doc_id.toString(), name: 'main.tex' },
+ ])
+ done()
+ }
+ )
+ })
+
+ describe('with one more than max_deleted_docs permits', function () {
+ let maxDeletedDocsBefore
+ beforeEach(function () {
+ maxDeletedDocsBefore = Settings.max_deleted_docs
+ Settings.max_deleted_docs = 2
+ })
+ afterEach(function () {
+ Settings.max_deleted_docs = maxDeletedDocsBefore
+ })
+
+ it('should omit the first deleted doc', function (done) {
+ DocstoreClient.getAllDeletedDocs(
+ this.project_id,
+ (error, deletedDocs) => {
+ if (error) return done(error)
+
+ expect(deletedDocs).to.deep.equal([
+ { _id: this.doc_id3.toString(), name: 'three.tex' },
+ { _id: this.doc_id2.toString(), name: 'two.tex' },
+ // dropped main.tex
+ ])
+ done()
+ }
+ )
+ })
+ })
+ })
+ })
+})
+
+describe("Destroying a project's documents", function () {
+ describe('when the doc exists', function () {
+ beforeEach(function (done) {
+ return db.docOps.insert(
+ { doc_id: ObjectId(this.doc_id), version: 1 },
+ function (err) {
+ if (err != null) {
+ return done(err)
+ }
+ return DocstoreClient.destroyAllDoc(this.project_id, done)
+ }
+ )
+ })
+
+ it('should remove the doc from the docs collection', function (done) {
+ return db.docs.find({ _id: this.doc_id }).toArray((err, docs) => {
+ expect(err).not.to.exist
+ expect(docs).to.deep.equal([])
+ return done()
+ })
+ })
+
+ return it('should remove the docOps from the docOps collection', function (done) {
+ return db.docOps.find({ doc_id: this.doc_id }).toArray((err, docOps) => {
+ expect(err).not.to.exist
+ expect(docOps).to.deep.equal([])
+ return done()
+ })
+ })
+ })
+
+ return describe('when the doc is archived', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.archiveAllDoc(this.project_id, err => {
+ if (err != null) {
+ return done(err)
+ }
+ return DocstoreClient.destroyAllDoc(this.project_id, done)
+ })
+ })
+
+ it('should remove the doc from the docs collection', function (done) {
+ return db.docs.find({ _id: this.doc_id }).toArray((err, docs) => {
+ expect(err).not.to.exist
+ expect(docs).to.deep.equal([])
+ return done()
+ })
+ })
+
+ it('should remove the docOps from the docOps collection', function (done) {
+ return db.docOps.find({ doc_id: this.doc_id }).toArray((err, docOps) => {
+ expect(err).not.to.exist
+ expect(docOps).to.deep.equal([])
+ return done()
+ })
+ })
+
+ return it('should remove the doc contents from s3', function (done) {
+ return DocstoreClient.getS3Doc(this.project_id, this.doc_id, error => {
+ expect(error).to.be.instanceOf(Errors.NotFoundError)
+ done()
+ })
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/GettingAllDocsTests.js b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
new file mode 100644
index 0000000000..76b3d70b78
--- /dev/null
+++ b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
@@ -0,0 +1,112 @@
+/* eslint-disable
+ handle-callback-err,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const { ObjectId } = require('mongodb')
+const async = require('async')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Getting all docs', function () {
+ beforeEach(function (done) {
+ this.project_id = ObjectId()
+ this.docs = [
+ {
+ _id: ObjectId(),
+ lines: ['one', 'two', 'three'],
+ ranges: { mock: 'one' },
+ rev: 2,
+ },
+ {
+ _id: ObjectId(),
+ lines: ['aaa', 'bbb', 'ccc'],
+ ranges: { mock: 'two' },
+ rev: 4,
+ },
+ {
+ _id: ObjectId(),
+ lines: ['111', '222', '333'],
+ ranges: { mock: 'three' },
+ rev: 6,
+ },
+ ]
+ this.deleted_doc = {
+ _id: ObjectId(),
+ lines: ['deleted'],
+ ranges: { mock: 'four' },
+ rev: 8,
+ }
+ const version = 42
+ const jobs = Array.from(this.docs).map(doc =>
+ (doc => {
+ return callback => {
+ return DocstoreClient.createDoc(
+ this.project_id,
+ doc._id,
+ doc.lines,
+ version,
+ doc.ranges,
+ callback
+ )
+ }
+ })(doc)
+ )
+ jobs.push(cb => {
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.deleted_doc._id,
+ this.deleted_doc.lines,
+ version,
+ this.deleted_doc.ranges,
+ err => {
+ return DocstoreClient.deleteDoc(
+ this.project_id,
+ this.deleted_doc._id,
+ cb
+ )
+ }
+ )
+ })
+ jobs.unshift(cb => DocstoreApp.ensureRunning(cb))
+ return async.series(jobs, done)
+ })
+
+ it('getAllDocs should return all the (non-deleted) docs', function (done) {
+ return DocstoreClient.getAllDocs(this.project_id, (error, res, docs) => {
+ if (error != null) {
+ throw error
+ }
+ docs.length.should.equal(this.docs.length)
+ for (let i = 0; i < docs.length; i++) {
+ const doc = docs[i]
+ doc.lines.should.deep.equal(this.docs[i].lines)
+ }
+ return done()
+ })
+ })
+
+ return it('getAllRanges should return all the (non-deleted) doc ranges', function (done) {
+ return DocstoreClient.getAllRanges(this.project_id, (error, res, docs) => {
+ if (error != null) {
+ throw error
+ }
+ docs.length.should.equal(this.docs.length)
+ for (let i = 0; i < docs.length; i++) {
+ const doc = docs[i]
+ doc.ranges.should.deep.equal(this.docs[i].ranges)
+ }
+ return done()
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js b/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js
new file mode 100644
index 0000000000..4dddbb862d
--- /dev/null
+++ b/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js
@@ -0,0 +1,127 @@
+const Settings = require('@overleaf/settings')
+const { ObjectId } = require('../../../app/js/mongodb')
+const DocstoreApp = require('./helpers/DocstoreApp')
+const DocstoreClient = require('./helpers/DocstoreClient')
+const { Storage } = require('@google-cloud/storage')
+
+describe('Getting A Doc from Archive', function () {
+ before(function (done) {
+ return DocstoreApp.ensureRunning(done)
+ })
+
+ before(async function () {
+ const storage = new Storage(Settings.docstore.gcs.endpoint)
+ await storage.createBucket(Settings.docstore.bucket)
+ await storage.createBucket(`${Settings.docstore.bucket}-deleted`)
+ })
+
+ describe('for an archived doc', function () {
+ before(function (done) {
+ this.project_id = ObjectId()
+ this.timeout(1000 * 30)
+ this.doc = {
+ _id: ObjectId(),
+ lines: ['foo', 'bar'],
+ ranges: {},
+ version: 2,
+ }
+ DocstoreClient.createDoc(
+ this.project_id,
+ this.doc._id,
+ this.doc.lines,
+ this.doc.version,
+ this.doc.ranges,
+ error => {
+ if (error) {
+ return done(error)
+ }
+ DocstoreClient.archiveDocById(
+ this.project_id,
+ this.doc._id,
+ (error, res) => {
+ this.res = res
+ if (error) {
+ return done(error)
+ }
+ done()
+ }
+ )
+ }
+ )
+ })
+
+ it('should successully archive the doc', function (done) {
+ this.res.statusCode.should.equal(204)
+ done()
+ })
+
+ it('should return the doc lines and version from persistent storage', function (done) {
+ return DocstoreClient.peekDoc(
+ this.project_id,
+ this.doc._id,
+ {},
+ (error, res, doc) => {
+ res.statusCode.should.equal(200)
+ res.headers['x-doc-status'].should.equal('archived')
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.version.should.equal(this.doc.version)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ return done()
+ }
+ )
+ })
+
+ it('should return the doc lines and version from persistent storage on subsequent requests', function (done) {
+ return DocstoreClient.peekDoc(
+ this.project_id,
+ this.doc._id,
+ {},
+ (error, res, doc) => {
+ res.statusCode.should.equal(200)
+ res.headers['x-doc-status'].should.equal('archived')
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.version.should.equal(this.doc.version)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ return done()
+ }
+ )
+ })
+
+ describe('for an non-archived doc', function () {
+ before(function (done) {
+ this.project_id = ObjectId()
+ this.timeout(1000 * 30)
+ this.doc = {
+ _id: ObjectId(),
+ lines: ['foo', 'bar'],
+ ranges: {},
+ version: 2,
+ }
+ DocstoreClient.createDoc(
+ this.project_id,
+ this.doc._id,
+ this.doc.lines,
+ this.doc.version,
+ this.doc.ranges,
+ done
+ )
+ })
+
+ it('should return the doc lines and version from mongo', function (done) {
+ return DocstoreClient.peekDoc(
+ this.project_id,
+ this.doc._id,
+ {},
+ (error, res, doc) => {
+ res.statusCode.should.equal(200)
+ res.headers['x-doc-status'].should.equal('active')
+ doc.lines.should.deep.equal(this.doc.lines)
+ doc.version.should.equal(this.doc.version)
+ doc.ranges.should.deep.equal(this.doc.ranges)
+ return done()
+ }
+ )
+ })
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/GettingDocsTests.js b/services/docstore/test/acceptance/js/GettingDocsTests.js
new file mode 100644
index 0000000000..857e0cb3ae
--- /dev/null
+++ b/services/docstore/test/acceptance/js/GettingDocsTests.js
@@ -0,0 +1,135 @@
+/* eslint-disable
+ camelcase,
+ handle-callback-err,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const { ObjectId } = require('mongodb')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Getting a doc', function () {
+ beforeEach(function (done) {
+ this.project_id = ObjectId()
+ this.doc_id = ObjectId()
+ this.lines = ['original', 'lines']
+ this.version = 42
+ this.ranges = {
+ changes: [
+ {
+ id: ObjectId().toString(),
+ op: { i: 'foo', p: 3 },
+ meta: {
+ user_id: ObjectId().toString(),
+ ts: new Date().toString(),
+ },
+ },
+ ],
+ }
+ return DocstoreApp.ensureRunning(() => {
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc_id,
+ this.lines,
+ this.version,
+ this.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the doc exists', function () {
+ return it('should get the doc lines and version', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.lines)
+ doc.version.should.equal(this.version)
+ doc.ranges.should.deep.equal(this.ranges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the doc does not exist', function () {
+ return it('should return a 404', function (done) {
+ const missing_doc_id = ObjectId()
+ return DocstoreClient.getDoc(
+ this.project_id,
+ missing_doc_id,
+ {},
+ (error, res, doc) => {
+ res.statusCode.should.equal(404)
+ return done()
+ }
+ )
+ })
+ })
+
+ return describe('when the doc is a deleted doc', function () {
+ beforeEach(function (done) {
+ this.deleted_doc_id = ObjectId()
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.deleted_doc_id,
+ this.lines,
+ this.version,
+ this.ranges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return DocstoreClient.deleteDoc(
+ this.project_id,
+ this.deleted_doc_id,
+ done
+ )
+ }
+ )
+ })
+
+ it('should return the doc', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.deleted_doc_id,
+ { include_deleted: true },
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.lines)
+ doc.version.should.equal(this.version)
+ doc.ranges.should.deep.equal(this.ranges)
+ doc.deleted.should.equal(true)
+ return done()
+ }
+ )
+ })
+
+ return it('should return a 404 when the query string is not set', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.deleted_doc_id,
+ {},
+ (error, res, doc) => {
+ res.statusCode.should.equal(404)
+ return done()
+ }
+ )
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/UpdatingDocsTests.js b/services/docstore/test/acceptance/js/UpdatingDocsTests.js
new file mode 100644
index 0000000000..eda60ae1aa
--- /dev/null
+++ b/services/docstore/test/acceptance/js/UpdatingDocsTests.js
@@ -0,0 +1,497 @@
+/* eslint-disable
+ handle-callback-err,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const { ObjectId } = require('mongodb')
+const DocstoreApp = require('./helpers/DocstoreApp')
+
+const DocstoreClient = require('./helpers/DocstoreClient')
+
+describe('Applying updates to a doc', function () {
+ beforeEach(function (done) {
+ this.project_id = ObjectId()
+ this.doc_id = ObjectId()
+ this.originalLines = ['original', 'lines']
+ this.newLines = ['new', 'lines']
+ this.originalRanges = {
+ changes: [
+ {
+ id: ObjectId().toString(),
+ op: { i: 'foo', p: 3 },
+ meta: {
+ user_id: ObjectId().toString(),
+ ts: new Date().toString(),
+ },
+ },
+ ],
+ }
+ this.newRanges = {
+ changes: [
+ {
+ id: ObjectId().toString(),
+ op: { i: 'bar', p: 6 },
+ meta: {
+ user_id: ObjectId().toString(),
+ ts: new Date().toString(),
+ },
+ },
+ ],
+ }
+ this.version = 42
+ return DocstoreApp.ensureRunning(() => {
+ return DocstoreClient.createDoc(
+ this.project_id,
+ this.doc_id,
+ this.originalLines,
+ this.version,
+ this.originalRanges,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when nothing has been updated', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.originalLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = false', function () {
+ return this.body.modified.should.equal(false)
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ doc.version.should.equal(this.version)
+ doc.ranges.should.deep.equal(this.originalRanges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the lines have changed', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = true', function () {
+ return this.body.modified.should.equal(true)
+ })
+
+ it('should return the rev', function () {
+ return this.body.rev.should.equal(2)
+ })
+
+ return it('should update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.newLines)
+ doc.version.should.equal(this.version)
+ doc.ranges.should.deep.equal(this.originalRanges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the version has changed', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.originalLines,
+ this.version + 1,
+ this.originalRanges,
+ (error, res, body) => {
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = true', function () {
+ return this.body.modified.should.equal(true)
+ })
+
+ it('should return the rev', function () {
+ return this.body.rev.should.equal(1)
+ })
+
+ return it('should update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ doc.version.should.equal(this.version + 1)
+ doc.ranges.should.deep.equal(this.originalRanges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the ranges have changed', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.originalLines,
+ this.version,
+ this.newRanges,
+ (error, res, body) => {
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = true', function () {
+ return this.body.modified.should.equal(true)
+ })
+
+ it('should return the rev', function () {
+ return this.body.rev.should.equal(2)
+ })
+
+ return it('should update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ doc.version.should.equal(this.version)
+ doc.ranges.should.deep.equal(this.newRanges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the doc does not exist', function () {
+ beforeEach(function (done) {
+ this.missing_doc_id = ObjectId()
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.missing_doc_id,
+ this.originalLines,
+ 0,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should create the doc', function () {
+ return this.body.rev.should.equal(1)
+ })
+
+ return it('should be retreivable', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.missing_doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ doc.version.should.equal(0)
+ doc.ranges.should.deep.equal(this.originalRanges)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when malformed doc lines are provided', function () {
+ describe('when the lines are not an array', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ { foo: 'bar' },
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return 400', function () {
+ return this.res.statusCode.should.equal(400)
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ return done()
+ }
+ )
+ })
+ })
+
+ return describe('when the lines are not present', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ null,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return 400', function () {
+ return this.res.statusCode.should.equal(400)
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ return done()
+ }
+ )
+ })
+ })
+ })
+
+ describe('when no version is provided', function () {
+ beforeEach(function (done) {
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.originalLines,
+ null,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return 400', function () {
+ return this.res.statusCode.should.equal(400)
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ doc.version.should.equal(this.version)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the content is large', function () {
+ beforeEach(function (done) {
+ const line = new Array(1025).join('x') // 1kb
+ this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1mb
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.largeLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = true', function () {
+ return this.body.modified.should.equal(true)
+ })
+
+ return it('should update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.largeLines)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when there is a large json payload', function () {
+ beforeEach(function (done) {
+ const line = new Array(1025).join('x') // 1kb
+ this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1kb
+ this.originalRanges.padding = Array.apply(null, Array(2049)).map(
+ () => line
+ ) // 2mb + 1kb
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.largeLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return modified = true', function () {
+ return this.body.modified.should.equal(true)
+ })
+
+ return it('should update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.largeLines)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the document body is too large', function () {
+ beforeEach(function (done) {
+ const line = new Array(1025).join('x') // 1kb
+ this.largeLines = Array.apply(null, Array(2049)).map(() => line) // 2mb + 1kb
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.largeLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ it('should return 413', function () {
+ return this.res.statusCode.should.equal(413)
+ })
+
+ it('should report body too large', function () {
+ return this.res.body.should.equal('document body too large')
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ return done()
+ }
+ )
+ })
+ })
+
+ return describe('when the json payload is too large', function () {
+ beforeEach(function (done) {
+ const line = new Array(1025).join('x') // 1kb
+ this.largeLines = Array.apply(null, Array(1024)).map(() => line) // 1kb
+ this.originalRanges.padding = Array.apply(null, Array(4096)).map(
+ () => line
+ ) // 4mb
+ return DocstoreClient.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.largeLines,
+ this.version,
+ this.originalRanges,
+ (error, res, body) => {
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+
+ return it('should not update the doc in the API', function (done) {
+ return DocstoreClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ (error, res, doc) => {
+ doc.lines.should.deep.equal(this.originalLines)
+ return done()
+ }
+ )
+ })
+ })
+})
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
new file mode 100644
index 0000000000..dd63eeb211
--- /dev/null
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
@@ -0,0 +1,37 @@
+const app = require('../../../../app')
+const { waitForDb } = require('../../../../app/js/mongodb')
+require('logger-sharelatex').logger.level('error')
+const settings = require('@overleaf/settings')
+
+module.exports = {
+ running: false,
+ initing: false,
+ callbacks: [],
+ ensureRunning(callback) {
+ if (callback == null) {
+ callback = function (error) {}
+ }
+ if (this.running) {
+ return callback()
+ } else if (this.initing) {
+ return this.callbacks.push(callback)
+ }
+ this.initing = true
+ this.callbacks.push(callback)
+ waitForDb().then(() => {
+ return app.listen(settings.internal.docstore.port, 'localhost', error => {
+ if (error != null) {
+ throw error
+ }
+ this.running = true
+ return (() => {
+ const result = []
+ for (callback of Array.from(this.callbacks)) {
+ result.push(callback())
+ }
+ return result
+ })()
+ })
+ })
+ },
+}
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
new file mode 100644
index 0000000000..e2c2706fbc
--- /dev/null
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
@@ -0,0 +1,195 @@
+let DocstoreClient
+const request = require('request').defaults({ jar: false })
+const settings = require('@overleaf/settings')
+const Persistor = require('../../../../app/js/PersistorManager')
+
+async function streamToString(stream) {
+ const chunks = []
+ return new Promise((resolve, reject) => {
+ stream.on('data', chunk => chunks.push(chunk))
+ stream.on('error', reject)
+ stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
+ })
+}
+
+async function getStringFromPersistor(persistor, bucket, key) {
+ const stream = await persistor.getObjectStream(bucket, key, {})
+ stream.resume()
+ return streamToString(stream)
+}
+
+module.exports = DocstoreClient = {
+ createDoc(projectId, docId, lines, version, ranges, callback) {
+ return DocstoreClient.updateDoc(
+ projectId,
+ docId,
+ lines,
+ version,
+ ranges,
+ callback
+ )
+ },
+
+ getDoc(projectId, docId, qs, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}`,
+ json: true,
+ qs,
+ },
+ callback
+ )
+ },
+
+ peekDoc(projectId, docId, qs, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}/peek`,
+ json: true,
+ qs,
+ },
+ callback
+ )
+ },
+
+ isDocDeleted(projectId, docId, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}/deleted`,
+ json: true,
+ },
+ callback
+ )
+ },
+
+ getAllDocs(projectId, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc`,
+ json: true,
+ },
+ (req, res, body) => {
+ callback(req, res, body)
+ }
+ )
+ },
+
+ getAllDeletedDocs(projectId, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc-deleted`,
+ json: true,
+ },
+ (error, res, body) => {
+ if (error) return callback(error)
+ if (res.statusCode !== 200) {
+ return callback(new Error('unexpected statusCode'))
+ }
+ callback(null, body)
+ }
+ )
+ },
+
+ getAllRanges(projectId, callback) {
+ request.get(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/ranges`,
+ json: true,
+ },
+ callback
+ )
+ },
+
+ updateDoc(projectId, docId, lines, version, ranges, callback) {
+ return request.post(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}`,
+ json: {
+ lines,
+ version,
+ ranges,
+ },
+ },
+ callback
+ )
+ },
+
+ deleteDoc(projectId, docId, callback) {
+ DocstoreClient.deleteDocWithDateAndName(
+ projectId,
+ docId,
+ new Date(),
+ 'main.tex',
+ callback
+ )
+ },
+
+ deleteDocWithDate(projectId, docId, date, callback) {
+ DocstoreClient.deleteDocWithDateAndName(
+ projectId,
+ docId,
+ date,
+ 'main.tex',
+ callback
+ )
+ },
+
+ deleteDocWithName(projectId, docId, name, callback) {
+ DocstoreClient.deleteDocWithDateAndName(
+ projectId,
+ docId,
+ new Date(),
+ name,
+ callback
+ )
+ },
+
+ deleteDocWithDateAndName(projectId, docId, deletedAt, name, callback) {
+ request.patch(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}`,
+ json: { name, deleted: true, deletedAt },
+ },
+ callback
+ )
+ },
+
+ archiveAllDoc(projectId, callback) {
+ request.post(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/archive`,
+ },
+ callback
+ )
+ },
+
+ archiveDocById(projectId, docId, callback) {
+ request.post(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/doc/${docId}/archive`,
+ },
+ callback
+ )
+ },
+
+ destroyAllDoc(projectId, callback) {
+ request.post(
+ {
+ url: `http://localhost:${settings.internal.docstore.port}/project/${projectId}/destroy`,
+ },
+ callback
+ )
+ },
+
+ getS3Doc(projectId, docId, callback) {
+ getStringFromPersistor(
+ Persistor,
+ settings.docstore.bucket,
+ `${projectId}/${docId}`
+ )
+ .then(data => {
+ callback(null, JSON.parse(data))
+ })
+ .catch(callback)
+ },
+}
diff --git a/services/docstore/test/acceptance/scripts/full-test.sh b/services/docstore/test/acceptance/scripts/full-test.sh
new file mode 100755
index 0000000000..afd5ef2ecb
--- /dev/null
+++ b/services/docstore/test/acceptance/scripts/full-test.sh
@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+# npm rebuild
+
+echo ">> Starting server..."
+
+grunt --no-color forever:app:start
+
+echo ">> Server started"
+
+sleep 20
+
+echo ">> Running acceptance tests..."
+grunt --no-color mochaTest:acceptance
+_test_exit_code=$?
+
+echo ">> Killing server"
+
+grunt --no-color forever:app:stop
+
+echo ">> Done"
+
+exit $_test_exit_code
diff --git a/services/docstore/test/setup.js b/services/docstore/test/setup.js
new file mode 100644
index 0000000000..e45df5834f
--- /dev/null
+++ b/services/docstore/test/setup.js
@@ -0,0 +1,42 @@
+const chai = require('chai')
+const sinon = require('sinon')
+const sinonChai = require('sinon-chai')
+const chaiAsPromised = require('chai-as-promised')
+const SandboxedModule = require('sandboxed-module')
+
+process.env.BACKEND = 'gcs'
+
+// Chai configuration
+chai.should()
+chai.use(sinonChai)
+chai.use(chaiAsPromised)
+
+// Global stubs
+const sandbox = sinon.createSandbox()
+const stubs = {
+ logger: {
+ log: sandbox.stub(),
+ warn: sandbox.stub(),
+ err: sandbox.stub(),
+ error: sandbox.stub(),
+ fatal: sandbox.stub(),
+ },
+}
+
+// SandboxedModule configuration
+SandboxedModule.configure({
+ requires: {
+ 'logger-sharelatex': stubs.logger,
+ },
+ globals: { Buffer, JSON, console, process },
+})
+
+exports.mochaHooks = {
+ beforeEach() {
+ this.logger = stubs.logger
+ },
+
+ afterEach() {
+ sandbox.reset()
+ },
+}
diff --git a/services/docstore/test/unit/js/DocArchiveManagerTests.js b/services/docstore/test/unit/js/DocArchiveManagerTests.js
new file mode 100644
index 0000000000..39369715d7
--- /dev/null
+++ b/services/docstore/test/unit/js/DocArchiveManagerTests.js
@@ -0,0 +1,595 @@
+const sinon = require('sinon')
+const { expect } = require('chai')
+const modulePath = '../../../app/js/DocArchiveManager.js'
+const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongodb')
+const Errors = require('../../../app/js/Errors')
+
+describe('DocArchiveManager', function () {
+ let DocArchiveManager,
+ PersistorManager,
+ MongoManager,
+ RangeManager,
+ Settings,
+ Crypto,
+ Streamifier,
+ HashDigest,
+ HashUpdate,
+ archivedDocs,
+ mongoDocs,
+ docJson,
+ md5Sum,
+ projectId,
+ readStream,
+ stream
+
+ beforeEach(function () {
+ md5Sum = 'decafbad'
+
+ RangeManager = {
+ jsonRangesToMongo: sinon.stub().returns({ mongo: 'ranges' }),
+ }
+ Settings = {
+ docstore: {
+ bucket: 'wombat',
+ },
+ parallelArchiveJobs: 3,
+ destroyBatchSize: 10,
+ destroyRetryCount: 3,
+ }
+ HashDigest = sinon.stub().returns(md5Sum)
+ HashUpdate = sinon.stub().returns({ digest: HashDigest })
+ Crypto = {
+ createHash: sinon.stub().returns({ update: HashUpdate }),
+ }
+ Streamifier = {
+ createReadStream: sinon.stub().returns({ stream: 'readStream' }),
+ }
+
+ projectId = ObjectId()
+ archivedDocs = [
+ {
+ _id: ObjectId(),
+ inS3: true,
+ rev: 2,
+ },
+ {
+ _id: ObjectId(),
+ inS3: true,
+ rev: 4,
+ },
+ {
+ _id: ObjectId(),
+ inS3: true,
+ rev: 6,
+ },
+ ]
+ mongoDocs = [
+ {
+ _id: ObjectId(),
+ lines: ['one', 'two', 'three'],
+ rev: 2,
+ },
+ {
+ _id: ObjectId(),
+ lines: ['aaa', 'bbb', 'ccc'],
+ rev: 4,
+ },
+ {
+ _id: ObjectId(),
+ inS3: true,
+ rev: 6,
+ },
+ {
+ _id: ObjectId(),
+ inS3: true,
+ rev: 6,
+ },
+ {
+ _id: ObjectId(),
+ lines: ['111', '222', '333'],
+ rev: 6,
+ },
+ ]
+
+ docJson = JSON.stringify({
+ lines: mongoDocs[0].lines,
+ ranges: mongoDocs[0].ranges,
+ schema_v: 1,
+ })
+
+ stream = {
+ on: sinon.stub(),
+ resume: sinon.stub(),
+ }
+ stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+ stream.on.withArgs('end').yields()
+
+ readStream = {
+ stream: 'readStream',
+ }
+
+ PersistorManager = {
+ getObjectStream: sinon.stub().resolves(stream),
+ sendStream: sinon.stub().resolves(),
+ getObjectMd5Hash: sinon.stub().resolves(md5Sum),
+ deleteObject: sinon.stub().resolves(),
+ }
+
+ const getNonArchivedProjectDocs = sinon.stub()
+ getNonArchivedProjectDocs
+ .onCall(0)
+ .resolves(mongoDocs.filter(doc => !doc.inS3))
+ getNonArchivedProjectDocs.onCall(1).resolves([])
+
+ const getArchivedProjectDocs = sinon.stub()
+ getArchivedProjectDocs.onCall(0).resolves(archivedDocs)
+ getArchivedProjectDocs.onCall(1).resolves([])
+
+ MongoManager = {
+ promises: {
+ markDocAsArchived: sinon.stub().resolves(),
+ upsertIntoDocCollection: sinon.stub().resolves(),
+ getProjectsDocs: sinon.stub().resolves(mongoDocs),
+ getNonDeletedArchivedProjectDocs: getArchivedProjectDocs,
+ getNonArchivedProjectDocs,
+ getArchivedProjectDocs,
+ findDoc: sinon.stub().rejects(new Errors.NotFoundError()),
+ destroyDoc: sinon.stub().resolves(),
+ },
+ }
+ for (const mongoDoc of mongoDocs.concat(archivedDocs)) {
+ MongoManager.promises.findDoc
+ .withArgs(projectId, mongoDoc._id, sinon.match.any)
+ .resolves(mongoDoc)
+ }
+
+ DocArchiveManager = SandboxedModule.require(modulePath, {
+ requires: {
+ '@overleaf/settings': Settings,
+ crypto: Crypto,
+ streamifier: Streamifier,
+ './MongoManager': MongoManager,
+ './RangeManager': RangeManager,
+ './PersistorManager': PersistorManager,
+ './Errors': Errors,
+ },
+ })
+ })
+
+ describe('archiveDoc', function () {
+ it('should resolve when passed a valid document', async function () {
+ await expect(
+ DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+ ).to.eventually.be.fulfilled
+ })
+
+ it('should throw an error if the doc has no lines', async function () {
+ const doc = mongoDocs[0]
+ doc.lines = null
+
+ await expect(
+ DocArchiveManager.promises.archiveDoc(projectId, doc)
+ ).to.eventually.be.rejectedWith('doc has no lines')
+ })
+
+ it('should add the schema version', async function () {
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[1])
+ expect(Streamifier.createReadStream).to.have.been.calledWith(
+ sinon.match(/"schema_v":1/)
+ )
+ })
+
+ it('should calculate the hex md5 sum of the content', async function () {
+ const json = JSON.stringify({
+ lines: mongoDocs[0].lines,
+ ranges: mongoDocs[0].ranges,
+ schema_v: 1,
+ })
+
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+ expect(Crypto.createHash).to.have.been.calledWith('md5')
+ expect(HashUpdate).to.have.been.calledWith(json)
+ expect(HashDigest).to.have.been.calledWith('hex')
+ })
+
+ it('should pass the md5 hash to the object persistor for verification', async function () {
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+ expect(PersistorManager.sendStream).to.have.been.calledWith(
+ sinon.match.any,
+ sinon.match.any,
+ sinon.match.any,
+ { sourceMd5: md5Sum }
+ )
+ })
+
+ it('should pass the correct bucket and key to the persistor', async function () {
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+
+ expect(PersistorManager.sendStream).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${mongoDocs[0]._id}`
+ )
+ })
+
+ it('should create a stream from the encoded json and send it', async function () {
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+ expect(Streamifier.createReadStream).to.have.been.calledWith(docJson)
+ expect(PersistorManager.sendStream).to.have.been.calledWith(
+ sinon.match.any,
+ sinon.match.any,
+ readStream
+ )
+ })
+
+ it('should mark the doc as archived', async function () {
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ mongoDocs[0]._id,
+ mongoDocs[0].rev
+ )
+ })
+
+ describe('with null bytes in the result', function () {
+ const _stringify = JSON.stringify
+
+ beforeEach(function () {
+ JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}')
+ })
+
+ afterEach(function () {
+ JSON.stringify = _stringify
+ })
+
+ it('should return an error', async function () {
+ await expect(
+ DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0])
+ ).to.eventually.be.rejectedWith('null bytes detected')
+ })
+ })
+ })
+
+ describe('unarchiveDoc', function () {
+ let docId
+
+ describe('when the doc is in S3', function () {
+ beforeEach(function () {
+ MongoManager.promises.findDoc = sinon.stub().resolves({ inS3: true })
+ docId = mongoDocs[0]._id
+ })
+
+ it('should resolve when passed a valid document', async function () {
+ await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId))
+ .to.eventually.be.fulfilled
+ })
+
+ it('should throw an error if the md5 does not match', async function () {
+ PersistorManager.getObjectMd5Hash.resolves('badf00d')
+ await expect(
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ ).to.eventually.be.rejected.and.be.instanceof(Errors.Md5MismatchError)
+ })
+
+ it('should update the doc lines in mongo', async function () {
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.upsertIntoDocCollection
+ ).to.have.been.calledWith(projectId, docId, {
+ lines: mongoDocs[0].lines,
+ })
+ })
+
+ it('should delete the doc in s3', async function () {
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(PersistorManager.deleteObject).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${docId}`
+ )
+ })
+
+ describe('doc contents', function () {
+ let mongoDoc, s3Doc
+
+ describe('when the doc has the old schema', function () {
+ beforeEach(function () {
+ mongoDoc = {
+ lines: ['doc', 'lines'],
+ }
+ s3Doc = ['doc', 'lines']
+ docJson = JSON.stringify(s3Doc)
+ stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+ })
+
+ it('should return the docs lines', async function () {
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.upsertIntoDocCollection
+ ).to.have.been.calledWith(projectId, docId, mongoDoc)
+ })
+ })
+
+ describe('with the new schema and ranges', function () {
+ beforeEach(function () {
+ s3Doc = {
+ lines: ['doc', 'lines'],
+ ranges: { json: 'ranges' },
+ schema_v: 1,
+ }
+ mongoDoc = {
+ lines: ['doc', 'lines'],
+ ranges: { mongo: 'ranges' },
+ }
+ docJson = JSON.stringify(s3Doc)
+ stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+ })
+
+ it('should return the doc lines and ranges', async function () {
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.upsertIntoDocCollection
+ ).to.have.been.calledWith(projectId, docId, mongoDoc)
+ })
+ })
+
+ describe('with the new schema and no ranges', function () {
+ beforeEach(function () {
+ s3Doc = {
+ lines: ['doc', 'lines'],
+ schema_v: 1,
+ }
+ mongoDoc = {
+ lines: ['doc', 'lines'],
+ }
+ docJson = JSON.stringify(s3Doc)
+ stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+ })
+
+ it('should return only the doc lines', async function () {
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.upsertIntoDocCollection
+ ).to.have.been.calledWith(projectId, docId, mongoDoc)
+ })
+ })
+
+ describe('with an unrecognised schema', function () {
+ beforeEach(function () {
+ s3Doc = {
+ lines: ['doc', 'lines'],
+ schema_v: 2,
+ }
+ docJson = JSON.stringify(s3Doc)
+ stream.on.withArgs('data').yields(Buffer.from(docJson, 'utf8'))
+ })
+
+ it('should throw an error', async function () {
+ await expect(
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ ).to.eventually.be.rejectedWith(
+ "I don't understand the doc format in s3"
+ )
+ })
+ })
+ })
+ })
+
+ it('should not do anything if the file is already unarchived', async function () {
+ MongoManager.promises.findDoc.resolves({ inS3: false })
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(PersistorManager.getObjectStream).not.to.have.been.called
+ })
+
+ describe('when the file is removed while we are processing it', function () {
+ beforeEach(function () {
+ MongoManager.promises.findDoc = sinon.stub().resolves({ inS3: true })
+ MongoManager.promises.findDoc.onSecondCall().resolves({ inS3: false })
+ })
+
+ it('should not throw an error if the file is unarchived before we get for its hash', async function () {
+ PersistorManager.getObjectMd5Hash = sinon
+ .stub()
+ .rejects(new Errors.NotFoundError())
+ await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId))
+ .to.eventually.be.fulfilled
+ expect(PersistorManager.getObjectStream).not.to.have.been.called
+ })
+
+ it('should not throw an error if the file is unarchived before we download it', async function () {
+ PersistorManager.getObjectStream = sinon
+ .stub()
+ .rejects(new Errors.NotFoundError())
+ await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId))
+ .to.eventually.be.fulfilled
+ expect(MongoManager.promises.upsertIntoDocCollection).not.to.have.been
+ .called
+ })
+ })
+
+ it('should throw an error if the file is not found but is still listed as archived', async function () {
+ PersistorManager.getObjectStream = sinon
+ .stub()
+ .rejects(new Errors.NotFoundError())
+ await expect(
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ ).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
+ })
+ })
+
+ describe('destroyDoc', function () {
+ let docId
+
+ beforeEach(function () {
+ docId = mongoDocs[0]._id
+ })
+
+ it('should resolve when passed a valid document', async function () {
+ await expect(DocArchiveManager.promises.destroyDoc(projectId, docId)).to
+ .eventually.be.fulfilled
+ })
+
+ it('should throw a not found error when there is no document', async function () {
+ await expect(
+ DocArchiveManager.promises.destroyDoc(projectId, 'wombat')
+ ).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
+ })
+
+ describe('when the doc is in s3', function () {
+ beforeEach(function () {
+ mongoDocs[0].inS3 = true
+ })
+
+ it('should delete the document from s3, if it is in s3', async function () {
+ await DocArchiveManager.promises.destroyDoc(projectId, docId)
+ expect(PersistorManager.deleteObject).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${docId}`
+ )
+ })
+
+ it('should delete the doc in mongo', async function () {
+ await DocArchiveManager.promises.destroyDoc(projectId, docId)
+ })
+
+ describe('when the destroy request errors', function () {
+ beforeEach(function () {
+ mongoDocs[0].inS3 = true
+ PersistorManager.deleteObject.onFirstCall().rejects(new Error('1'))
+ PersistorManager.deleteObject.onSecondCall().rejects(new Error('2'))
+ PersistorManager.deleteObject.onThirdCall().resolves()
+ })
+
+ it('should retry', async function () {
+ await DocArchiveManager.promises.destroyDoc(projectId, docId)
+ expect(PersistorManager.deleteObject).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${docId}`
+ )
+ expect(PersistorManager.deleteObject.callCount).to.equal(3)
+ })
+ })
+
+ describe('when the destroy request errors permanent', function () {
+ beforeEach(function () {
+ mongoDocs[0].inS3 = true
+ PersistorManager.deleteObject.rejects(new Error('permanent'))
+ })
+
+ it('should retry and fail eventually', async function () {
+ await expect(DocArchiveManager.promises.destroyDoc(projectId, docId))
+ .to.eventually.be.rejected
+ expect(PersistorManager.deleteObject).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${docId}`
+ )
+ expect(PersistorManager.deleteObject.callCount).to.equal(4)
+ })
+ })
+ })
+
+ describe('when the doc is not in s3', function () {
+ beforeEach(function () {
+ mongoDocs[0].inS3 = false
+ })
+
+ it('should not delete the document from s3, if it is not in s3', async function () {
+ await DocArchiveManager.promises.destroyDoc(projectId, docId)
+ expect(PersistorManager.deleteObject).not.to.have.been.called
+ })
+
+ it('should delete the doc in mongo', async function () {
+ await DocArchiveManager.promises.destroyDoc(projectId, docId)
+ })
+ })
+ })
+
+ describe('archiveAllDocs', function () {
+ it('should resolve with valid arguments', async function () {
+ await expect(DocArchiveManager.promises.archiveAllDocs(projectId)).to
+ .eventually.be.fulfilled
+ })
+
+ it('should archive all project docs which are not in s3', async function () {
+ await DocArchiveManager.promises.archiveAllDocs(projectId)
+ // not inS3
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ mongoDocs[0]._id
+ )
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ mongoDocs[1]._id
+ )
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ mongoDocs[4]._id
+ )
+
+ // inS3
+ expect(
+ MongoManager.promises.markDocAsArchived
+ ).not.to.have.been.calledWith(mongoDocs[2]._id)
+ expect(
+ MongoManager.promises.markDocAsArchived
+ ).not.to.have.been.calledWith(mongoDocs[3]._id)
+ })
+ })
+
+ describe('unArchiveAllDocs', function () {
+ it('should resolve with valid arguments', async function () {
+ await expect(DocArchiveManager.promises.unArchiveAllDocs(projectId)).to
+ .eventually.be.fulfilled
+ })
+
+ it('should unarchive all inS3 docs', async function () {
+ await DocArchiveManager.promises.unArchiveAllDocs(projectId)
+
+ for (const doc of archivedDocs) {
+ expect(PersistorManager.getObjectStream).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${doc._id}`
+ )
+ }
+ })
+ })
+
+ describe('destroyAllDocs', function () {
+ beforeEach(function () {
+ MongoManager.promises.getProjectsDocs.onCall(0).resolves(mongoDocs)
+ MongoManager.promises.getProjectsDocs.onCall(1).resolves([])
+ })
+
+ it('should resolve with valid arguments', async function () {
+ await expect(DocArchiveManager.promises.destroyAllDocs(projectId)).to
+ .eventually.be.fulfilled
+ })
+
+ it('should delete all docs that are in s3 from s3', async function () {
+ await DocArchiveManager.promises.destroyAllDocs(projectId)
+
+ // not inS3
+ for (const index of [0, 1, 4]) {
+ expect(PersistorManager.deleteObject).not.to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${mongoDocs[index]._id}`
+ )
+ }
+
+ // inS3
+ for (const index of [2, 3]) {
+ expect(PersistorManager.deleteObject).to.have.been.calledWith(
+ Settings.docstore.bucket,
+ `${projectId}/${mongoDocs[index]._id}`
+ )
+ }
+ })
+
+ it('should destroy all docs in mongo', async function () {
+ await DocArchiveManager.promises.destroyAllDocs(projectId)
+
+ for (const mongoDoc of mongoDocs) {
+ expect(MongoManager.promises.destroyDoc).to.have.been.calledWith(
+ mongoDoc._id
+ )
+ }
+ })
+ })
+})
diff --git a/services/docstore/test/unit/js/DocManagerTests.js b/services/docstore/test/unit/js/DocManagerTests.js
new file mode 100644
index 0000000000..f81cc6a241
--- /dev/null
+++ b/services/docstore/test/unit/js/DocManagerTests.js
@@ -0,0 +1,863 @@
+/* eslint-disable
+ camelcase,
+ handle-callback-err,
+ no-dupe-keys,
+ no-return-assign,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+const { assert, expect } = require('chai')
+const modulePath = require('path').join(__dirname, '../../../app/js/DocManager')
+const { ObjectId } = require('mongodb')
+const Errors = require('../../../app/js/Errors')
+
+describe('DocManager', function () {
+ beforeEach(function () {
+ this.DocManager = SandboxedModule.require(modulePath, {
+ requires: {
+ './MongoManager': (this.MongoManager = {}),
+ './DocArchiveManager': (this.DocArchiveManager = {}),
+ './RangeManager': (this.RangeManager = {
+ jsonRangesToMongo(r) {
+ return r
+ },
+ shouldUpdateRanges: sinon.stub().returns(false),
+ }),
+ '@overleaf/settings': (this.settings = { docstore: {} }),
+ './Errors': Errors,
+ },
+ })
+ this.doc_id = ObjectId().toString()
+ this.project_id = ObjectId().toString()
+ this.another_project_id = ObjectId().toString()
+ this.callback = sinon.stub()
+ return (this.stubbedError = new Error('blew up'))
+ })
+
+ describe('getFullDoc', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub()
+ return (this.doc = {
+ _id: this.doc_id,
+ lines: ['2134'],
+ })
+ })
+
+ it('should call get doc with a quick filter', function (done) {
+ this.DocManager._getDoc.callsArgWith(3, null, this.doc)
+ return this.DocManager.getFullDoc(
+ this.project_id,
+ this.doc_id,
+ (err, doc) => {
+ doc.should.equal(this.doc)
+ this.DocManager._getDoc
+ .calledWith(this.project_id, this.doc_id, {
+ lines: true,
+ rev: true,
+ deleted: true,
+ version: true,
+ ranges: true,
+ inS3: true,
+ })
+ .should.equal(true)
+ return done()
+ }
+ )
+ })
+
+ return it('should return error when get doc errors', function (done) {
+ this.DocManager._getDoc.callsArgWith(3, 'error')
+ return this.DocManager.getFullDoc(
+ this.project_id,
+ this.doc_id,
+ (err, exist) => {
+ err.should.equal('error')
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('getRawDoc', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub()
+ return (this.doc = { lines: ['2134'] })
+ })
+
+ it('should call get doc with a quick filter', function (done) {
+ this.DocManager._getDoc.callsArgWith(3, null, this.doc)
+ return this.DocManager.getDocLines(
+ this.project_id,
+ this.doc_id,
+ (err, doc) => {
+ doc.should.equal(this.doc)
+ this.DocManager._getDoc
+ .calledWith(this.project_id, this.doc_id, {
+ lines: true,
+ inS3: true,
+ })
+ .should.equal(true)
+ return done()
+ }
+ )
+ })
+
+ return it('should return error when get doc errors', function (done) {
+ this.DocManager._getDoc.callsArgWith(3, 'error')
+ return this.DocManager.getDocLines(
+ this.project_id,
+ this.doc_id,
+ (err, exist) => {
+ err.should.equal('error')
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('getDoc', function () {
+ beforeEach(function () {
+ this.project = { name: 'mock-project' }
+ this.doc = {
+ _id: this.doc_id,
+ project_id: this.project_id,
+ lines: ['mock-lines'],
+ }
+ this.version = 42
+ this.MongoManager.findDoc = sinon.stub()
+ return (this.MongoManager.getDocVersion = sinon
+ .stub()
+ .yields(null, this.version))
+ })
+
+ describe('when using a filter', function () {
+ beforeEach(function () {
+ return this.MongoManager.findDoc.yields(null, this.doc)
+ })
+
+ it('should error if inS3 is not set to true', function (done) {
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { inS3: false },
+ err => {
+ expect(err).to.exist
+ return done()
+ }
+ )
+ })
+
+ it('should always get inS3 even when no filter is passed', function (done) {
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ undefined,
+ err => {
+ this.MongoManager.findDoc.called.should.equal(false)
+ expect(err).to.exist
+ return done()
+ }
+ )
+ })
+
+ return it('should not error if inS3 is set to true', function (done) {
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { inS3: true },
+ err => {
+ expect(err).to.not.exist
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('when the doc is in the doc collection', function () {
+ beforeEach(function () {
+ this.MongoManager.findDoc.yields(null, this.doc)
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { version: true, inS3: true },
+ this.callback
+ )
+ })
+
+ it('should get the doc from the doc collection', function () {
+ return this.MongoManager.findDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ it('should get the doc version from the docOps collection', function () {
+ return this.MongoManager.getDocVersion
+ .calledWith(this.doc_id)
+ .should.equal(true)
+ })
+
+ return it('should return the callback with the doc with the version', function () {
+ this.callback.called.should.equal(true)
+ const doc = this.callback.args[0][1]
+ doc.lines.should.equal(this.doc.lines)
+ return doc.version.should.equal(this.version)
+ })
+ })
+
+ describe('without the version filter', function () {
+ beforeEach(function () {
+ this.MongoManager.findDoc.yields(null, this.doc)
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { version: false, inS3: true },
+ this.callback
+ )
+ })
+
+ return it('should not get the doc version from the docOps collection', function () {
+ return this.MongoManager.getDocVersion.called.should.equal(false)
+ })
+ })
+
+ describe('when MongoManager.findDoc errors', function () {
+ beforeEach(function () {
+ this.MongoManager.findDoc.yields(this.stubbedError)
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { version: true, inS3: true },
+ this.callback
+ )
+ })
+
+ return it('should return the error', function () {
+ return this.callback.calledWith(this.stubbedError).should.equal(true)
+ })
+ })
+
+ describe('when the doc is archived', function () {
+ beforeEach(function () {
+ this.doc = {
+ _id: this.doc_id,
+ project_id: this.project_id,
+ lines: ['mock-lines'],
+ inS3: true,
+ }
+ this.MongoManager.findDoc.yields(null, this.doc)
+ this.DocArchiveManager.unarchiveDoc = (
+ project_id,
+ doc_id,
+ callback
+ ) => {
+ this.doc.inS3 = false
+ return callback()
+ }
+ sinon.spy(this.DocArchiveManager, 'unarchiveDoc')
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { version: true, inS3: true },
+ this.callback
+ )
+ })
+
+ it('should call the DocArchive to unarchive the doc', function () {
+ return this.DocArchiveManager.unarchiveDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ it('should look up the doc twice', function () {
+ return this.MongoManager.findDoc.calledTwice.should.equal(true)
+ })
+
+ return it('should return the doc', function () {
+ return this.callback.calledWith(null, this.doc).should.equal(true)
+ })
+ })
+
+ return describe('when the doc does not exist in the docs collection', function () {
+ beforeEach(function () {
+ this.MongoManager.findDoc = sinon.stub().yields(null, null)
+ return this.DocManager._getDoc(
+ this.project_id,
+ this.doc_id,
+ { version: true, inS3: true },
+ this.callback
+ )
+ })
+
+ return it('should return a NotFoundError', function () {
+ return this.callback
+ .calledWith(
+ sinon.match.has(
+ 'message',
+ `No such doc: ${this.doc_id} in project ${this.project_id}`
+ )
+ )
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('getAllNonDeletedDocs', function () {
+ describe('when the project exists', function () {
+ beforeEach(function () {
+ this.docs = [
+ {
+ _id: this.doc_id,
+ project_id: this.project_id,
+ lines: ['mock-lines'],
+ },
+ ]
+ this.MongoManager.getProjectsDocs = sinon
+ .stub()
+ .callsArgWith(3, null, this.docs)
+ this.DocArchiveManager.unArchiveAllDocs = sinon
+ .stub()
+ .callsArgWith(1, null, this.docs)
+ this.filter = { lines: true }
+ return this.DocManager.getAllNonDeletedDocs(
+ this.project_id,
+ this.filter,
+ this.callback
+ )
+ })
+
+ it('should get the project from the database', function () {
+ return this.MongoManager.getProjectsDocs
+ .calledWith(this.project_id, { include_deleted: false }, this.filter)
+ .should.equal(true)
+ })
+
+ return it('should return the docs', function () {
+ return this.callback.calledWith(null, this.docs).should.equal(true)
+ })
+ })
+
+ return describe('when there are no docs for the project', function () {
+ beforeEach(function () {
+ this.MongoManager.getProjectsDocs = sinon
+ .stub()
+ .callsArgWith(3, null, null)
+ this.DocArchiveManager.unArchiveAllDocs = sinon
+ .stub()
+ .callsArgWith(1, null)
+ return this.DocManager.getAllNonDeletedDocs(
+ this.project_id,
+ this.filter,
+ this.callback
+ )
+ })
+
+ return it('should return a NotFoundError', function () {
+ return this.callback
+ .calledWith(
+ sinon.match.has('message', `No docs for project ${this.project_id}`)
+ )
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('patchDoc', function () {
+ describe('when the doc exists', function () {
+ beforeEach(function () {
+ this.lines = ['mock', 'doc', 'lines']
+ this.rev = 77
+ this.MongoManager.findDoc = sinon
+ .stub()
+ .yields(null, { _id: ObjectId(this.doc_id) })
+ this.MongoManager.patchDoc = sinon.stub().yields(null)
+ this.DocArchiveManager.archiveDocById = sinon.stub().yields(null)
+ this.meta = {}
+ })
+
+ describe('standard path', function () {
+ beforeEach(function (done) {
+ this.callback = sinon.stub().callsFake(done)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should get the doc', function () {
+ expect(this.MongoManager.findDoc).to.have.been.calledWith(
+ this.project_id,
+ this.doc_id
+ )
+ })
+
+ it('should persist the meta', function () {
+ expect(this.MongoManager.patchDoc).to.have.been.calledWith(
+ this.project_id,
+ this.doc_id,
+ this.meta
+ )
+ })
+
+ it('should return the callback', function () {
+ expect(this.callback).to.have.been.calledWith(null)
+ })
+ })
+
+ describe('background flush disabled and deleting a doc', function () {
+ beforeEach(function (done) {
+ this.settings.docstore.archiveOnSoftDelete = false
+ this.meta.deleted = true
+
+ this.callback = sinon.stub().callsFake(done)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should not flush the doc out of mongo', function () {
+ expect(this.DocArchiveManager.archiveDocById).to.not.have.been.called
+ })
+ })
+
+ describe('background flush enabled and not deleting a doc', function () {
+ beforeEach(function (done) {
+ this.settings.docstore.archiveOnSoftDelete = false
+ this.meta.deleted = false
+ this.callback = sinon.stub().callsFake(done)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should not flush the doc out of mongo', function () {
+ expect(this.DocArchiveManager.archiveDocById).to.not.have.been.called
+ })
+ })
+
+ describe('background flush enabled and deleting a doc', function () {
+ beforeEach(function () {
+ this.settings.docstore.archiveOnSoftDelete = true
+ this.meta.deleted = true
+ })
+
+ describe('when the background flush succeeds', function () {
+ beforeEach(function (done) {
+ this.DocArchiveManager.archiveDocById = sinon.stub().yields(null)
+ this.callback = sinon.stub().callsFake(done)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should not log a warning', function () {
+ expect(this.logger.warn).to.not.have.been.called
+ })
+
+ it('should flush the doc out of mongo', function () {
+ expect(
+ this.DocArchiveManager.archiveDocById
+ ).to.have.been.calledWith(this.project_id, this.doc_id)
+ })
+ })
+
+ describe('when the background flush fails', function () {
+ beforeEach(function (done) {
+ this.err = new Error('foo')
+ this.DocArchiveManager.archiveDocById = sinon
+ .stub()
+ .yields(this.err)
+ this.callback = sinon.stub().callsFake(done)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should log a warning', function () {
+ expect(this.logger.warn).to.have.been.calledWith(
+ sinon.match({
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ err: this.err,
+ }),
+ 'archiving a single doc in the background failed'
+ )
+ })
+
+ it('should not fail the delete process', function () {
+ expect(this.callback).to.have.been.calledWith(null)
+ })
+ })
+ })
+ })
+
+ describe('when the doc does not exist', function () {
+ beforeEach(function () {
+ this.MongoManager.findDoc = sinon.stub().yields(null)
+ this.DocManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ {},
+ this.callback
+ )
+ })
+
+ it('should return a NotFoundError', function () {
+ expect(this.callback).to.have.been.calledWith(
+ sinon.match.has(
+ 'message',
+ `No such project/doc to delete: ${this.project_id}/${this.doc_id}`
+ )
+ )
+ })
+ })
+ })
+
+ return describe('updateDoc', function () {
+ beforeEach(function () {
+ this.oldDocLines = ['old', 'doc', 'lines']
+ this.newDocLines = ['new', 'doc', 'lines']
+ this.originalRanges = {
+ changes: [
+ {
+ id: ObjectId().toString(),
+ op: { i: 'foo', p: 3 },
+ meta: {
+ user_id: ObjectId().toString(),
+ ts: new Date().toString(),
+ },
+ },
+ ],
+ }
+ this.newRanges = {
+ changes: [
+ {
+ id: ObjectId().toString(),
+ op: { i: 'bar', p: 6 },
+ meta: {
+ user_id: ObjectId().toString(),
+ ts: new Date().toString(),
+ },
+ },
+ ],
+ }
+ this.version = 42
+ this.doc = {
+ _id: this.doc_id,
+ project_id: this.project_id,
+ lines: this.oldDocLines,
+ rev: (this.rev = 5),
+ version: this.version,
+ ranges: this.originalRanges,
+ }
+
+ this.MongoManager.upsertIntoDocCollection = sinon.stub().callsArg(3)
+ this.MongoManager.setDocVersion = sinon.stub().yields()
+ return (this.DocManager._getDoc = sinon.stub())
+ })
+
+ describe('when only the doc lines have changed', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newDocLines,
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should get the existing doc', function () {
+ return this.DocManager._getDoc
+ .calledWith(this.project_id, this.doc_id, {
+ version: true,
+ rev: true,
+ lines: true,
+ version: true,
+ ranges: true,
+ inS3: true,
+ })
+ .should.equal(true)
+ })
+
+ it('should upsert the document to the doc collection', function () {
+ return this.MongoManager.upsertIntoDocCollection
+ .calledWith(this.project_id, this.doc_id, { lines: this.newDocLines })
+ .should.equal(true)
+ })
+
+ it('should not update the version', function () {
+ return this.MongoManager.setDocVersion.called.should.equal(false)
+ })
+
+ return it('should return the callback with the new rev', function () {
+ return this.callback
+ .calledWith(null, true, this.rev + 1)
+ .should.equal(true)
+ })
+ })
+
+ describe('when the doc ranges have changed', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+ this.RangeManager.shouldUpdateRanges.returns(true)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.oldDocLines,
+ this.version,
+ this.newRanges,
+ this.callback
+ )
+ })
+
+ it('should upsert the ranges', function () {
+ return this.MongoManager.upsertIntoDocCollection
+ .calledWith(this.project_id, this.doc_id, { ranges: this.newRanges })
+ .should.equal(true)
+ })
+
+ it('should not update the version', function () {
+ return this.MongoManager.setDocVersion.called.should.equal(false)
+ })
+
+ return it('should return the callback with the new rev', function () {
+ return this.callback
+ .calledWith(null, true, this.rev + 1)
+ .should.equal(true)
+ })
+ })
+
+ describe('when only the version has changed', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.oldDocLines,
+ this.version + 1,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should not change the lines or ranges', function () {
+ return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+ false
+ )
+ })
+
+ it('should update the version', function () {
+ return this.MongoManager.setDocVersion
+ .calledWith(this.doc_id, this.version + 1)
+ .should.equal(true)
+ })
+
+ return it('should return the callback with the old rev', function () {
+ return this.callback.calledWith(null, true, this.rev).should.equal(true)
+ })
+ })
+
+ describe('when the doc has not changed at all', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.oldDocLines,
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should not update the ranges or lines', function () {
+ return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+ false
+ )
+ })
+
+ it('should not update the version', function () {
+ return this.MongoManager.setDocVersion.called.should.equal(false)
+ })
+
+ return it('should return the callback with the old rev and modified == false', function () {
+ return this.callback
+ .calledWith(null, false, this.rev)
+ .should.equal(true)
+ })
+ })
+
+ describe('when the version is null', function () {
+ beforeEach(function () {
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newDocLines,
+ null,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ return it('should return an error', function () {
+ return this.callback
+ .calledWith(
+ sinon.match.has('message', 'no lines, version or ranges provided')
+ )
+ .should.equal(true)
+ })
+ })
+
+ describe('when the lines are null', function () {
+ beforeEach(function () {
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ null,
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ return it('should return an error', function () {
+ return this.callback
+ .calledWith(
+ sinon.match.has('message', 'no lines, version or ranges provided')
+ )
+ .should.equal(true)
+ })
+ })
+
+ describe('when the ranges are null', function () {
+ beforeEach(function () {
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newDocLines,
+ this.version,
+ null,
+ this.callback
+ )
+ })
+
+ return it('should return an error', function () {
+ return this.callback
+ .calledWith(
+ sinon.match.has('message', 'no lines, version or ranges provided')
+ )
+ .should.equal(true)
+ })
+ })
+
+ describe('when there is a generic error getting the doc', function () {
+ beforeEach(function () {
+ this.error = new Error('doc could not be found')
+ this.DocManager._getDoc = sinon
+ .stub()
+ .callsArgWith(3, this.error, null, null)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newDocLines,
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should not upsert the document to the doc collection', function () {
+ return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+ false
+ )
+ })
+
+ return it('should return the callback with the error', function () {
+ return this.callback.calledWith(this.error).should.equal(true)
+ })
+ })
+
+ describe('when the doc lines have not changed', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, this.doc)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.oldDocLines.slice(),
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should not update the doc', function () {
+ return this.MongoManager.upsertIntoDocCollection.called.should.equal(
+ false
+ )
+ })
+
+ return it('should return the callback with the existing rev', function () {
+ return this.callback
+ .calledWith(null, false, this.rev)
+ .should.equal(true)
+ })
+ })
+
+ return describe('when the doc does not exist', function () {
+ beforeEach(function () {
+ this.DocManager._getDoc = sinon.stub().callsArgWith(3, null, null, null)
+ return this.DocManager.updateDoc(
+ this.project_id,
+ this.doc_id,
+ this.newDocLines,
+ this.version,
+ this.originalRanges,
+ this.callback
+ )
+ })
+
+ it('should upsert the document to the doc collection', function () {
+ return this.MongoManager.upsertIntoDocCollection
+ .calledWith(this.project_id, this.doc_id, {
+ lines: this.newDocLines,
+ ranges: this.originalRanges,
+ })
+ .should.equal(true)
+ })
+
+ it('should set the version', function () {
+ return this.MongoManager.setDocVersion
+ .calledWith(this.doc_id, this.version)
+ .should.equal(true)
+ })
+
+ return it('should return the callback with the new rev', function () {
+ return this.callback.calledWith(null, true, 1).should.equal(true)
+ })
+ })
+ })
+})
diff --git a/services/docstore/test/unit/js/HttpControllerTests.js b/services/docstore/test/unit/js/HttpControllerTests.js
new file mode 100644
index 0000000000..e60e0e705c
--- /dev/null
+++ b/services/docstore/test/unit/js/HttpControllerTests.js
@@ -0,0 +1,513 @@
+/* eslint-disable
+ no-return-assign,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+const { assert, expect } = require('chai')
+const modulePath = require('path').join(
+ __dirname,
+ '../../../app/js/HttpController'
+)
+const { ObjectId } = require('mongodb')
+
+describe('HttpController', function () {
+ beforeEach(function () {
+ const settings = {
+ max_doc_length: 2 * 1024 * 1024,
+ }
+ this.HttpController = SandboxedModule.require(modulePath, {
+ requires: {
+ './DocManager': (this.DocManager = {}),
+ './DocArchiveManager': (this.DocArchiveManager = {}),
+ '@overleaf/settings': settings,
+ './HealthChecker': {},
+ },
+ })
+ this.res = {
+ send: sinon.stub(),
+ sendStatus: sinon.stub(),
+ json: sinon.stub(),
+ setHeader: sinon.stub(),
+ }
+ this.res.status = sinon.stub().returns(this.res)
+ this.req = { query: {} }
+ this.next = sinon.stub()
+ this.project_id = 'mock-project-id'
+ this.doc_id = 'mock-doc-id'
+ this.doc = {
+ _id: this.doc_id,
+ lines: ['mock', 'lines', ' here', '', '', ' spaces '],
+ version: 42,
+ rev: 5,
+ }
+ return (this.deletedDoc = {
+ deleted: true,
+ _id: this.doc_id,
+ lines: ['mock', 'lines', ' here', '', '', ' spaces '],
+ version: 42,
+ rev: 5,
+ })
+ })
+
+ describe('getDoc', function () {
+ describe('without deleted docs', function () {
+ beforeEach(function () {
+ this.req.params = {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ }
+ this.DocManager.getFullDoc = sinon
+ .stub()
+ .callsArgWith(2, null, this.doc)
+ return this.HttpController.getDoc(this.req, this.res, this.next)
+ })
+
+ it('should get the document with the version (including deleted)', function () {
+ return this.DocManager.getFullDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ return it('should return the doc as JSON', function () {
+ return this.res.json
+ .calledWith({
+ _id: this.doc_id,
+ lines: this.doc.lines,
+ rev: this.doc.rev,
+ version: this.doc.version,
+ })
+ .should.equal(true)
+ })
+ })
+
+ return describe('which is deleted', function () {
+ beforeEach(function () {
+ this.req.params = {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ }
+ return (this.DocManager.getFullDoc = sinon
+ .stub()
+ .callsArgWith(2, null, this.deletedDoc))
+ })
+
+ it('should get the doc from the doc manager', function () {
+ this.HttpController.getDoc(this.req, this.res, this.next)
+ return this.DocManager.getFullDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ it('should return 404 if the query string delete is not set ', function () {
+ this.HttpController.getDoc(this.req, this.res, this.next)
+ return this.res.sendStatus.calledWith(404).should.equal(true)
+ })
+
+ return it('should return the doc as JSON if include_deleted is set to true', function () {
+ this.req.query.include_deleted = 'true'
+ this.HttpController.getDoc(this.req, this.res, this.next)
+ return this.res.json
+ .calledWith({
+ _id: this.doc_id,
+ lines: this.doc.lines,
+ rev: this.doc.rev,
+ deleted: true,
+ version: this.doc.version,
+ })
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('getRawDoc', function () {
+ beforeEach(function () {
+ this.req.params = {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ }
+ this.DocManager.getDocLines = sinon.stub().callsArgWith(2, null, this.doc)
+ return this.HttpController.getRawDoc(this.req, this.res, this.next)
+ })
+
+ it('should get the document without the version', function () {
+ return this.DocManager.getDocLines
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ it('should set the content type header', function () {
+ return this.res.setHeader
+ .calledWith('content-type', 'text/plain')
+ .should.equal(true)
+ })
+
+ return it('should send the raw version of the doc', function () {
+ return assert.deepEqual(
+ this.res.send.args[0][0],
+ `${this.doc.lines[0]}\n${this.doc.lines[1]}\n${this.doc.lines[2]}\n${this.doc.lines[3]}\n${this.doc.lines[4]}\n${this.doc.lines[5]}`
+ )
+ })
+ })
+
+ describe('getAllDocs', function () {
+ describe('normally', function () {
+ beforeEach(function () {
+ this.req.params = { project_id: this.project_id }
+ this.docs = [
+ {
+ _id: ObjectId(),
+ lines: ['mock', 'lines', 'one'],
+ rev: 2,
+ },
+ {
+ _id: ObjectId(),
+ lines: ['mock', 'lines', 'two'],
+ rev: 4,
+ },
+ ]
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ return this.HttpController.getAllDocs(this.req, this.res, this.next)
+ })
+
+ it('should get all the (non-deleted) docs', function () {
+ return this.DocManager.getAllNonDeletedDocs
+ .calledWith(this.project_id, { lines: true, rev: true })
+ .should.equal(true)
+ })
+
+ return it('should return the doc as JSON', function () {
+ return this.res.json
+ .calledWith([
+ {
+ _id: this.docs[0]._id.toString(),
+ lines: this.docs[0].lines,
+ rev: this.docs[0].rev,
+ },
+ {
+ _id: this.docs[1]._id.toString(),
+ lines: this.docs[1].lines,
+ rev: this.docs[1].rev,
+ },
+ ])
+ .should.equal(true)
+ })
+ })
+
+ return describe('with a null doc', function () {
+ beforeEach(function () {
+ this.req.params = { project_id: this.project_id }
+ this.docs = [
+ {
+ _id: ObjectId(),
+ lines: ['mock', 'lines', 'one'],
+ rev: 2,
+ },
+ null,
+ {
+ _id: ObjectId(),
+ lines: ['mock', 'lines', 'two'],
+ rev: 4,
+ },
+ ]
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ return this.HttpController.getAllDocs(this.req, this.res, this.next)
+ })
+
+ it('should return the non null docs as JSON', function () {
+ return this.res.json
+ .calledWith([
+ {
+ _id: this.docs[0]._id.toString(),
+ lines: this.docs[0].lines,
+ rev: this.docs[0].rev,
+ },
+ {
+ _id: this.docs[2]._id.toString(),
+ lines: this.docs[2].lines,
+ rev: this.docs[2].rev,
+ },
+ ])
+ .should.equal(true)
+ })
+
+ return it('should log out an error', function () {
+ return this.logger.error
+ .calledWith(
+ {
+ err: sinon.match.has('message', 'null doc'),
+ project_id: this.project_id,
+ },
+ 'encountered null doc'
+ )
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('getAllRanges', function () {
+ return describe('normally', function () {
+ beforeEach(function () {
+ this.req.params = { project_id: this.project_id }
+ this.docs = [
+ {
+ _id: ObjectId(),
+ ranges: { mock_ranges: 'one' },
+ },
+ {
+ _id: ObjectId(),
+ ranges: { mock_ranges: 'two' },
+ },
+ ]
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ return this.HttpController.getAllRanges(this.req, this.res, this.next)
+ })
+
+ it('should get all the (non-deleted) doc ranges', function () {
+ return this.DocManager.getAllNonDeletedDocs
+ .calledWith(this.project_id, { ranges: true })
+ .should.equal(true)
+ })
+
+ return it('should return the doc as JSON', function () {
+ return this.res.json
+ .calledWith([
+ {
+ _id: this.docs[0]._id.toString(),
+ ranges: this.docs[0].ranges,
+ },
+ {
+ _id: this.docs[1]._id.toString(),
+ ranges: this.docs[1].ranges,
+ },
+ ])
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('updateDoc', function () {
+ beforeEach(function () {
+ return (this.req.params = {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ })
+ })
+
+ describe('when the doc lines exist and were updated', function () {
+ beforeEach(function () {
+ this.req.body = {
+ lines: (this.lines = ['hello', 'world']),
+ version: (this.version = 42),
+ ranges: (this.ranges = { changes: 'mock' }),
+ }
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .yields(null, true, (this.rev = 5))
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should update the document', function () {
+ return this.DocManager.updateDoc
+ .calledWith(
+ this.project_id,
+ this.doc_id,
+ this.lines,
+ this.version,
+ this.ranges
+ )
+ .should.equal(true)
+ })
+
+ return it('should return a modified status', function () {
+ return this.res.json
+ .calledWith({ modified: true, rev: this.rev })
+ .should.equal(true)
+ })
+ })
+
+ describe('when the doc lines exist and were not updated', function () {
+ beforeEach(function () {
+ this.req.body = {
+ lines: (this.lines = ['hello', 'world']),
+ version: (this.version = 42),
+ ranges: {},
+ }
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .yields(null, false, (this.rev = 5))
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ return it('should return a modified status', function () {
+ return this.res.json
+ .calledWith({ modified: false, rev: this.rev })
+ .should.equal(true)
+ })
+ })
+
+ describe('when the doc lines are not provided', function () {
+ beforeEach(function () {
+ this.req.body = { version: 42, ranges: {} }
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should not update the document', function () {
+ return this.DocManager.updateDoc.called.should.equal(false)
+ })
+
+ return it('should return a 400 (bad request) response', function () {
+ return this.res.sendStatus.calledWith(400).should.equal(true)
+ })
+ })
+
+ describe('when the doc version are not provided', function () {
+ beforeEach(function () {
+ this.req.body = { version: 42, lines: ['hello world'] }
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should not update the document', function () {
+ return this.DocManager.updateDoc.called.should.equal(false)
+ })
+
+ return it('should return a 400 (bad request) response', function () {
+ return this.res.sendStatus.calledWith(400).should.equal(true)
+ })
+ })
+
+ describe('when the doc ranges is not provided', function () {
+ beforeEach(function () {
+ this.req.body = { lines: ['foo'], version: 42 }
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should not update the document', function () {
+ return this.DocManager.updateDoc.called.should.equal(false)
+ })
+
+ return it('should return a 400 (bad request) response', function () {
+ return this.res.sendStatus.calledWith(400).should.equal(true)
+ })
+ })
+
+ return describe('when the doc body is too large', function () {
+ beforeEach(function () {
+ this.req.body = {
+ lines: (this.lines = Array(2049).fill('a'.repeat(1024))),
+ version: (this.version = 42),
+ ranges: (this.ranges = { changes: 'mock' }),
+ }
+ return this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should return a 413 (too large) response', function () {
+ return sinon.assert.calledWith(this.res.status, 413)
+ })
+
+ return it('should report that the document body is too large', function () {
+ return sinon.assert.calledWith(this.res.send, 'document body too large')
+ })
+ })
+ })
+
+ describe('patchDoc', function () {
+ beforeEach(function () {
+ this.req.params = {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ }
+ this.req.body = { name: 'foo.tex' }
+ this.DocManager.patchDoc = sinon.stub().yields(null)
+ this.HttpController.patchDoc(this.req, this.res, this.next)
+ })
+
+ it('should delete the document', function () {
+ expect(this.DocManager.patchDoc).to.have.been.calledWith(
+ this.project_id,
+ this.doc_id
+ )
+ })
+
+ it('should return a 204 (No Content)', function () {
+ expect(this.res.sendStatus).to.have.been.calledWith(204)
+ })
+
+ describe('with an invalid payload', function () {
+ beforeEach(function () {
+ this.req.body = { cannot: 'happen' }
+
+ this.DocManager.patchDoc = sinon.stub().yields(null)
+ this.HttpController.patchDoc(this.req, this.res, this.next)
+ })
+
+ it('should log a message', function () {
+ expect(this.logger.fatal).to.have.been.calledWith(
+ { field: 'cannot' },
+ 'joi validation for pathDoc is broken'
+ )
+ })
+
+ it('should not pass the invalid field along', function () {
+ expect(this.DocManager.patchDoc).to.have.been.calledWith(
+ this.project_id,
+ this.doc_id,
+ {}
+ )
+ })
+ })
+ })
+
+ describe('archiveAllDocs', function () {
+ beforeEach(function () {
+ this.req.params = { project_id: this.project_id }
+ this.DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1)
+ return this.HttpController.archiveAllDocs(this.req, this.res, this.next)
+ })
+
+ it('should archive the project', function () {
+ return this.DocArchiveManager.archiveAllDocs
+ .calledWith(this.project_id)
+ .should.equal(true)
+ })
+
+ return it('should return a 204 (No Content)', function () {
+ return this.res.sendStatus.calledWith(204).should.equal(true)
+ })
+ })
+
+ return describe('destroyAllDocs', function () {
+ beforeEach(function () {
+ this.req.params = { project_id: this.project_id }
+ this.DocArchiveManager.destroyAllDocs = sinon.stub().callsArg(1)
+ return this.HttpController.destroyAllDocs(this.req, this.res, this.next)
+ })
+
+ it('should destroy the docs', function () {
+ return sinon.assert.calledWith(
+ this.DocArchiveManager.destroyAllDocs,
+ this.project_id
+ )
+ })
+
+ return it('should return 204', function () {
+ return sinon.assert.calledWith(this.res.sendStatus, 204)
+ })
+ })
+})
diff --git a/services/docstore/test/unit/js/MongoManagerTests.js b/services/docstore/test/unit/js/MongoManagerTests.js
new file mode 100644
index 0000000000..5bfcb3c179
--- /dev/null
+++ b/services/docstore/test/unit/js/MongoManagerTests.js
@@ -0,0 +1,375 @@
+/* eslint-disable
+ handle-callback-err,
+ no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+const modulePath = require('path').join(
+ __dirname,
+ '../../../app/js/MongoManager'
+)
+const { ObjectId } = require('mongodb')
+const { assert } = require('chai')
+const Errors = require('../../../app/js/Errors')
+
+describe('MongoManager', function () {
+ beforeEach(function () {
+ this.MongoManager = SandboxedModule.require(modulePath, {
+ requires: {
+ './mongodb': {
+ db: (this.db = { docs: {}, docOps: {} }),
+ ObjectId,
+ },
+ '@overleaf/metrics': { timeAsyncMethod: sinon.stub() },
+ '@overleaf/settings': { max_deleted_docs: 42 },
+ './Errors': Errors,
+ },
+ })
+ this.project_id = ObjectId().toString()
+ this.doc_id = ObjectId().toString()
+ this.callback = sinon.stub()
+ return (this.stubbedErr = new Error('hello world'))
+ })
+
+ describe('findDoc', function () {
+ beforeEach(function () {
+ this.doc = { name: 'mock-doc' }
+ this.db.docs.findOne = sinon.stub().callsArgWith(2, null, this.doc)
+ this.filter = { lines: true }
+ return this.MongoManager.findDoc(
+ this.project_id,
+ this.doc_id,
+ this.filter,
+ this.callback
+ )
+ })
+
+ it('should find the doc', function () {
+ this.db.docs.findOne
+ .calledWith(
+ {
+ _id: ObjectId(this.doc_id),
+ project_id: ObjectId(this.project_id),
+ },
+ {
+ projection: this.filter,
+ }
+ )
+ .should.equal(true)
+ })
+
+ return it('should call the callback with the doc', function () {
+ return this.callback.calledWith(null, this.doc).should.equal(true)
+ })
+ })
+
+ describe('patchDoc', function () {
+ beforeEach(function (done) {
+ this.db.docs.updateOne = sinon.stub().yields(null)
+ this.meta = { name: 'foo.tex' }
+ this.callback.callsFake(done)
+ this.MongoManager.patchDoc(
+ this.project_id,
+ this.doc_id,
+ this.meta,
+ this.callback
+ )
+ })
+
+ it('should pass the parameter along', function () {
+ this.db.docs.updateOne.should.have.been.calledWith(
+ {
+ _id: ObjectId(this.doc_id),
+ project_id: ObjectId(this.project_id),
+ },
+ {
+ $set: this.meta,
+ },
+ this.callback
+ )
+ })
+ })
+
+ describe('getProjectsDocs', function () {
+ beforeEach(function () {
+ this.filter = { lines: true }
+ this.doc1 = { name: 'mock-doc1' }
+ this.doc2 = { name: 'mock-doc2' }
+ this.doc3 = { name: 'mock-doc3' }
+ this.doc4 = { name: 'mock-doc4' }
+ this.db.docs.find = sinon.stub().returns({
+ toArray: sinon
+ .stub()
+ .callsArgWith(0, null, [this.doc, this.doc3, this.doc4]),
+ })
+ })
+
+ describe('with included_deleted = false', function () {
+ beforeEach(function () {
+ return this.MongoManager.getProjectsDocs(
+ this.project_id,
+ { include_deleted: false },
+ this.filter,
+ this.callback
+ )
+ })
+
+ it('should find the non-deleted docs via the project_id', function () {
+ return this.db.docs.find
+ .calledWith(
+ {
+ project_id: ObjectId(this.project_id),
+ deleted: { $ne: true },
+ },
+ {
+ projection: this.filter,
+ }
+ )
+ .should.equal(true)
+ })
+
+ return it('should call the callback with the docs', function () {
+ return this.callback
+ .calledWith(null, [this.doc, this.doc3, this.doc4])
+ .should.equal(true)
+ })
+ })
+
+ return describe('with included_deleted = true', function () {
+ beforeEach(function () {
+ return this.MongoManager.getProjectsDocs(
+ this.project_id,
+ { include_deleted: true },
+ this.filter,
+ this.callback
+ )
+ })
+
+ it('should find all via the project_id', function () {
+ return this.db.docs.find
+ .calledWith(
+ {
+ project_id: ObjectId(this.project_id),
+ },
+ {
+ projection: this.filter,
+ }
+ )
+ .should.equal(true)
+ })
+
+ return it('should call the callback with the docs', function () {
+ return this.callback
+ .calledWith(null, [this.doc, this.doc3, this.doc4])
+ .should.equal(true)
+ })
+ })
+ })
+
+ describe('getProjectsDeletedDocs', function () {
+ beforeEach(function (done) {
+ this.filter = { name: true }
+ this.doc1 = { _id: '1', name: 'mock-doc1.tex' }
+ this.doc2 = { _id: '2', name: 'mock-doc2.tex' }
+ this.doc3 = { _id: '3', name: 'mock-doc3.tex' }
+ this.db.docs.find = sinon.stub().returns({
+ toArray: sinon.stub().yields(null, [this.doc1, this.doc2, this.doc3]),
+ })
+ this.callback.callsFake(done)
+ this.MongoManager.getProjectsDeletedDocs(
+ this.project_id,
+ this.filter,
+ this.callback
+ )
+ })
+
+ it('should find the deleted docs via the project_id', function () {
+ this.db.docs.find
+ .calledWith({
+ project_id: ObjectId(this.project_id),
+ deleted: true,
+ })
+ .should.equal(true)
+ })
+
+ it('should filter, sort by deletedAt and limit', function () {
+ this.db.docs.find
+ .calledWith(sinon.match.any, {
+ projection: this.filter,
+ sort: { deletedAt: -1 },
+ limit: 42,
+ })
+ .should.equal(true)
+ })
+
+ it('should call the callback with the docs', function () {
+ this.callback
+ .calledWith(null, [this.doc1, this.doc2, this.doc3])
+ .should.equal(true)
+ })
+ })
+
+ describe('upsertIntoDocCollection', function () {
+ beforeEach(function () {
+ this.db.docs.updateOne = sinon.stub().callsArgWith(3, this.stubbedErr)
+ return (this.oldRev = 77)
+ })
+
+ it('should upsert the document', function (done) {
+ return this.MongoManager.upsertIntoDocCollection(
+ this.project_id,
+ this.doc_id,
+ { lines: this.lines },
+ err => {
+ const args = this.db.docs.updateOne.args[0]
+ assert.deepEqual(args[0], { _id: ObjectId(this.doc_id) })
+ assert.equal(args[1].$set.lines, this.lines)
+ assert.equal(args[1].$inc.rev, 1)
+ assert.deepEqual(args[1].$set.project_id, ObjectId(this.project_id))
+ return done()
+ }
+ )
+ })
+
+ return it('should return the error', function (done) {
+ return this.MongoManager.upsertIntoDocCollection(
+ this.project_id,
+ this.doc_id,
+ { lines: this.lines },
+ err => {
+ err.should.equal(this.stubbedErr)
+ return done()
+ }
+ )
+ })
+ })
+
+ describe('destroyDoc', function () {
+ beforeEach(function (done) {
+ this.db.docs.deleteOne = sinon.stub().yields()
+ this.db.docOps.deleteOne = sinon.stub().yields()
+ return this.MongoManager.destroyDoc('123456789012', done)
+ })
+
+ it('should destroy the doc', function () {
+ return sinon.assert.calledWith(this.db.docs.deleteOne, {
+ _id: ObjectId('123456789012'),
+ })
+ })
+
+ return it('should destroy the docOps', function () {
+ return sinon.assert.calledWith(this.db.docOps.deleteOne, {
+ doc_id: ObjectId('123456789012'),
+ })
+ })
+ })
+
+ describe('getDocVersion', function () {
+ describe('when the doc exists', function () {
+ beforeEach(function () {
+ this.doc = { version: (this.version = 42) }
+ this.db.docOps.findOne = sinon.stub().callsArgWith(2, null, this.doc)
+ return this.MongoManager.getDocVersion(this.doc_id, this.callback)
+ })
+
+ it('should look for the doc in the database', function () {
+ return this.db.docOps.findOne
+ .calledWith(
+ { doc_id: ObjectId(this.doc_id) },
+ {
+ projection: { version: 1 },
+ }
+ )
+ .should.equal(true)
+ })
+
+ return it('should call the callback with the version', function () {
+ return this.callback.calledWith(null, this.version).should.equal(true)
+ })
+ })
+
+ return describe("when the doc doesn't exist", function () {
+ beforeEach(function () {
+ this.db.docOps.findOne = sinon.stub().callsArgWith(2, null, null)
+ return this.MongoManager.getDocVersion(this.doc_id, this.callback)
+ })
+
+ return it('should call the callback with 0', function () {
+ return this.callback.calledWith(null, 0).should.equal(true)
+ })
+ })
+ })
+
+ describe('setDocVersion', function () {
+ beforeEach(function () {
+ this.version = 42
+ this.db.docOps.updateOne = sinon.stub().callsArg(3)
+ return this.MongoManager.setDocVersion(
+ this.doc_id,
+ this.version,
+ this.callback
+ )
+ })
+
+ it('should update the doc version', function () {
+ return this.db.docOps.updateOne
+ .calledWith(
+ {
+ doc_id: ObjectId(this.doc_id),
+ },
+ {
+ $set: {
+ version: this.version,
+ },
+ },
+ {
+ upsert: true,
+ }
+ )
+ .should.equal(true)
+ })
+
+ return it('should call the callback', function () {
+ return this.callback.called.should.equal(true)
+ })
+ })
+
+ describe('withRevCheck', function () {
+ this.beforeEach(function () {
+ this.doc = { _id: ObjectId(), name: 'mock-doc', rev: 1 }
+ this.testFunction = sinon.stub().yields(null, 'foo')
+ })
+
+ it('should call the callback when the rev has not changed', function (done) {
+ this.db.docs.findOne = sinon.stub().callsArgWith(2, null, { rev: 1 })
+ this.MongoManager.withRevCheck(
+ this.doc,
+ this.testFunction,
+ (err, result) => {
+ result.should.equal('foo')
+ assert.isNull(err)
+ done()
+ }
+ )
+ })
+
+ it('should return an error when the rev has changed', function (done) {
+ this.db.docs.findOne = sinon.stub().callsArgWith(2, null, { rev: 2 })
+ this.MongoManager.withRevCheck(
+ this.doc,
+ this.testFunction,
+ (err, result) => {
+ err.should.be.instanceof(Errors.DocModifiedError)
+ done()
+ }
+ )
+ })
+ })
+})
diff --git a/services/docstore/test/unit/js/RangeManagerTests.js b/services/docstore/test/unit/js/RangeManagerTests.js
new file mode 100644
index 0000000000..271d9daa69
--- /dev/null
+++ b/services/docstore/test/unit/js/RangeManagerTests.js
@@ -0,0 +1,255 @@
+/* eslint-disable
+ camelcase,
+ no-return-assign,
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+const { assert, expect } = require('chai')
+const modulePath = require('path').join(
+ __dirname,
+ '../../../app/js/RangeManager'
+)
+const { ObjectId } = require('mongodb')
+const _ = require('underscore')
+
+describe('RangeManager', function () {
+ beforeEach(function () {
+ return (this.RangeManager = SandboxedModule.require(modulePath, {
+ requires: {
+ './mongodb': {
+ ObjectId,
+ },
+ },
+ }))
+ })
+
+ describe('jsonRangesToMongo', function () {
+ it('should convert ObjectIds and dates to proper objects', function () {
+ const change_id = ObjectId().toString()
+ const comment_id = ObjectId().toString()
+ const user_id = ObjectId().toString()
+ const thread_id = ObjectId().toString()
+ const ts = new Date().toJSON()
+ return this.RangeManager.jsonRangesToMongo({
+ changes: [
+ {
+ id: change_id,
+ op: { i: 'foo', p: 3 },
+ metadata: {
+ user_id,
+ ts,
+ },
+ },
+ ],
+ comments: [
+ {
+ id: comment_id,
+ op: { c: 'foo', p: 3, t: thread_id },
+ },
+ ],
+ }).should.deep.equal({
+ changes: [
+ {
+ id: ObjectId(change_id),
+ op: { i: 'foo', p: 3 },
+ metadata: {
+ user_id: ObjectId(user_id),
+ ts: new Date(ts),
+ },
+ },
+ ],
+ comments: [
+ {
+ id: ObjectId(comment_id),
+ op: { c: 'foo', p: 3, t: ObjectId(thread_id) },
+ },
+ ],
+ })
+ })
+
+ it('should leave malformed ObjectIds as they are', function () {
+ const change_id = 'foo'
+ const comment_id = 'bar'
+ const user_id = 'baz'
+ return this.RangeManager.jsonRangesToMongo({
+ changes: [
+ {
+ id: change_id,
+ metadata: {
+ user_id,
+ },
+ },
+ ],
+ comments: [
+ {
+ id: comment_id,
+ },
+ ],
+ }).should.deep.equal({
+ changes: [
+ {
+ id: change_id,
+ metadata: {
+ user_id,
+ },
+ },
+ ],
+ comments: [
+ {
+ id: comment_id,
+ },
+ ],
+ })
+ })
+
+ return it('should be consistent when transformed through json -> mongo -> json', function () {
+ const change_id = ObjectId().toString()
+ const comment_id = ObjectId().toString()
+ const user_id = ObjectId().toString()
+ const thread_id = ObjectId().toString()
+ const ts = new Date().toJSON()
+ const ranges1 = {
+ changes: [
+ {
+ id: change_id,
+ op: { i: 'foo', p: 3 },
+ metadata: {
+ user_id,
+ ts,
+ },
+ },
+ ],
+ comments: [
+ {
+ id: comment_id,
+ op: { c: 'foo', p: 3, t: thread_id },
+ },
+ ],
+ }
+ const ranges1_copy = JSON.parse(JSON.stringify(ranges1)) // jsonRangesToMongo modifies in place
+ const ranges2 = JSON.parse(
+ JSON.stringify(this.RangeManager.jsonRangesToMongo(ranges1_copy))
+ )
+ return ranges1.should.deep.equal(ranges2)
+ })
+ })
+
+ return describe('shouldUpdateRanges', function () {
+ beforeEach(function () {
+ this.ranges = {
+ changes: [
+ {
+ id: ObjectId(),
+ op: { i: 'foo', p: 3 },
+ metadata: {
+ user_id: ObjectId(),
+ ts: new Date(),
+ },
+ },
+ ],
+ comments: [
+ {
+ id: ObjectId(),
+ op: { c: 'foo', p: 3, t: ObjectId() },
+ },
+ ],
+ }
+ return (this.ranges_copy = this.RangeManager.jsonRangesToMongo(
+ JSON.parse(JSON.stringify(this.ranges))
+ ))
+ })
+
+ describe('with a blank new range', function () {
+ return it('should throw an error', function () {
+ return expect(() => {
+ return this.RangeManager.shouldUpdateRanges(this.ranges, null)
+ }).to.throw(Error)
+ })
+ })
+
+ describe('with a blank old range', function () {
+ return it('should treat it like {}', function () {
+ this.RangeManager.shouldUpdateRanges(null, {}).should.equal(false)
+ return this.RangeManager.shouldUpdateRanges(
+ null,
+ this.ranges
+ ).should.equal(true)
+ })
+ })
+
+ describe('with no changes', function () {
+ return it('should return false', function () {
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(false)
+ })
+ })
+
+ return describe('with changes', function () {
+ it('should return true when the change id changes', function () {
+ this.ranges_copy.changes[0].id = ObjectId()
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ it('should return true when the change user id changes', function () {
+ this.ranges_copy.changes[0].metadata.user_id = ObjectId()
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ it('should return true when the change ts changes', function () {
+ this.ranges_copy.changes[0].metadata.ts = new Date(Date.now() + 1000)
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ it('should return true when the change op changes', function () {
+ this.ranges_copy.changes[0].op.i = 'bar'
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ it('should return true when the comment id changes', function () {
+ this.ranges_copy.comments[0].id = ObjectId()
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ it('should return true when the comment offset changes', function () {
+ this.ranges_copy.comments[0].op.p = 17
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+
+ return it('should return true when the comment content changes', function () {
+ this.ranges_copy.comments[0].op.c = 'bar'
+ return this.RangeManager.shouldUpdateRanges(
+ this.ranges,
+ this.ranges_copy
+ ).should.equal(true)
+ })
+ })
+ })
+})