diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 44f5ec3263..fa498f3f1b 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1014,6 +1014,15 @@ "type-detect": "^4.0.5" } }, + "chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "requires": { + "check-error": "^1.0.2" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -1284,6 +1293,12 @@ "integrity": "sha1-gAwN0eCov7yVg1wgKtIg/jF+WhI=", "dev": true }, + "disrequire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/disrequire/-/disrequire-1.1.0.tgz", + "integrity": "sha512-c3lya+wBcnfNipVE7XQC85J6Fty9XWsbNrUub8XT1Qk3mwO6f8tR7P6Ah3X09A3HTQ1biwjcwTLFkGlEejUzUw==", + "dev": true + }, "dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -4446,53 +4461,115 @@ "dev": true }, "request": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.14.0.tgz", - "integrity": "sha1-DYrLsLFMGrguAAt9OB+oyA0afYg=", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { - "form-data": "~0.0.3", - "mime": "~1.2.7" + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" }, "dependencies": { - "form-data": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.7.tgz", - "integrity": "sha1-chEYKiaiZs45cQ3IvEqBtwQIWb4=", + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { - "async": "~0.1.9", - "combined-stream": "~0.0.4", - "mime": "~1.2.2" - }, - "dependencies": { - "async": { - "version": "0.1.22", - "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz", - "integrity": "sha1-D8GqoIig4+8Ovi2IMbqw3PiEUGE=" - }, - "combined-stream": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.4.tgz", - "integrity": "sha1-LRpDNH2+lRWkonlnMuW4hHOECyI=", - "requires": { - "delayed-stream": "0.0.5" - }, - "dependencies": { - "delayed-stream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", - "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=" - } - } - } + "delayed-stream": "~1.0.0" } }, - "mime": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz", - "integrity": "sha1-AJzUCGe9Nd5SGzuWbwTi+NTRPQk=" + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "uuid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" } } }, + "request-promise-core": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.3.tgz", + "integrity": "sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==", + "requires": { + "lodash": "^4.17.15" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + } + } + }, + "request-promise-native": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz", + "integrity": "sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ==", + "requires": { + "request-promise-core": "1.1.3", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -4878,6 +4955,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, "stream-browserify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index d39d1027be..9515c1850c 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -36,7 +36,8 @@ "pngcrush": "0.0.3", "range-parser": "^1.0.2", "recluster": "^0.3.7", - "request": "2.14.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.8", "response": "0.14.0", "rimraf": "2.2.8", "settings-sharelatex": "^1.1.0", @@ -48,6 +49,8 @@ "babel-eslint": "^10.0.3", "bunyan": "^1.3.5", "chai": "4.2.0", + "chai-as-promised": "^7.1.1", + "disrequire": "^1.1.0", "eslint": "^6.4.0", "eslint-config-prettier": "^6.7.0", "eslint-config-standard": "^14.1.0", diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 3a5103d5f6..4035262cbc 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -1,109 +1,112 @@ -/* eslint-disable - handle-callback-err, - standard/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const app = require('../../../app') -require('logger-sharelatex').logger.level('info') const logger = require('logger-sharelatex') const Settings = require('settings-sharelatex') +const fs = require('fs') +const Path = require('path') const request = require('request') +const { promisify } = require('util') +const disrequire = require('disrequire') const S3_TRIES = 30 -module.exports = { - running: false, - initing: false, - callbacks: [], - ensureRunning(callback) { - if (callback == null) { - callback = function(error) {} - } - if (this.running) { - return callback() - } else if (this.initing) { - return this.callbacks.push(callback) - } else { - this.initing = true - this.callbacks.push(callback) - return app.listen( - __guard__( - Settings.internal != null ? Settings.internal.filestore : undefined, - x => x.port - ), - 'localhost', - error => { - if (error != null) { - throw error - } - this.running = true - logger.log('filestore running in dev mode') +logger.logger.level('info') - return (() => { - const result = [] - for (callback of Array.from(this.callbacks)) { - result.push(callback()) - } - return result - })() +const fsReaddir = promisify(fs.readdir) + +class FilestoreApp { + constructor() { + this.running = false + this.initing = false + } + + async runServer() { + if (this.running) { + return + } + + if (this.initing) { + return this.waitForInit() + } + this.initing = true + + this.app = await FilestoreApp.requireApp() + + await new Promise((resolve, reject) => { + this.server = this.app.listen( + Settings.internal.filestore.port, + 'localhost', + err => { + if (err) { + return reject(err) + } + resolve() } ) - } - }, + }) - waitForS3(callback, tries) { - if ( - !(Settings.filestore.s3 != null - ? Settings.filestore.s3.endpoint - : undefined) - ) { - return callback() - } - if (!tries) { - tries = 1 - } - - return request.get( - `${Settings.filestore.s3.endpoint}/`, - (err, response) => { - console.log( - err, - response != null ? response.statusCode : undefined, - tries - ) - if ( - !err && - [200, 404].includes( - response != null ? response.statusCode : undefined - ) - ) { - return callback() - } - - if (tries === S3_TRIES) { - return callback('timed out waiting for S3') - } - - return setTimeout(() => { - return this.waitForS3(callback, tries + 1) - }, 1000) + if (Settings.filestore.backend === 's3') { + try { + await FilestoreApp.waitForS3() + } catch (err) { + await this.stop() + throw err } - ) + } + + this.initing = false + } + + async waitForInit() { + while (this.initing) { + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + + async stop() { + if (this.server) { + await new Promise(resolve => { + this.server.close(resolve) + }) + delete this.server + } + } + + static async waitForS3() { + let tries = 0 + if (!Settings.filestore.s3.endpoint) { + return + } + + let s3Available = false + + while (tries < S3_TRIES && !s3Available) { + try { + const response = await promisify(request.get)( + `${Settings.filestore.s3.endpoint}/` + ) + if ([200, 404].includes(response.statusCode)) { + s3Available = true + } + } catch (err) { + } finally { + tries++ + if (!s3Available) { + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + } + } + + static async requireApp() { + // unload the app, as we may be doing this on multiple runs with + // different settings, which affect startup in some cases + const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js')) + files.forEach(file => { + disrequire(Path.resolve(__dirname, '../../../app/js', file)) + }) + disrequire(Path.resolve(__dirname, '../../../app')) + + return require('../../../app') } } -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} +module.exports = FilestoreApp diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js new file mode 100644 index 0000000000..3315569a24 --- /dev/null +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -0,0 +1,299 @@ +const chai = require('chai') +const { expect } = chai +const fs = require('fs') +const Settings = require('settings-sharelatex') +const Path = require('path') +const FilestoreApp = require('./FilestoreApp') +const rp = require('request-promise-native').defaults({ + resolveWithFullResponse: true +}) +const Stream = require('stream') +const request = require('request') +const { promisify } = require('util') +chai.use(require('chai-as-promised')) + +const fsWriteFile = promisify(fs.writeFile) +const fsStat = promisify(fs.stat) +const pipeline = promisify(Stream.pipeline) + +async function getMetric(filestoreUrl, metric) { + const res = await rp.get(`${filestoreUrl}/metrics`) + expect(res.statusCode).to.equal(200) + const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm') + const found = metricRegex.exec(res.body) + return parseInt(found ? found[1] : 0) || 0 +} + +// store settings for multiple backends, so that we can test each one. +// fs will always be available - add others if they are configured +const BackendSettings = { + FSPersistor: { + backend: 'fs', + stores: { + user_files: Path.resolve(__dirname, '../../../user_files'), + public_files: Path.resolve(__dirname, '../../../public_files'), + template_files: Path.resolve(__dirname, '../../../template_files') + } + } +} + +if (process.env.AWS_ACCESS_KEY_ID) { + BackendSettings.S3Persistor = { + backend: 's3', + s3: { + key: process.env.AWS_ACCESS_KEY_ID, + secret: process.env.AWS_SECRET_ACCESS_KEY, + endpoint: process.env.AWS_S3_ENDPOINT + }, + stores: { + user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME, + template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME, + public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME + } + } +} + +describe('Filestore', function() { + this.timeout(1000 * 10) + const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}` + + // redefine the test suite for every available backend + Object.keys(BackendSettings).forEach(backend => { + describe(backend, function() { + let app, previousEgress, previousIngress + + before(async function() { + // create the app with the relevant filestore settings + Settings.filestore = BackendSettings[backend] + app = new FilestoreApp() + await app.runServer() + }) + + after(async function() { + return app.stop() + }) + + beforeEach(async function() { + // retrieve previous metrics from the app + if (Settings.filestore.backend === 's3') { + ;[previousEgress, previousIngress] = await Promise.all([ + getMetric(filestoreUrl, 's3_egress'), + getMetric(filestoreUrl, 's3_ingress') + ]) + } + }) + + it('should send a 200 for the status endpoint', async function() { + const response = await rp(`${filestoreUrl}/status`) + expect(response.statusCode).to.equal(200) + expect(response.body).to.contain('filestore') + expect(response.body).to.contain('up') + }) + + it('should send a 200 for the health-check endpoint', async function() { + const response = await rp(`${filestoreUrl}/health_check`) + expect(response.statusCode).to.equal(200) + expect(response.body).to.equal('OK') + }) + + describe('with a file on the server', function() { + let fileId, fileUrl + + const localFileReadPath = + '/tmp/filestore_acceptance_tests_file_read.txt' + const constantFileContent = [ + 'hello world', + `line 2 goes here ${Math.random()}`, + 'there are 3 lines in all' + ].join('\n') + + before(async function() { + await fsWriteFile(localFileReadPath, constantFileContent) + }) + + beforeEach(async function() { + fileId = Math.random() + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + + const writeStream = request.post(fileUrl) + const readStream = fs.createReadStream(localFileReadPath) + // consume the result to ensure the http request has been fully processed + const resultStream = fs.createWriteStream('/dev/null') + await pipeline(readStream, writeStream, resultStream) + }) + + it('should return 404 for a non-existant id', async function() { + const options = { uri: fileUrl + '___this_is_clearly_wrong___' } + await expect( + rp.get(options) + ).to.eventually.be.rejected.and.have.property('statusCode', 404) + }) + + it('should return the file size on a HEAD request', async function() { + const expectedLength = Buffer.byteLength(constantFileContent) + const res = await rp.head(fileUrl) + expect(res.statusCode).to.equal(200) + expect(res.headers['content-length']).to.equal( + expectedLength.toString() + ) + }) + + it('should be able get the file back', async function() { + const res = await rp.get(fileUrl) + expect(res.body).to.equal(constantFileContent) + }) + + it('should be able to get back the first 9 bytes of the file', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + const res = await rp.get(options) + expect(res.body).to.equal('hello wor') + }) + + it('should be able to get back bytes 4 through 10 of the file', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=4-10' + } + } + const res = await rp.get(options) + expect(res.body).to.equal('o world') + }) + + it('should be able to delete the file', async function() { + const response = await rp.del(fileUrl) + expect(response.statusCode).to.equal(204) + await expect( + rp.get(fileUrl) + ).to.eventually.be.rejected.and.have.property('statusCode', 404) + }) + + it('should be able to copy files', async function() { + const newProjectID = 'acceptance_tests_copyied_project' + const newFileId = Math.random() + const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` + const opts = { + method: 'put', + uri: newFileUrl, + json: { + source: { + project_id: 'acceptance_tests', + file_id: fileId + } + } + } + let response = await rp(opts) + expect(response.statusCode).to.equal(200) + response = await rp.del(fileUrl) + expect(response.statusCode).to.equal(204) + response = await rp.get(newFileUrl) + expect(response.body).to.equal(constantFileContent) + }) + + if (backend === 'S3Persistor') { + it('should record an egress metric for the upload', async function() { + const metric = await getMetric(filestoreUrl, 's3_egress') + expect(metric - previousEgress).to.equal(constantFileContent.length) + }) + + it('should record an ingress metric when downloading the file', async function() { + await rp.get(fileUrl) + const metric = await getMetric(filestoreUrl, 's3_ingress') + expect(metric - previousIngress).to.equal( + constantFileContent.length + ) + }) + + it('should record an ingress metric for a partial download', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + await rp.get(options) + const metric = await getMetric(filestoreUrl, 's3_ingress') + expect(metric - previousIngress).to.equal(9) + }) + } + }) + + describe('with a pdf file', function() { + let fileId, fileUrl, localFileSize + const localFileReadPath = Path.resolve( + __dirname, + '../../fixtures/test.pdf' + ) + + beforeEach(async function() { + fileId = Math.random() + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + const stat = await fsStat(localFileReadPath) + localFileSize = stat.size + const writeStream = request.post(fileUrl) + const endStream = fs.createWriteStream('/dev/null') + const readStream = fs.createReadStream(localFileReadPath) + await pipeline(readStream, writeStream, endStream) + }) + + it('should be able get the file back', async function() { + const response = await rp.get(fileUrl) + expect(response.body.substring(0, 8)).to.equal('%PDF-1.5') + }) + + if (backend === 'S3Persistor') { + it('should record an egress metric for the upload', async function() { + const metric = await getMetric(filestoreUrl, 's3_egress') + expect(metric - previousEgress).to.equal(localFileSize) + }) + } + + describe('getting the preview image', function() { + this.timeout(1000 * 20) + let previewFileUrl + + beforeEach(function() { + previewFileUrl = `${fileUrl}?style=preview` + }) + + it('should not time out', async function() { + const response = await rp.get(previewFileUrl) + expect(response.statusCode).to.equal(200) + }) + + it('should respond with image data', async function() { + // note: this test relies of the imagemagick conversion working + const response = await rp.get(previewFileUrl) + expect(response.body.length).to.be.greaterThan(400) + expect(response.body.substr(1, 3)).to.equal('PNG') + }) + }) + + describe('warming the cache', function() { + this.timeout(1000 * 20) + let previewFileUrl + + beforeEach(function() { + previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true` + }) + + it('should not time out', async function() { + const response = await rp.get(previewFileUrl) + expect(response.statusCode).to.equal(200) + }) + + it("should respond with only an 'OK'", async function() { + // note: this test relies of the imagemagick conversion working + const response = await rp.get(previewFileUrl) + expect(response.body).to.equal('OK') + }) + }) + }) + }) + }) +}) diff --git a/services/filestore/test/acceptance/js/SendingFileTest.js b/services/filestore/test/acceptance/js/SendingFileTest.js deleted file mode 100644 index c20fa01c42..0000000000 --- a/services/filestore/test/acceptance/js/SendingFileTest.js +++ /dev/null @@ -1,326 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-path-concat, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') -const sinon = require('sinon') -const chai = require('chai') -const should = chai.should() -const { expect } = chai -const modulePath = '../../../app/js/LocalFileWriter.js' -const SandboxedModule = require('sandboxed-module') -const fs = require('fs') -const request = require('request') -const settings = require('settings-sharelatex') -const FilestoreApp = require('./FilestoreApp') -const async = require('async') - -const getMetric = (filestoreUrl, metric, cb) => - request.get(`${filestoreUrl}/metrics`, function(err, res) { - expect(res.statusCode).to.equal(200) - const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm') - return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0')) - }) - -describe('Filestore', function() { - before(function(done) { - this.localFileReadPath = '/tmp/filestore_acceptence_tests_file_read.txt' - this.localFileWritePath = '/tmp/filestore_acceptence_tests_file_write.txt' - - this.constantFileContent = [ - 'hello world', - `line 2 goes here ${Math.random()}`, - 'there are 3 lines in all' - ].join('\n') - - this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}` - return fs.writeFile( - this.localFileReadPath, - this.constantFileContent, - function(err) { - if (err) { - return done(err) - } - return FilestoreApp.waitForS3(done) - } - ) - }) - - beforeEach(function(done) { - return FilestoreApp.ensureRunning(() => { - return async.parallel( - [ - cb => { - return fs.unlink(this.localFileWritePath, () => cb()) - }, - cb => { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - this.previousEgress = metric - return cb() - }) - }, - cb => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - this.previousIngress = metric - return cb() - }) - } - ], - done - ) - }) - }) - - it('should send a 200 for status endpoint', function(done) { - return request(`${this.filestoreUrl}/status`, function( - err, - response, - body - ) { - response.statusCode.should.equal(200) - body.indexOf('filestore').should.not.equal(-1) - body.indexOf('up').should.not.equal(-1) - return done() - }) - }) - - describe('with a file on the server', function() { - beforeEach(function(done) { - this.timeout(1000 * 10) - this.file_id = Math.random() - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` - - const writeStream = request.post(this.fileUrl) - - writeStream.on('end', done) - return fs.createReadStream(this.localFileReadPath).pipe(writeStream) - }) - - it('should return 404 for a non-existant id', function(done) { - this.timeout(1000 * 20) - const options = { uri: this.fileUrl + '___this_is_clearly_wrong___' } - return request.get(options, (err, response, body) => { - response.statusCode.should.equal(404) - return done() - }) - }) - - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal( - this.constantFileContent.length - ) - return done() - }) - }) - - it('should return the file size on a HEAD request', function(done) { - const expectedLength = Buffer.byteLength(this.constantFileContent) - return request.head(this.fileUrl, (err, res) => { - expect(res.statusCode).to.equal(200) - expect(res.headers['content-length']).to.equal( - expectedLength.toString() - ) - return done() - }) - }) - - it('should be able get the file back', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, (err, response, body) => { - body.should.equal(this.constantFileContent) - return done() - }) - }) - - it('should record an ingress metric when downloading the file', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, () => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal( - this.constantFileContent.length - ) - return done() - }) - }) - }) - - it('should be able to get back the first 9 bytes of the file', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=0-8' - } - } - return request.get(options, (err, response, body) => { - body.should.equal('hello wor') - return done() - }) - }) - - it('should record an ingress metric for a partial download', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=0-8' - } - } - return request.get(options, () => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal(9) - return done() - }) - }) - }) - - it('should be able to get back bytes 4 through 10 of the file', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=4-10' - } - } - return request.get(options, (err, response, body) => { - body.should.equal('o world') - return done() - }) - }) - - it('should be able to delete the file', function(done) { - this.timeout(1000 * 20) - return request.del(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(204) - return request.get(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(404) - return done() - }) - }) - }) - - return it('should be able to copy files', function(done) { - this.timeout(1000 * 20) - - const newProjectID = 'acceptence_tests_copyied_project' - const newFileId = Math.random() - const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}` - const opts = { - method: 'put', - uri: newFileUrl, - json: { - source: { - project_id: 'acceptence_tests', - file_id: this.file_id - } - } - } - return request(opts, (err, response, body) => { - response.statusCode.should.equal(200) - return request.del(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(204) - return request.get(newFileUrl, (err, response, body) => { - body.should.equal(this.constantFileContent) - return done() - }) - }) - }) - }) - }) - - return describe('with a pdf file', function() { - beforeEach(function(done) { - this.timeout(1000 * 10) - this.file_id = Math.random() - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` - this.localFileReadPath = __dirname + '/../../fixtures/test.pdf' - return fs.stat(this.localFileReadPath, (err, stat) => { - this.localFileSize = stat.size - const writeStream = request.post(this.fileUrl) - - writeStream.on('end', done) - return fs.createReadStream(this.localFileReadPath).pipe(writeStream) - }) - }) - - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal(this.localFileSize) - return done() - }) - }) - - it('should be able get the file back', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, (err, response, body) => { - expect(body.substring(0, 8)).to.equal('%PDF-1.5') - return done() - }) - }) - - describe('getting the preview image', function() { - beforeEach(function() { - return (this.previewFileUrl = `${this.fileUrl}?style=preview`) - }) - - it('should not time out', function(done) { - this.timeout(1000 * 20) - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response).to.not.equal(null) - return done() - }) - }) - - return it('should respond with image data', function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20) - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200) - expect(body.length).to.be.greaterThan(400) - return done() - }) - }) - }) - - return describe('warming the cache', function() { - beforeEach(function() { - return (this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true') - }) - - it('should not time out', function(done) { - this.timeout(1000 * 20) - return request.get(this.fileUrl, (err, response, body) => { - expect(response).to.not.equal(null) - return done() - }) - }) - - return it("should respond with only an 'OK'", function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20) - return request.get(this.fileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200) - body.should.equal('OK') - return done() - }) - }) - }) - }) -}) - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -}