diff --git a/services/filestore/.gitignore b/services/filestore/.gitignore index 6ab729d0ae..d51931c976 100644 --- a/services/filestore/.gitignore +++ b/services/filestore/.gitignore @@ -40,6 +40,7 @@ test/IntergrationTests/js/* data/*/* app.js +cluster.js app/js/* test/IntergrationTests/js/* test/UnitTests/js/* diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee new file mode 100644 index 0000000000..d0101b93a4 --- /dev/null +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -0,0 +1,87 @@ +logger = require "logger-sharelatex" +aws = require "aws-sdk" +_ = require "underscore" +fs = require "fs" +Errors = require "./Errors" + +s3 = new aws.S3() + +module.exports = + sendFile: (bucketName, key, fsPath, callback)-> + logger.log bucketName:bucketName, key:key, "send file data to s3" + stream = fs.createReadStream fsPath + s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) -> + if err? + logger.err err: err, Bucket: bucketName, Key: key, "error sending file data to s3" + callback err + + sendStream: (bucketName, key, stream, callback)-> + logger.log bucketName:bucketName, key:key, "send file stream to s3" + s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) -> + if err? + logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3" + callback err + + getFileStream: (bucketName, key, opts, callback = (err, res)->)-> + logger.log bucketName:bucketName, key:key, "get file stream from s3" + callback = _.once callback + params = + Bucket:bucketName + Key: key + if opts.start? and opts.end? + params['Range'] = "bytes=#{opts.start}-#{opts.end}" + request = s3.getObject params + stream = request.createReadStream() + stream.on 'readable', () -> + callback null, stream + stream.on 'error', (err) -> + logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3" + if err.code == 'NoSuchKey' + return callback new Errors.NotFoundError "File not found in S3: #{bucketName}:#{key}" + callback err + + copyFile: (bucketName, sourceKey, destKey, callback)-> + logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3" + source = bucketName + '/' + sourceKey + s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) -> + if err? + logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in s3" + callback err + + deleteFile: (bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "delete file in s3" + s3.deleteObject {Bucket: bucketName, Key: key}, (err) -> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting file in s3" + callback err + + deleteDirectory: (bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "delete directory in s3" + s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" + return callback err + if data.Contents.length == 0 + logger.log bucketName:bucketName, key:key, "the directory is empty" + return callback() + keys = _.map data.Contents, (entry)-> + Key: entry.Key + s3.deleteObjects + Bucket: bucketName + Delete: + Objects: keys + Quiet: true + , (err) -> + if err? + logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3" + callback err + + checkIfFileExists:(bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "check file existence in s3" + s3.headObject {Bucket: bucketName, Key: key}, (err, data) -> + if err? + return (callback null, false) if err.code == 'NotFound' + logger.err err:err, bucketName:bucketName, key:key, "something went wrong checking head in s3" + return callback err + callback null, data.ETag? + diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 2ade1f3a5b..d1b72806cf 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -3,6 +3,7 @@ fs = require("fs") LocalFileWriter = require("./LocalFileWriter") Errors = require('./Errors') rimraf = require("rimraf") +_ = require "underscore" filterName = (key) -> return key.replace /\//g, "_" @@ -29,9 +30,7 @@ module.exports = # opts may be {start: Number, end: Number} getFileStream: (location, name, opts, _callback = (err, res)->) -> - callback = (args...) -> - _callback(args...) - _callback = () -> + callback = _.once _callback filteredName = filterName name logger.log location:location, name:filteredName, "getting file" sourceStream = fs.createReadStream "#{location}/#{filteredName}", opts diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index 698e311ca2..f22285edae 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -10,16 +10,16 @@ ImageOptimiser = require("./ImageOptimiser") module.exports = insertFile: (bucket, key, stream, callback)-> - convetedKey = KeyBuilder.getConvertedFolderKey(key) - PersistorManager.deleteDirectory bucket, convetedKey, (error) -> + convertedKey = KeyBuilder.getConvertedFolderKey key + PersistorManager.deleteDirectory bucket, convertedKey, (error) -> return callback(error) if error? PersistorManager.sendStream bucket, key, stream, callback deleteFile: (bucket, key, callback)-> - convetedKey = KeyBuilder.getConvertedFolderKey(key) + convertedKey = KeyBuilder.getConvertedFolderKey key async.parallel [ (done)-> PersistorManager.deleteFile bucket, key, done - (done)-> PersistorManager.deleteDirectory bucket, convetedKey, done + (done)-> PersistorManager.deleteDirectory bucket, convertedKey, done ], callback getFile: (bucket, key, opts = {}, callback)-> @@ -36,49 +36,48 @@ module.exports = callback err, fileStream _getConvertedFile: (bucket, key, opts, callback)-> - convetedKey = KeyBuilder.addCachingToKey(key, opts) - PersistorManager.checkIfFileExists bucket, convetedKey, (err, exists)=> + convertedKey = KeyBuilder.addCachingToKey key, opts + PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=> if err? - return callback(err) + return callback err if exists - PersistorManager.getFileStream bucket, convetedKey, opts, callback + PersistorManager.getFileStream bucket, convertedKey, opts, callback else - @_getConvertedFileAndCache bucket, key, convetedKey, opts, callback + @_getConvertedFileAndCache bucket, key, convertedKey, opts, callback - _getConvertedFileAndCache: (bucket, key, convetedKey, opts, callback)-> - self = @ + _getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)-> convertedFsPath = "" async.series [ - (cb)-> - self._convertFile bucket, key, opts, (err, fileSystemPath)-> + (cb) => + @_convertFile bucket, key, opts, (err, fileSystemPath) -> convertedFsPath = fileSystemPath cb err (cb)-> ImageOptimiser.compressPng convertedFsPath, cb (cb)-> - PersistorManager.sendFile bucket, convetedKey, convertedFsPath, cb + PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb ], (err)-> if err? return callback(err) - PersistorManager.getFileStream bucket, convetedKey, opts, callback + PersistorManager.getFileStream bucket, convertedKey, opts, callback - _convertFile: (bucket, origonalKey, opts, callback)-> - @_writeS3FileToDisk bucket, origonalKey, opts, (err, origonalFsPath)-> + _convertFile: (bucket, originalKey, opts, callback)-> + @_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)-> if err? return callback(err) done = (err, destPath)-> if err? - logger.err err:err, bucket:bucket, origonalKey:origonalKey, opts:opts, "error converting file" + logger.err err:err, bucket:bucket, originalKey:originalKey, opts:opts, "error converting file" return callback(err) - LocalFileWriter.deleteFile origonalFsPath, -> + LocalFileWriter.deleteFile originalFsPath, -> callback(err, destPath) if opts.format? - FileConverter.convert origonalFsPath, opts.format, done + FileConverter.convert originalFsPath, opts.format, done else if opts.style == "thumbnail" - FileConverter.thumbnail origonalFsPath, done + FileConverter.thumbnail originalFsPath, done else if opts.style == "preview" - FileConverter.preview origonalFsPath, done + FileConverter.preview originalFsPath, done else return callback(new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")) diff --git a/services/filestore/app/coffee/PersistorManager.coffee b/services/filestore/app/coffee/PersistorManager.coffee index 1dad923098..aa5c80599d 100644 --- a/services/filestore/app/coffee/PersistorManager.coffee +++ b/services/filestore/app/coffee/PersistorManager.coffee @@ -7,6 +7,8 @@ settings.filestore.backend ||= "s3" logger.log backend:settings.filestore.backend, "Loading backend" module.exports = switch settings.filestore.backend + when "aws-sdk" + require "./AWSSDKPersistorManager" when "s3" require("./S3PersistorManager") when "fs" diff --git a/services/filestore/package.json b/services/filestore/package.json index 03443724e8..184899250e 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -8,17 +8,12 @@ }, "dependencies": { "async": "~0.2.10", - "bunyan": "^1.3.5", + "aws-sdk": "^2.1.39", "coffee-script": "~1.7.1", "express": "~3.4.8", - "grunt-bunyan": "^0.5.0", - "grunt-execute": "^0.2.2", - "grunt-mocha-test": "~0.8.2", "heapdump": "^0.3.2", "knox": "~0.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", - "longjohn": "~0.2.2", - "lynx": "0.0.11", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", "node-transloadit": "0.0.4", "node-uuid": "~1.4.1", @@ -36,7 +31,11 @@ "sinon": "", "chai": "", "sandboxed-module": "", + "bunyan": "^1.3.5", "grunt": "0.4.1", + "grunt-bunyan": "^0.5.0", + "grunt-execute": "^0.2.2", + "grunt-mocha-test": "~0.8.2", "grunt-contrib-requirejs": "0.4.1", "grunt-contrib-coffee": "0.7.0", "grunt-contrib-watch": "0.5.3", diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee new file mode 100644 index 0000000000..0ca8c65ffc --- /dev/null +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee @@ -0,0 +1,249 @@ +sinon = require 'sinon' +chai = require 'chai' + +should = chai.should() +expect = chai.expect + +modulePath = "../../../app/js/AWSSDKPersistorManager.js" +SandboxedModule = require 'sandboxed-module' + +describe "AWSSDKPersistorManager", -> + beforeEach -> + @settings = + filestore: + backend: "aws-sdk" + @s3 = + upload: sinon.stub() + getObject: sinon.stub() + copyObject: sinon.stub() + deleteObject: sinon.stub() + listObjects: sinon.stub() + deleteObjects: sinon.stub() + headObject: sinon.stub() + @awssdk = + S3: sinon.stub().returns @s3 + + @requires = + "aws-sdk": @awssdk + "settings-sharelatex": @settings + "logger-sharelatex": + log:-> + err:-> + "fs": @fs = + createReadStream: sinon.stub() + "./Errors": @Errors = + NotFoundError: sinon.stub() + @key = "my/key" + @bucketName = "my-bucket" + @error = "my error" + @AWSSDKPersistorManager = SandboxedModule.require modulePath, requires: @requires + + describe "sendFile", -> + beforeEach -> + @stream = {} + @fsPath = "/usr/local/some/file" + @fs.createReadStream.returns @stream + + it "should put the file with s3.upload", (done) -> + @s3.upload.callsArgWith 1 + @AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) => + expect(err).to.not.be.ok + expect(@s3.upload.calledOnce, "called only once").to.be.true + expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream) + , "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.upload", (done) -> + @s3.upload.callsArgWith 1, @error + @AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) => + expect(err).to.equal @error + done() + + + describe "sendStream", -> + beforeEach -> + @stream = {} + + it "should put the file with s3.upload", (done) -> + @s3.upload.callsArgWith 1 + @AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) => + expect(err).to.not.be.ok + expect(@s3.upload.calledOnce, "called only once").to.be.true + expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.upload", (done) -> + @s3.upload.callsArgWith 1, @error + @AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) => + expect(err).to.equal @error + done() + + describe "getFileStream", -> + beforeEach -> + @opts = {} + @stream = {} + @read_stream = + on: @read_stream_on = sinon.stub() + @object = + createReadStream: sinon.stub().returns @read_stream + @s3.getObject.returns @object + + it "should return a stream from s3.getObject", (done) -> + @read_stream_on.withArgs('readable').callsArgWith 1 + + @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => + expect(@read_stream_on.calledTwice) + expect(err).to.not.be.ok + expect(stream, "returned the stream").to.equal @read_stream + expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key), + "called with correct arguments").to.be.true + done() + + describe "with start and end options", -> + beforeEach -> + @opts = + start: 0 + end: 8 + it "should pass headers to the s3.GetObject", (done) -> + @read_stream_on.withArgs('readable').callsArgWith 1 + @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => + expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key, Range: 'bytes=0-8'), + "called with correct arguments").to.be.true + done() + + describe "error conditions", -> + describe "when the file doesn't exist", -> + beforeEach -> + @error = new Error() + @error.code = 'NoSuchKey' + it "should produce a NotFoundError", (done) -> + @read_stream_on.withArgs('error').callsArgWith 1, @error + @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => + expect(stream).to.not.be.ok + expect(err).to.be.ok + expect(err instanceof @Errors.NotFoundError, "error is a correct instance").to.equal true + done() + + describe "when there is some other error", -> + beforeEach -> + @error = new Error() + it "should dispatch the error from s3 object stream", (done) -> + @read_stream_on.withArgs('error').callsArgWith 1, @error + @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => + expect(stream).to.not.be.ok + expect(err).to.be.ok + expect(err).to.equal @error + done() + + describe "copyFile", -> + beforeEach -> + @destKey = "some/key" + @stream = {} + + it "should copy the file with s3.copyObject", (done) -> + @s3.copyObject.callsArgWith 1 + @AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) => + expect(err).to.not.be.ok + expect(@s3.copyObject.calledOnce, "called only once").to.be.true + expect((@s3.copyObject.calledWith Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.copyObject", (done) -> + @s3.copyObject.callsArgWith 1, @error + @AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) => + expect(err).to.equal @error + done() + + describe "deleteFile", -> + it "should delete the file with s3.deleteObject", (done) -> + @s3.deleteObject.callsArgWith 1 + @AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) => + expect(err).to.not.be.ok + expect(@s3.deleteObject.calledOnce, "called only once").to.be.true + expect((@s3.deleteObject.calledWith Bucket: @bucketName, Key: @key), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.deleteObject", (done) -> + @s3.deleteObject.callsArgWith 1, @error + @AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) => + expect(err).to.equal @error + done() + + describe "deleteDirectory", -> + + it "should list the directory content using s3.listObjects", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: [] + @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => + expect(err).to.not.be.ok + expect(@s3.listObjects.calledOnce, "called only once").to.be.true + expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.listObjects", (done) -> + @s3.listObjects.callsArgWith 1, @error + @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => + expect(err).to.equal @error + done() + + describe "with directory content", -> + beforeEach -> + @fileList = [ + Key: 'foo' + , Key: 'bar' + , Key: 'baz' + ] + + it "should forward the file keys to s3.deleteObjects", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: @fileList + @s3.deleteObjects.callsArgWith 1 + @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => + expect(err).to.not.be.ok + expect(@s3.deleteObjects.calledOnce, "called only once").to.be.true + expect((@s3.deleteObjects.calledWith + Bucket: @bucketName + Delete: + Quiet: true + Objects: @fileList), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.deleteObjects", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: @fileList + @s3.deleteObjects.callsArgWith 1, @error + @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => + expect(err).to.equal @error + done() + + + describe "checkIfFileExists", -> + + it "should check for the file with s3.headObject", (done) -> + @s3.headObject.callsArgWith 1, null, {} + @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => + expect(err).to.not.be.ok + expect(@s3.headObject.calledOnce, "called only once").to.be.true + expect((@s3.headObject.calledWith Bucket: @bucketName, Key: @key), + "called with correct arguments").to.be.true + done() + + it "should return false on an inexistant file", (done) -> + @s3.headObject.callsArgWith 1, null, {} + @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => + expect(exists).to.be.false + done() + + it "should return true on an existing file", (done) -> + @s3.headObject.callsArgWith 1, null, ETag: "etag" + @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => + expect(exists).to.be.true + done() + + it "should dispatch the error from s3.headObject", (done) -> + @s3.headObject.callsArgWith 1, @error + @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => + expect(err).to.equal @error + done()