diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index e78355f4e6..04c13eb4b5 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -85,6 +85,8 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile +app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize + app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> res.send filename diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index d0101b93a4..5be80506f5 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -85,3 +85,17 @@ module.exports = return callback err callback null, data.ETag? + directorySize:(bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "get project size in s3" + s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" + return callback err + if data.Contents.length == 0 + logger.log bucketName:bucketName, key:key, "the directory is empty" + return callback() + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + callback null, totalSize + diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index d1b72806cf..0868216a15 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -87,3 +87,19 @@ module.exports = fs.exists "#{location}/#{filteredName}", (exists) -> logger.log location:location, name:filteredName, exists:exists, "checked if file exists" callback null, exists + + directorySize:(location, name, callback)-> + filteredName = filterName name.replace(/\/$/,'') + logger.log location:location, name:filteredName, "get project size in file system" + fs.readdir "#{location}/#{filteredName}", (err, files) -> + if err? + logger.err err:err, location:location, name:filteredName, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each files, (entry)-> + fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r' + fileStats = fs.fstatSync(fd) + totalSize += fileStats.size + fs.closeSync fd + logger.log totalSize:totalSize, "total size", files:files + callback null, totalSize diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee index fed469e700..e1c212711a 100644 --- a/services/filestore/app/coffee/FileController.coffee +++ b/services/filestore/app/coffee/FileController.coffee @@ -83,3 +83,14 @@ module.exports = FileController = else range = parsed[0] {start: range.start, end: range.end} + + directorySize: (req, res)-> + metrics.inc "projectSize" + {project_id, bucket} = req + logger.log project_id:project_id, bucket:bucket, "reciving request to project size" + FileHandler.getDirectorySize bucket, project_id, (err, size)-> + if err? + logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" + res.send 500 + else + res.json {'total bytes' : size} diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index f22285edae..718195a484 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -87,3 +87,10 @@ module.exports = if err? return callback(err) LocalFileWriter.writeStream fileStream, key, callback + + getDirectorySize: (bucket, project_id, callback)-> + logger.log bucket:bucket, project_id:project_id, "getting project size" + PersistorManager.directorySize bucket, project_id, (err, size)-> + if err? + logger.err bucket:bucket, project_id:project_id, "error getting size" + callback err, size diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 0b5e41525f..18f1d96952 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -42,4 +42,9 @@ module.exports = opts = req.query next() + publicProjectKey: (req, res, next)-> + {project_id} = req.params + req.project_id = project_id + req.bucket = settings.filestore.stores.user_files + next() diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 15798185a3..b1a03fb4f4 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -137,3 +137,19 @@ module.exports = exists = res.statusCode == 200 logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3" callback(err, exists) + + directorySize:(bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "get project size in s3" + s3Client = knox.createClient + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: bucketName + s3Client.list prefix:key, (err, data)-> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + logger.log totalSize:totalSize, "total size" + callback null, totalSize diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee index 0ca8c65ffc..92fb968914 100644 --- a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee @@ -247,3 +247,26 @@ describe "AWSSDKPersistorManager", -> @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => expect(err).to.equal @error done() + + describe "directorySize", -> + + it "should list the directory content using s3.listObjects", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: [] + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => + expect(err).to.not.be.ok + expect(@s3.listObjects.calledOnce, "called only once").to.be.true + expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key), + "called with correct arguments").to.be.true + done() + + it "should dispatch the error from s3.listObjects", (done) -> + @s3.listObjects.callsArgWith 1, @error + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => + expect(err).to.equal @error + done() + + it "should sum directory files sizes", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: [ { Size: 1024 }, { Size: 2048 }] + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err, size) => + expect(size).to.equal 3072 + done() diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 75a4376b8c..63fe9eded7 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -18,6 +18,10 @@ describe "FSPersistorManagerTests", -> unlink:sinon.stub() rmdir:sinon.stub() exists:sinon.stub() + readdir:sinon.stub() + openSync:sinon.stub() + fstatSync:sinon.stub() + closeSync:sinon.stub() @Rimraf = sinon.stub() @LocalFileWriter = writeStream: sinon.stub() @@ -208,3 +212,18 @@ describe "FSPersistorManagerTests", -> @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => exists.should.be.false done() + + describe "directorySize", -> + + it "should propogate the error", (done) -> + @Fs.readdir.callsArgWith(1, @error) + @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => + err.should.equal @error + done() + + it "should sum directory files size", (done) -> + @Fs.readdir.callsArgWith(1, null, [ {'file1'}, {'file2'} ]) + @Fs.fstatSync.returns({size : 1024}) + @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => + expect(totalsize).to.equal 2048 + done() \ No newline at end of file diff --git a/services/filestore/test/unit/coffee/FileControllerTests.coffee b/services/filestore/test/unit/coffee/FileControllerTests.coffee index 1a2c3e81ea..591644de60 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.coffee +++ b/services/filestore/test/unit/coffee/FileControllerTests.coffee @@ -22,6 +22,7 @@ describe "FileController", -> getFile: sinon.stub() deleteFile: sinon.stub() insertFile: sinon.stub() + getDirectorySize: sinon.stub() @LocalFileWriter = {} @controller = SandboxedModule.require modulePath, requires: "./LocalFileWriter":@LocalFileWriter @@ -152,3 +153,18 @@ describe "FileController", -> result = @controller._get_range('carrots=0-200') expect(result).to.equal null done() + + describe "directorySize", -> + + it "should return total directory size bytes", (done) -> + @FileHandler.getDirectorySize.callsArgWith(2, null, 1024) + @controller.directorySize @req, json:(result)=> + expect(result['total bytes']).to.equal 1024 + done() + + it "should send a 500 if there was an error", (done)-> + @FileHandler.getDirectorySize.callsArgWith(2, "error") + @res.send = (code)-> + code.should.equal 500 + done() + @controller.directorySize @req, @res diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee index e67a8d38b7..ab757b9360 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee +++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee @@ -20,6 +20,7 @@ describe "FileHandler", -> deleteDirectory: sinon.stub() sendStream: sinon.stub() insertFile: sinon.stub() + directorySize: sinon.stub() @LocalFileWriter = writeStream: sinon.stub() deleteFile: sinon.stub() @@ -192,3 +193,13 @@ describe "FileHandler", -> @FileConverter.convert.calledWith(@stubbedPath, @format).should.equal true @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true done() + + describe "getDirectorySize", -> + + beforeEach -> + @PersistorManager.directorySize.callsArgWith(2) + + it "should call the filestore manager to get directory size", (done)-> + @handler.getDirectorySize @bucket, @key, => + @PersistorManager.directorySize.calledWith(@bucket, @key).should.equal true + done() diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 78fec6cea5..3a3e7b0d86 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -256,3 +256,16 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> err.should.equal @error done() + + describe "directorySize", -> + + beforeEach -> + @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + + it "should sum directory files size", (done) -> + data = + Contents: [ {Size: 1024}, {Size: 2048} ] + @stubbedKnoxClient.list.callsArgWith(1, null, data) + @S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=> + totalSize.should.equal 3072 + done()