From 099d0fef2f7337a0dfcc32fe0aaea86dd25dffd3 Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Fri, 11 Mar 2016 22:31:46 -0300 Subject: [PATCH 1/5] scaffolding project size --- services/filestore/app.coffee | 2 ++ .../filestore/app/coffee/KeyBuilder.coffee | 5 +++++ .../app/coffee/ProjectController.coffee | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 services/filestore/app/coffee/ProjectController.coffee diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index e78355f4e6..93e86164a7 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -85,6 +85,8 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile +app.get "project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize + app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> res.send filename diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 0b5e41525f..18f1d96952 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -42,4 +42,9 @@ module.exports = opts = req.query next() + publicProjectKey: (req, res, next)-> + {project_id} = req.params + req.project_id = project_id + req.bucket = settings.filestore.stores.user_files + next() diff --git a/services/filestore/app/coffee/ProjectController.coffee b/services/filestore/app/coffee/ProjectController.coffee new file mode 100644 index 0000000000..5ac64dd632 --- /dev/null +++ b/services/filestore/app/coffee/ProjectController.coffee @@ -0,0 +1,18 @@ +settings = require("settings-sharelatex") +logger = require("logger-sharelatex") +metrics = require("metrics-sharelatex") +Errors = require('./Errors') + + +module.exports = projectController = + + projectSize: (req, res)-> + metrics.inc "projectSize" + {project_id, bucket} = req + logger.log project_id:project_id, bucket:bucket, "reciving request to project size" + FileHandler.getFile bucket, project_id, req, (err)-> + if err? + logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" + res.send 500 + else + res.send 200 \ No newline at end of file From 5d2fc89137e9df4bad3e6d941a943ba9bcb42709 Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Sat, 12 Mar 2016 04:35:49 -0300 Subject: [PATCH 2/5] scaffolding project methods --- .../app/coffee/AWSSDKPersistorManager.coffee | 4 ++++ .../filestore/app/coffee/FSPersistorManager.coffee | 4 ++++ .../filestore/app/coffee/ProjectController.coffee | 4 ++-- services/filestore/app/coffee/ProjectHandler.coffee | 13 +++++++++++++ .../filestore/app/coffee/S3PersistorManager.coffee | 4 ++++ 5 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 services/filestore/app/coffee/ProjectHandler.coffee diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index d0101b93a4..0c5cb959f9 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -85,3 +85,7 @@ module.exports = return callback err callback null, data.ETag? + getProjectSize:(bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "get project size in s3" + callback null, 1024 + diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index d1b72806cf..17f11217c1 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -87,3 +87,7 @@ module.exports = fs.exists "#{location}/#{filteredName}", (exists) -> logger.log location:location, name:filteredName, exists:exists, "checked if file exists" callback null, exists + + getProjectSize:(location, name, callback)-> + logger.log location:location, name:name, "get project size in file system" + callback null, 1024 diff --git a/services/filestore/app/coffee/ProjectController.coffee b/services/filestore/app/coffee/ProjectController.coffee index 5ac64dd632..21f30f117b 100644 --- a/services/filestore/app/coffee/ProjectController.coffee +++ b/services/filestore/app/coffee/ProjectController.coffee @@ -10,9 +10,9 @@ module.exports = projectController = metrics.inc "projectSize" {project_id, bucket} = req logger.log project_id:project_id, bucket:bucket, "reciving request to project size" - FileHandler.getFile bucket, project_id, req, (err)-> + ProjectHandler.getSize bucket, project_id, req, (err, size)-> if err? logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" res.send 500 else - res.send 200 \ No newline at end of file + res.json {'total bytes' : size} \ No newline at end of file diff --git a/services/filestore/app/coffee/ProjectHandler.coffee b/services/filestore/app/coffee/ProjectHandler.coffee new file mode 100644 index 0000000000..e6022c4aa3 --- /dev/null +++ b/services/filestore/app/coffee/ProjectHandler.coffee @@ -0,0 +1,13 @@ +settings = require("settings-sharelatex") +PersistorManager = require("./PersistorManager") +logger = require("logger-sharelatex") +async = require("async") + +module.exports = + + getSize: (bucket, project_id, opts = {}, callback)-> + logger.log bucket:bucket, project_id:project_id, opts:opts, "getting project size" + PersistorManager.getProjectSize bucket, project_id, opts, (err, size)-> + if err? + logger.err bucket:bucket, project_id:project_id, opts:opts, "error getting size" + callback err, size \ No newline at end of file diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 15798185a3..b6380eceb8 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -137,3 +137,7 @@ module.exports = exists = res.statusCode == 200 logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3" callback(err, exists) + + getProjectSize:(bucketName, key, callback)-> + logger.log bucketName:bucketName, key:key, "get project size in s3" + callback null, 1024 From 1f20744a68d2ca80f204721310affdbd5d4c5999 Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Sat, 12 Mar 2016 11:08:07 -0300 Subject: [PATCH 3/5] add size function for each persitor --- services/filestore/app.coffee | 3 ++- .../app/coffee/AWSSDKPersistorManager.coffee | 12 +++++++++++- .../app/coffee/FSPersistorManager.coffee | 16 ++++++++++++++-- .../app/coffee/ProjectController.coffee | 4 ++-- .../filestore/app/coffee/ProjectHandler.coffee | 8 ++++---- .../app/coffee/S3PersistorManager.coffee | 14 +++++++++++++- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 93e86164a7..ad05bd5124 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -4,6 +4,7 @@ logger.initialize("filestore") settings = require("settings-sharelatex") request = require("request") fileController = require("./app/js/FileController") +projectController = require("./app/js/ProjectController") keyBuilder = require("./app/js/KeyBuilder") domain = require("domain") appIsOk = true @@ -85,7 +86,7 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile -app.get "project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize +app.get "/project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index 0c5cb959f9..ca72901f8f 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -87,5 +87,15 @@ module.exports = getProjectSize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" - callback null, 1024 + s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" + return callback err + if data.Contents.length == 0 + logger.log bucketName:bucketName, key:key, "the directory is empty" + return callback() + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + callback null, totalSize diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 17f11217c1..c146b162ff 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -89,5 +89,17 @@ module.exports = callback null, exists getProjectSize:(location, name, callback)-> - logger.log location:location, name:name, "get project size in file system" - callback null, 1024 + filteredName = filterName name.replace(/\/$/,'') + logger.log location:location, name:filteredName, "get project size in file system" + fs.readdir "#{location}/#{filteredName}", (err, files) -> + if err? + logger.err err:err, location:location, name:filteredName, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each files, (entry)-> + fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r' + fileStats = fs.fstatSync(fd) + totalSize += fileStats.size + fs.closeSync fd + logger.log totalSize:totalSize, "total size", files:files + callback null, totalSize diff --git a/services/filestore/app/coffee/ProjectController.coffee b/services/filestore/app/coffee/ProjectController.coffee index 21f30f117b..e31d2539b5 100644 --- a/services/filestore/app/coffee/ProjectController.coffee +++ b/services/filestore/app/coffee/ProjectController.coffee @@ -2,7 +2,7 @@ settings = require("settings-sharelatex") logger = require("logger-sharelatex") metrics = require("metrics-sharelatex") Errors = require('./Errors') - +ProjectHandler = require("./ProjectHandler") module.exports = projectController = @@ -10,7 +10,7 @@ module.exports = projectController = metrics.inc "projectSize" {project_id, bucket} = req logger.log project_id:project_id, bucket:bucket, "reciving request to project size" - ProjectHandler.getSize bucket, project_id, req, (err, size)-> + ProjectHandler.getSize bucket, project_id, (err, size)-> if err? logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" res.send 500 diff --git a/services/filestore/app/coffee/ProjectHandler.coffee b/services/filestore/app/coffee/ProjectHandler.coffee index e6022c4aa3..53ed2df183 100644 --- a/services/filestore/app/coffee/ProjectHandler.coffee +++ b/services/filestore/app/coffee/ProjectHandler.coffee @@ -5,9 +5,9 @@ async = require("async") module.exports = - getSize: (bucket, project_id, opts = {}, callback)-> - logger.log bucket:bucket, project_id:project_id, opts:opts, "getting project size" - PersistorManager.getProjectSize bucket, project_id, opts, (err, size)-> + getSize: (bucket, project_id, callback)-> + logger.log bucket:bucket, project_id:project_id, "getting project size" + PersistorManager.getProjectSize bucket, project_id, (err, size)-> if err? - logger.err bucket:bucket, project_id:project_id, opts:opts, "error getting size" + logger.err bucket:bucket, project_id:project_id, "error getting size" callback err, size \ No newline at end of file diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index b6380eceb8..4accd517cb 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -140,4 +140,16 @@ module.exports = getProjectSize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" - callback null, 1024 + s3Client = knox.createClient + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: bucketName + s3Client.list prefix:key, (err, data)-> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + logger.log totalSize:totalSize, "total size" + callback null, totalSize From 0bb12f32fa626fa61f2731b8221e2a87f4d22572 Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Sun, 13 Mar 2016 16:22:14 -0300 Subject: [PATCH 4/5] remove project controller / handler add unit test scaffolding --- services/filestore/app.coffee | 3 +-- .../app/coffee/AWSSDKPersistorManager.coffee | 2 +- .../app/coffee/FSPersistorManager.coffee | 2 +- .../filestore/app/coffee/FileController.coffee | 11 +++++++++++ .../filestore/app/coffee/FileHandler.coffee | 7 +++++++ .../app/coffee/ProjectController.coffee | 18 ------------------ .../filestore/app/coffee/ProjectHandler.coffee | 13 ------------- .../app/coffee/S3PersistorManager.coffee | 2 +- .../coffee/AWSSDKPersistorManagerTests.coffee | 8 ++++++++ .../unit/coffee/FSPersistorManagerTests.coffee | 5 +++++ .../unit/coffee/FileControllerTests.coffee | 5 +++++ .../test/unit/coffee/FileHandlerTests.coffee | 5 +++++ .../unit/coffee/S3PersistorManagerTests.coffee | 5 +++++ 13 files changed, 50 insertions(+), 36 deletions(-) delete mode 100644 services/filestore/app/coffee/ProjectController.coffee delete mode 100644 services/filestore/app/coffee/ProjectHandler.coffee diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index ad05bd5124..04c13eb4b5 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -4,7 +4,6 @@ logger.initialize("filestore") settings = require("settings-sharelatex") request = require("request") fileController = require("./app/js/FileController") -projectController = require("./app/js/ProjectController") keyBuilder = require("./app/js/KeyBuilder") domain = require("domain") appIsOk = true @@ -86,7 +85,7 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile -app.get "/project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize +app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index ca72901f8f..5be80506f5 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -85,7 +85,7 @@ module.exports = return callback err callback null, data.ETag? - getProjectSize:(bucketName, key, callback)-> + directorySize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> if err? diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index c146b162ff..0868216a15 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -88,7 +88,7 @@ module.exports = logger.log location:location, name:filteredName, exists:exists, "checked if file exists" callback null, exists - getProjectSize:(location, name, callback)-> + directorySize:(location, name, callback)-> filteredName = filterName name.replace(/\/$/,'') logger.log location:location, name:filteredName, "get project size in file system" fs.readdir "#{location}/#{filteredName}", (err, files) -> diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee index fed469e700..e1c212711a 100644 --- a/services/filestore/app/coffee/FileController.coffee +++ b/services/filestore/app/coffee/FileController.coffee @@ -83,3 +83,14 @@ module.exports = FileController = else range = parsed[0] {start: range.start, end: range.end} + + directorySize: (req, res)-> + metrics.inc "projectSize" + {project_id, bucket} = req + logger.log project_id:project_id, bucket:bucket, "reciving request to project size" + FileHandler.getDirectorySize bucket, project_id, (err, size)-> + if err? + logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" + res.send 500 + else + res.json {'total bytes' : size} diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index f22285edae..718195a484 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -87,3 +87,10 @@ module.exports = if err? return callback(err) LocalFileWriter.writeStream fileStream, key, callback + + getDirectorySize: (bucket, project_id, callback)-> + logger.log bucket:bucket, project_id:project_id, "getting project size" + PersistorManager.directorySize bucket, project_id, (err, size)-> + if err? + logger.err bucket:bucket, project_id:project_id, "error getting size" + callback err, size diff --git a/services/filestore/app/coffee/ProjectController.coffee b/services/filestore/app/coffee/ProjectController.coffee deleted file mode 100644 index e31d2539b5..0000000000 --- a/services/filestore/app/coffee/ProjectController.coffee +++ /dev/null @@ -1,18 +0,0 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -metrics = require("metrics-sharelatex") -Errors = require('./Errors') -ProjectHandler = require("./ProjectHandler") - -module.exports = projectController = - - projectSize: (req, res)-> - metrics.inc "projectSize" - {project_id, bucket} = req - logger.log project_id:project_id, bucket:bucket, "reciving request to project size" - ProjectHandler.getSize bucket, project_id, (err, size)-> - if err? - logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" - res.send 500 - else - res.json {'total bytes' : size} \ No newline at end of file diff --git a/services/filestore/app/coffee/ProjectHandler.coffee b/services/filestore/app/coffee/ProjectHandler.coffee deleted file mode 100644 index 53ed2df183..0000000000 --- a/services/filestore/app/coffee/ProjectHandler.coffee +++ /dev/null @@ -1,13 +0,0 @@ -settings = require("settings-sharelatex") -PersistorManager = require("./PersistorManager") -logger = require("logger-sharelatex") -async = require("async") - -module.exports = - - getSize: (bucket, project_id, callback)-> - logger.log bucket:bucket, project_id:project_id, "getting project size" - PersistorManager.getProjectSize bucket, project_id, (err, size)-> - if err? - logger.err bucket:bucket, project_id:project_id, "error getting size" - callback err, size \ No newline at end of file diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 4accd517cb..b1a03fb4f4 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -138,7 +138,7 @@ module.exports = logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3" callback(err, exists) - getProjectSize:(bucketName, key, callback)-> + directorySize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" s3Client = knox.createClient key: settings.filestore.s3.key diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee index 0ca8c65ffc..262f7aa94c 100644 --- a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee @@ -247,3 +247,11 @@ describe "AWSSDKPersistorManager", -> @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => expect(err).to.equal @error done() + + describe "directorySize", -> + + it "should list the directory content using s3.listObjects", (done) -> + done() + + it "should sum directory files size", (done) -> + done() diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 75a4376b8c..1d7eb17e88 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -208,3 +208,8 @@ describe "FSPersistorManagerTests", -> @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => exists.should.be.false done() + + describe "directorySize", -> + + it "should sum directory files size", (done) -> + done() \ No newline at end of file diff --git a/services/filestore/test/unit/coffee/FileControllerTests.coffee b/services/filestore/test/unit/coffee/FileControllerTests.coffee index 1a2c3e81ea..94d50d67c1 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.coffee +++ b/services/filestore/test/unit/coffee/FileControllerTests.coffee @@ -152,3 +152,8 @@ describe "FileController", -> result = @controller._get_range('carrots=0-200') expect(result).to.equal null done() + + describe "directorySize", -> + + it "should call the file handler the directory size", (done) -> + done() diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee index e67a8d38b7..405ccc2cc1 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee +++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee @@ -192,3 +192,8 @@ describe "FileHandler", -> @FileConverter.convert.calledWith(@stubbedPath, @format).should.equal true @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true done() + + describe "directorySize", -> + + it "should call the filestore manager to get directory size", (done) -> + done() diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 78fec6cea5..e1f7b7b7a9 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -256,3 +256,8 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> err.should.equal @error done() + + describe "directorySize", -> + + it "should sum directory files size", (done) -> + done() From ffaac4c81ba360e927144756440a3e48ea0dde4a Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Sun, 13 Mar 2016 20:45:48 -0300 Subject: [PATCH 5/5] add unit tests --- .../coffee/AWSSDKPersistorManagerTests.coffee | 21 ++++++++++++++++--- .../coffee/FSPersistorManagerTests.coffee | 16 +++++++++++++- .../unit/coffee/FileControllerTests.coffee | 17 ++++++++++++--- .../test/unit/coffee/FileHandlerTests.coffee | 12 ++++++++--- .../coffee/S3PersistorManagerTests.coffee | 14 ++++++++++--- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee index 262f7aa94c..92fb968914 100644 --- a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee @@ -251,7 +251,22 @@ describe "AWSSDKPersistorManager", -> describe "directorySize", -> it "should list the directory content using s3.listObjects", (done) -> - done() + @s3.listObjects.callsArgWith 1, null, Contents: [] + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => + expect(err).to.not.be.ok + expect(@s3.listObjects.calledOnce, "called only once").to.be.true + expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key), + "called with correct arguments").to.be.true + done() - it "should sum directory files size", (done) -> - done() + it "should dispatch the error from s3.listObjects", (done) -> + @s3.listObjects.callsArgWith 1, @error + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => + expect(err).to.equal @error + done() + + it "should sum directory files sizes", (done) -> + @s3.listObjects.callsArgWith 1, null, Contents: [ { Size: 1024 }, { Size: 2048 }] + @AWSSDKPersistorManager.directorySize @bucketName, @key, (err, size) => + expect(size).to.equal 3072 + done() diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 1d7eb17e88..63fe9eded7 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -18,6 +18,10 @@ describe "FSPersistorManagerTests", -> unlink:sinon.stub() rmdir:sinon.stub() exists:sinon.stub() + readdir:sinon.stub() + openSync:sinon.stub() + fstatSync:sinon.stub() + closeSync:sinon.stub() @Rimraf = sinon.stub() @LocalFileWriter = writeStream: sinon.stub() @@ -211,5 +215,15 @@ describe "FSPersistorManagerTests", -> describe "directorySize", -> + it "should propogate the error", (done) -> + @Fs.readdir.callsArgWith(1, @error) + @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => + err.should.equal @error + done() + it "should sum directory files size", (done) -> - done() \ No newline at end of file + @Fs.readdir.callsArgWith(1, null, [ {'file1'}, {'file2'} ]) + @Fs.fstatSync.returns({size : 1024}) + @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => + expect(totalsize).to.equal 2048 + done() \ No newline at end of file diff --git a/services/filestore/test/unit/coffee/FileControllerTests.coffee b/services/filestore/test/unit/coffee/FileControllerTests.coffee index 94d50d67c1..591644de60 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.coffee +++ b/services/filestore/test/unit/coffee/FileControllerTests.coffee @@ -22,6 +22,7 @@ describe "FileController", -> getFile: sinon.stub() deleteFile: sinon.stub() insertFile: sinon.stub() + getDirectorySize: sinon.stub() @LocalFileWriter = {} @controller = SandboxedModule.require modulePath, requires: "./LocalFileWriter":@LocalFileWriter @@ -153,7 +154,17 @@ describe "FileController", -> expect(result).to.equal null done() - describe "directorySize", -> + describe "directorySize", -> - it "should call the file handler the directory size", (done) -> - done() + it "should return total directory size bytes", (done) -> + @FileHandler.getDirectorySize.callsArgWith(2, null, 1024) + @controller.directorySize @req, json:(result)=> + expect(result['total bytes']).to.equal 1024 + done() + + it "should send a 500 if there was an error", (done)-> + @FileHandler.getDirectorySize.callsArgWith(2, "error") + @res.send = (code)-> + code.should.equal 500 + done() + @controller.directorySize @req, @res diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee index 405ccc2cc1..ab757b9360 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee +++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee @@ -20,6 +20,7 @@ describe "FileHandler", -> deleteDirectory: sinon.stub() sendStream: sinon.stub() insertFile: sinon.stub() + directorySize: sinon.stub() @LocalFileWriter = writeStream: sinon.stub() deleteFile: sinon.stub() @@ -193,7 +194,12 @@ describe "FileHandler", -> @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true done() - describe "directorySize", -> + describe "getDirectorySize", -> - it "should call the filestore manager to get directory size", (done) -> - done() + beforeEach -> + @PersistorManager.directorySize.callsArgWith(2) + + it "should call the filestore manager to get directory size", (done)-> + @handler.getDirectorySize @bucket, @key, => + @PersistorManager.directorySize.calledWith(@bucket, @key).should.equal true + done() diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index e1f7b7b7a9..3a3e7b0d86 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -257,7 +257,15 @@ describe "S3PersistorManagerTests", -> err.should.equal @error done() - describe "directorySize", -> + describe "directorySize", -> - it "should sum directory files size", (done) -> - done() + beforeEach -> + @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires + + it "should sum directory files size", (done) -> + data = + Contents: [ {Size: 1024}, {Size: 2048} ] + @stubbedKnoxClient.list.callsArgWith(1, null, data) + @S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=> + totalSize.should.equal 3072 + done()