From 038b3476fe4f1a7253941e4512087ffd78620f4a Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Mon, 22 Oct 2018 14:09:56 +0100 Subject: [PATCH 01/13] Add some details to the README --- services/filestore/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/services/filestore/README.md b/services/filestore/README.md index 420214b56f..cd126ab053 100644 --- a/services/filestore/README.md +++ b/services/filestore/README.md @@ -5,6 +5,18 @@ An API for CRUD operations on binary files stored in S3 [![Build Status](https://travis-ci.org/sharelatex/filestore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/filestore-sharelatex) +filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/sharelatex/filestore-sharelatex/blob/master/app.coffee . URLs are of the form: + +* `/project/:project_id/file/:file_id` +* `/template/:template_id/v/:version/:format` +* `/project/:project_id/public/:public_file_id` +* `/project/:project_id/size` +* `/bucket/:bucket/key/*` +* `/heapdump` +* `/shutdown` +* `/status` - returns `filestore sharelatex up` or `server is being shut down` (HTTP 500) +* `/health_check` + License ------- From 06bfd4c270e8c01a79188612f5a02d7fcfdd7be2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Nov 2018 16:07:55 +0000 Subject: [PATCH 02/13] add dockerfile to install packages needed by filestore --- services/filestore/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 services/filestore/Dockerfile diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile new file mode 100644 index 0000000000..5f4f63aec0 --- /dev/null +++ b/services/filestore/Dockerfile @@ -0,0 +1,5 @@ +FROM node:6.9.5 + +RUN apt-get update +# we also need imagemagick but it is already in the node docker image +RUN apt-get install -y --no-install-recommends ghostscript optipng From 40ac59090bfba92d95ec55e03f983b3c6bd39514 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Nov 2018 16:22:09 +0000 Subject: [PATCH 03/13] make backends consistent by deleting temporary files --- .../app/coffee/FSPersistorManager.coffee | 20 +++++++++++++++---- .../app/coffee/S3PersistorManager.coffee | 10 ++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 0868216a15..733202e4cd 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -13,10 +13,19 @@ module.exports = sendFile: ( location, target, source, callback = (err)->) -> filteredTarget = filterName target logger.log location:location, target:filteredTarget, source:source, "sending file" - fs.rename source, "#{location}/#{filteredTarget}", (err) -> - if err!=null + done = _.once (err) -> + if err? logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file" - callback err + callback(err) + # actually copy the file (instead of moving it) to maintain consistent behaviour + # between the different implementations + sourceStream = fs.createReadStream source + sourceStream.on 'error', done + targetStream = fs.createWriteStream "#{location}/#{filteredTarget}" + targetStream.on 'error', done + targetStream.on 'finish', () -> + done() + sourceStream.pipe targetStream sendStream: ( location, target, sourceStream, callback = (err)->) -> logger.log location:location, target:target, "sending file stream" @@ -26,7 +35,10 @@ module.exports = if err? logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" return callback err - @sendFile location, target, fsPath, callback + @sendFile location, target, fsPath, (err) -> + # delete the temporary file created above and return the original error + LocalFileWriter.deleteFile fsPath, () -> + callback(err) # opts may be {start: Number, end: Number} getFileStream: (location, name, opts, _callback = (err, res)->) -> diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 2bd6eb0e9b..3055810131 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -42,9 +42,8 @@ module.exports = if res.statusCode != 200 logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file" return callback("non 200 response from s3 on put file") - LocalFileWriter.deleteFile fsPath, (err)-> - logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3" - callback(err) + logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3" + callback(err) putEventEmiter.on "error", (err)-> logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file" callback err @@ -57,7 +56,10 @@ module.exports = if err? logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk" return callback(err) - @sendFile bucketName, key, fsPath, callback + @sendFile bucketName, key, fsPath, (err) -> + # delete the temporary file created above and return the original error + LocalFileWriter.deleteFile fsPath, () -> + callback(err) # opts may be {start: Number, end: Number} getFileStream: (bucketName, key, opts, callback = (err, res)->)-> From 78ccf453dbb308ab2ad636c1e9e4582415b01460 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Nov 2018 16:51:06 +0000 Subject: [PATCH 04/13] update unit tests --- .../coffee/FSPersistorManagerTests.coffee | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 46def22a05..f762b7a89f 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -25,6 +25,7 @@ describe "FSPersistorManagerTests", -> @Rimraf = sinon.stub() @LocalFileWriter = writeStream: sinon.stub() + deleteFile: sinon.stub() @requires = "./LocalFileWriter":@LocalFileWriter "fs":@Fs @@ -43,10 +44,32 @@ describe "FSPersistorManagerTests", -> @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires describe "sendFile", -> - it "should put the file", (done) -> - @Fs.rename.callsArgWith(2,@error) + beforeEach -> + @Fs.createReadStream = sinon.stub().returns({ + on: -> + pipe: -> + }) + + it "should copy the file", (done) -> + @Fs.createWriteStream =sinon.stub().returns({ + on: (event, handler) -> + process.nextTick(handler) if event is 'finish' + }) @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> - @Fs.rename.calledWith( @name2, "#{@location}/#{@name1Filtered}" ).should.equal true + @Fs.createReadStream.calledWith(@name2).should.equal true + @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true + done() + + it "should return an error if the file cannot be stored", (done) -> + @Fs.createWriteStream =sinon.stub().returns({ + on: (event, handler) => + if event is 'error' + process.nextTick () => + handler(@error) + }) + @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> + @Fs.createReadStream.calledWith(@name2).should.equal true + @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true err.should.equal @error done() @@ -54,6 +77,7 @@ describe "FSPersistorManagerTests", -> beforeEach -> @FSPersistorManager.sendFile = sinon.stub().callsArgWith(3) @LocalFileWriter.writeStream.callsArgWith(2, null, @name1) + @LocalFileWriter.deleteFile.callsArg(1) @SourceStream = on:-> From 8744b3aa4eefc2769be624e4d92029bb9080c0b4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 7 Nov 2018 16:23:55 +0000 Subject: [PATCH 05/13] serve file from disk to avoid read-after-write inconsistency --- .../filestore/app/coffee/FileHandler.coffee | 19 ++++++++++++++++++- .../app/coffee/LocalFileWriter.coffee | 17 +++++++++++++++++ .../test/unit/coffee/FileHandlerTests.coffee | 10 +++++++--- .../unit/coffee/LocalFileWriterTests.coffee | 15 +++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index 2eeb09bc74..94c6a7395c 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -64,7 +64,24 @@ module.exports = FileHandler = LocalFileWriter.deleteFile convertedFsPath, -> LocalFileWriter.deleteFile originalFsPath, -> return callback(err) - PersistorManager.getFileStream bucket, convertedKey, opts, callback + # Send back the converted file from the local copy to avoid problems + # with the file not being present in S3 yet. As described in the + # documentation below, we have already made a 'HEAD' request in + # checkIfFileExists so we only have "eventual consistency" if we try + # to stream it from S3 here. This was a cause of many 403 errors. + # + # "Amazon S3 provides read-after-write consistency for PUTS of new + # objects in your S3 bucket in all regions with one caveat. The + # caveat is that if you make a HEAD or GET request to the key name + # (to find if the object exists) before creating the object, Amazon + # S3 provides eventual consistency for read-after-write."" + # https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel + LocalFileWriter.getStream convertedFsPath, (err, readStream) -> + return callback(err) if err? + readStream.on 'end', () -> + logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file" + LocalFileWriter.deleteFile convertedFsPath, -> + callback(null, readStream) _convertFile: (bucket, originalKey, opts, callback)-> @_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)-> diff --git a/services/filestore/app/coffee/LocalFileWriter.coffee b/services/filestore/app/coffee/LocalFileWriter.coffee index 47b2b91e77..72422b7696 100644 --- a/services/filestore/app/coffee/LocalFileWriter.coffee +++ b/services/filestore/app/coffee/LocalFileWriter.coffee @@ -5,6 +5,7 @@ _ = require("underscore") logger = require("logger-sharelatex") metrics = require("metrics-sharelatex") Settings = require("settings-sharelatex") +Errors = require "./Errors" module.exports = @@ -26,6 +27,22 @@ module.exports = callback err stream.pipe writeStream + getStream: (fsPath, _callback = (err, res)->) -> + callback = _.once _callback + timer = new metrics.Timer("readingFile") + logger.log fsPath:fsPath, "reading file locally" + readStream = fs.createReadStream(fsPath) + readStream.on "end", -> + timer.done() + logger.log fsPath:fsPath, "finished reading file locally" + readStream.on "error", (err)-> + logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream" + if err.code == 'ENOENT' + callback new Errors.NotFoundError(err.message), null + else + callback err + callback null, readStream + deleteFile: (fsPath, callback)-> if !fsPath? or fsPath == "" return callback() diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee index ab757b9360..50b8a17524 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee +++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee @@ -23,6 +23,7 @@ describe "FileHandler", -> directorySize: sinon.stub() @LocalFileWriter = writeStream: sinon.stub() + getStream: sinon.stub() deleteFile: sinon.stub() @FileConverter = convert: sinon.stub() @@ -152,17 +153,20 @@ describe "FileHandler", -> it "should _convertFile ", (done)-> @stubbedStream = {"something":"here"} + @localStream = { + on: -> + } @PersistorManager.sendFile = sinon.stub().callsArgWith(3) - @PersistorManager.getFileStream = sinon.stub().callsArgWith(3, null, @stubbedStream) + @LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream) @convetedKey = @key+"converted" @handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath) @ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) @handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=> @handler._convertFile.called.should.equal true @PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true - @PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true @ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true - fsStream.should.equal @stubbedStream + @LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true + fsStream.should.equal @localStream done() describe "_convertFile", -> diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee b/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee index 0b9eec035e..a6bc964e0f 100644 --- a/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee +++ b/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee @@ -15,8 +15,11 @@ describe "LocalFileWriter", -> on: (type, cb)-> if type == "finish" cb() + @readStream = + on: -> @fs = createWriteStream : sinon.stub().returns(@writeStream) + createReadStream: sinon.stub().returns(@readStream) unlink: sinon.stub() @settings = path: @@ -51,6 +54,18 @@ describe "LocalFileWriter", -> fsPath.should.equal @stubbedFsPath done() + describe "getStream", -> + + it "should read the stream from the file ", (done)-> + @writer.getStream @stubbedFsPath, (err, stream)=> + @fs.createReadStream.calledWith(@stubbedFsPath).should.equal true + done() + + it "should send the stream in the callback", (done)-> + @writer.getStream @stubbedFsPath, (err, readStream)=> + readStream.should.equal @readStream + done() + describe "delete file", -> it "should unlink the file", (done)-> From c2bfdd095ca82e73f637c81d2f2f60110b014eac Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 8 Nov 2018 16:29:23 +0000 Subject: [PATCH 06/13] upgrade logger-sharelatex to v1.5.8 --- services/filestore/npm-shrinkwrap.json | 10 +++++----- services/filestore/package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 34fdb6b539..dcbdd2848d 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1094,9 +1094,9 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz" }, "logger-sharelatex": { - "version": "1.5.7", - "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7", - "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#13562f8866708fc86aef8202bf5a2ce4d1c6eed7", + "version": "1.5.8", + "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", + "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#3f841b014572706e472c47fe0d0c0c1e569bad8c", "dependencies": { "ansi-styles": { "version": "1.1.0", @@ -1238,9 +1238,9 @@ } }, "nan": { - "version": "2.11.0", + "version": "2.11.1", "from": "nan@>=2.0.8 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.0.tgz", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz", "optional": true }, "ncp": { diff --git a/services/filestore/package.json b/services/filestore/package.json index 0910f4c4db..80d4a06240 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -18,7 +18,7 @@ "fs-extra": "^1.0.0", "heapdump": "^0.3.2", "knox": "~0.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", "node-transloadit": "0.0.4", "node-uuid": "~1.4.1", From 69b164092cdccb469052464ea1f3c3697a06d54d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 9 Nov 2018 14:04:26 +0000 Subject: [PATCH 07/13] suppress unnecessary error logging for NotFound --- services/filestore/app/coffee/FileController.coffee | 2 +- services/filestore/app/coffee/FileHandler.coffee | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee index 27ac078379..24fd5229de 100644 --- a/services/filestore/app/coffee/FileController.coffee +++ b/services/filestore/app/coffee/FileController.coffee @@ -29,10 +29,10 @@ module.exports = FileController = logger.log start: range.start, end: range.end, "getting range of bytes from file" FileHandler.getFile bucket, key, options, (err, fileStream)-> if err? - logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file" if err instanceof Errors.NotFoundError return res.send 404 else + logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file" return res.send 500 else if req.query.cacheWarm logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream" diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee index 2eeb09bc74..8fc7cd037a 100644 --- a/services/filestore/app/coffee/FileHandler.coffee +++ b/services/filestore/app/coffee/FileHandler.coffee @@ -6,6 +6,7 @@ FileConverter = require("./FileConverter") KeyBuilder = require("./KeyBuilder") async = require("async") ImageOptimiser = require("./ImageOptimiser") +Errors = require('./Errors') module.exports = FileHandler = @@ -32,7 +33,7 @@ module.exports = FileHandler = _getStandardFile: (bucket, key, opts, callback)-> PersistorManager.getFileStream bucket, key, opts, (err, fileStream)-> - if err? + if err? and !(err instanceof Errors.NotFoundError) logger.err bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream" callback err, fileStream From 23a6d6e81d3cfacc8cb1707b00527368bdc4dbe7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 9 Nov 2018 14:05:07 +0000 Subject: [PATCH 08/13] consider 403 and 404 as NotFound errors --- services/filestore/app/coffee/S3PersistorManager.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index 2bd6eb0e9b..0bb713deb0 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -74,7 +74,9 @@ module.exports = s3Stream = s3Client.get(key, headers) s3Stream.end() s3Stream.on 'response', (res) -> - if res.statusCode == 404 + if res.statusCode in [403, 404] + # S3 returns a 403 instead of a 404 when the user doesn't have + # permission to list the bucket contents. logger.log bucketName:bucketName, key:key, "file not found in s3" return callback new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null if res.statusCode not in [200, 206] From 28271655e18401e471bdf8be61c0464005ffa570 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 6 Dec 2018 08:19:39 +0000 Subject: [PATCH 09/13] avoid problems with caching of apt-get operations --- services/filestore/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile index 5f4f63aec0..49a9940d7f 100644 --- a/services/filestore/Dockerfile +++ b/services/filestore/Dockerfile @@ -1,5 +1,4 @@ FROM node:6.9.5 -RUN apt-get update # we also need imagemagick but it is already in the node docker image -RUN apt-get install -y --no-install-recommends ghostscript optipng +RUN apt-get update && apt-get install -y --no-install-recommends ghostscript optipng From 621425656823df768603751bb0781ef367ea5b49 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Wed, 2 Jan 2019 09:47:38 +0000 Subject: [PATCH 10/13] Bump logger to v1.5.8 --- services/filestore/npm-shrinkwrap.json | 11 ++++++++--- services/filestore/package.json | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 14a32672a7..09537db297 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1142,9 +1142,9 @@ "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz" }, "logger-sharelatex": { - "version": "1.5.7", - "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7", - "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#13562f8866708fc86aef8202bf5a2ce4d1c6eed7", + "version": "1.5.8", + "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", + "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#3f841b014572706e472c47fe0d0c0c1e569bad8c", "dependencies": { "coffee-script": { "version": "1.12.4", @@ -1202,6 +1202,11 @@ "version": "1.6.0", "from": "coffee-script@1.6.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + }, + "underscore": { + "version": "1.6.0", + "from": "underscore@>=1.6.0 <1.7.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" } } }, diff --git a/services/filestore/package.json b/services/filestore/package.json index fab7033316..e09395c41f 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -30,7 +30,7 @@ "fs-extra": "^1.0.0", "heapdump": "^0.3.2", "knox": "~0.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7", "mocha": "5.2.0", "node-transloadit": "0.0.4", From 3b6eba33baf04238d17312edfedded5e46f3ea1a Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Wed, 2 Jan 2019 09:53:53 +0000 Subject: [PATCH 11/13] Remove `Metrics.inc "startup"` from app.coffee --- services/filestore/app.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index b172e3ba88..549cd19bfe 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -23,8 +23,6 @@ Metrics.memory.monitor(logger) app.use Metrics.http.monitor(logger) -Metrics.inc "startup" - app.use (req, res, next)-> Metrics.inc "http-request" next() From 594b403bd36f16f1ec9098562b735188f3bdf13f Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Wed, 2 Jan 2019 09:55:31 +0000 Subject: [PATCH 12/13] Move metrics init to start of app.coffee --- services/filestore/app.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 549cd19bfe..c1484142b1 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -1,3 +1,5 @@ +Metrics = require "metrics-sharelatex" +Metrics.initialize("filestore") express = require('express') bodyParser = require "body-parser" logger = require('logger-sharelatex') @@ -15,8 +17,6 @@ app = express() if settings.sentry?.dsn? logger.initializeErrorReporting(settings.sentry.dsn) -Metrics = require "metrics-sharelatex" -Metrics.initialize("filestore") Metrics.open_sockets.monitor(logger) Metrics.event_loop?.monitor(logger) Metrics.memory.monitor(logger) From 676380974871ca72b587c083db1e948f687dab8b Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 3 Jan 2019 16:27:45 +0000 Subject: [PATCH 13/13] Bump logger to v1.5.9 --- services/filestore/npm-shrinkwrap.json | 6 +++--- services/filestore/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 3076829a85..43599f526b 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1155,9 +1155,9 @@ "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" }, "logger-sharelatex": { - "version": "1.5.8", - "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", - "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#3f841b014572706e472c47fe0d0c0c1e569bad8c", + "version": "1.5.9", + "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", + "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#e8e1b95052f62e107336053e4a983f81cdbdf589", "dependencies": { "coffee-script": { "version": "1.12.4", diff --git a/services/filestore/package.json b/services/filestore/package.json index e6be998bef..4f25056dee 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -30,7 +30,7 @@ "fs-extra": "^1.0.0", "heapdump": "^0.3.2", "knox": "~0.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", "mocha": "5.2.0", "node-transloadit": "0.0.4",