From f4af82282fdb407eb6f121d339a497c5e8351c6e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 16:06:02 +0000 Subject: [PATCH 01/32] use buildKey for prefixing name and host --- libraries/metrics/metrics.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 22dfd22548..e869ca0cff 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -20,8 +20,8 @@ require "./uv_threadpool_size" module.exports = Metrics = initialize: (_name) -> - name = _name - collectDefaultMetrics({ timeout: 5000, prefix: name + "_" }) + name = _name + "_" + collectDefaultMetrics({ timeout: 5000, prefix: name+"_"}) registerDestructor: (func) -> destructors.push func @@ -46,7 +46,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !counters[key] counters[key] = new prom.Counter({ - name: key, + name: buildKey(key), help: key, labelNames: ['name','host'] }) @@ -73,7 +73,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !gauges[key] gauges[key] = new prom.Gauge({ - name: key, + name: buildKey(key), help: key, labelNames: ['name','host'] }) @@ -84,7 +84,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !gauges[key] gauges[key] = new prom.Gauge({ - name: key, + name: buildKey(key), help: key, labelNames: ['name','host'] }) From 816c9348f65fc0e19af991e9d2507b4871bd7c74 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 16:13:40 +0000 Subject: [PATCH 02/32] don't put dots in key name --- libraries/metrics/metrics.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index e869ca0cff..822496b57b 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -46,7 +46,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !counters[key] counters[key] = new prom.Counter({ - name: buildKey(key), + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) @@ -73,7 +73,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !gauges[key] gauges[key] = new prom.Gauge({ - name: buildKey(key), + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) @@ -84,7 +84,7 @@ module.exports = Metrics = key = this.sanitizeKey(key) if !gauges[key] gauges[key] = new prom.Gauge({ - name: buildKey(key), + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) From 5039287ee432d06292083785bee8be64ca7b3a8d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 16:28:36 +0000 Subject: [PATCH 03/32] don't default to adding underscore to name --- libraries/metrics/metrics.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 822496b57b..50fdc1be6e 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -20,7 +20,7 @@ require "./uv_threadpool_size" module.exports = Metrics = initialize: (_name) -> - name = _name + "_" + name = _name collectDefaultMetrics({ timeout: 5000, prefix: name+"_"}) registerDestructor: (func) -> From 6f708fd26acf459303c48d9f0cbdcc80aed5babd Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 17:17:02 +0000 Subject: [PATCH 04/32] add summaries --- libraries/metrics/metrics.coffee | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 50fdc1be6e..9cf1f49758 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -13,6 +13,7 @@ buildGlobalKey = (key)-> "#{name}.global.#{key}" counters = {} gauges = {} +summaries = {} destructors = [] @@ -44,7 +45,7 @@ module.exports = Metrics = inc : (key, sampleRate = 1)-> statsd.increment buildKey(key), sampleRate key = this.sanitizeKey(key) - if !counters[key] + if !counters[key]? counters[key] = new prom.Counter({ name: "#{name}_#{key}", help: key, @@ -66,18 +67,26 @@ module.exports = Metrics = done:-> timeSpan = new Date - this.start statsd.timing(buildKey(this.key), timeSpan, this.sampleRate) + if !summaries[key]? + summary = new client.Summary({ + name: "#{name}_#{key}", + help: key, + maxAgeSeconds: 600, + ageBuckets: 10 + }) + summaries[key].observe(timeSpan) return timeSpan gauge : (key, value, sampleRate = 1)-> statsd.gauge buildKey(key), value, sampleRate key = this.sanitizeKey(key) - if !gauges[key] + if !gauges[key]? gauges[key] = new prom.Gauge({ - name: "#{name}_#{key}", + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) - gauges[key].set({name: name, host: hostname},this.sanitizeValue(value)) + gauges[key].set({name: name, host: hostname}, this.sanitizeValue(value)) globalGauge: (key, value, sampleRate = 1)-> statsd.gauge buildGlobalKey(key), value, sampleRate From 6854a64b73e03634fde09c104fcc5f13d9782906 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 17:20:49 +0000 Subject: [PATCH 05/32] fix this.key --- libraries/metrics/metrics.coffee | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 9cf1f49758..9c59c9f727 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -64,17 +64,18 @@ module.exports = Metrics = this.start = new Date() this.key = key this.sampleRate = sampleRate + done:-> timeSpan = new Date - this.start statsd.timing(buildKey(this.key), timeSpan, this.sampleRate) - if !summaries[key]? + if !summaries[this.key]? summary = new client.Summary({ - name: "#{name}_#{key}", - help: key, + name: "#{name}_#{this.key}", + help: this.key, maxAgeSeconds: 600, ageBuckets: 10 }) - summaries[key].observe(timeSpan) + summaries[this.key].observe(timeSpan) return timeSpan gauge : (key, value, sampleRate = 1)-> From bd8fd1141e145c27f703244a303486dae82abb71 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 17:25:37 +0000 Subject: [PATCH 06/32] client -> prom --- libraries/metrics/metrics.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 9c59c9f727..5b7049cc7f 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -69,7 +69,7 @@ module.exports = Metrics = timeSpan = new Date - this.start statsd.timing(buildKey(this.key), timeSpan, this.sampleRate) if !summaries[this.key]? - summary = new client.Summary({ + summary = new prom.Summary({ name: "#{name}_#{this.key}", help: this.key, maxAgeSeconds: 600, From 7fc49d1eb546357998dce7748a335754052cdf56 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 17:50:54 +0000 Subject: [PATCH 07/32] metrics of different types can clash, share promMetrics --- libraries/metrics/metrics.coffee | 34 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 5b7049cc7f..fdc45cf4f1 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -11,9 +11,7 @@ hostname = require('os').hostname() buildKey = (key)-> "#{name}.#{hostname}.#{key}" buildGlobalKey = (key)-> "#{name}.global.#{key}" -counters = {} -gauges = {} -summaries = {} +promMetrics = {} destructors = [] @@ -45,13 +43,13 @@ module.exports = Metrics = inc : (key, sampleRate = 1)-> statsd.increment buildKey(key), sampleRate key = this.sanitizeKey(key) - if !counters[key]? - counters[key] = new prom.Counter({ - name: "#{name}_#{key}", + if !promMetrics[key]? + promMetrics[key] = new prom.Counter({ + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) - counters[key].inc({name: name, host: hostname}) + promMetrics[key].inc({name: name, host: hostname}) count : (key, count, sampleRate = 1)-> statsd.count buildKey(key), count, sampleRate @@ -68,37 +66,37 @@ module.exports = Metrics = done:-> timeSpan = new Date - this.start statsd.timing(buildKey(this.key), timeSpan, this.sampleRate) - if !summaries[this.key]? - summary = new prom.Summary({ - name: "#{name}_#{this.key}", + if !promMetrics[this.key] + promMetrics[this.key] = new prom.Summary({ + name: "#{name}_timer_#{this.key}".replace(/\./g,"_"), help: this.key, maxAgeSeconds: 600, ageBuckets: 10 }) - summaries[this.key].observe(timeSpan) + promMetrics[this.key].observe(timeSpan) return timeSpan gauge : (key, value, sampleRate = 1)-> statsd.gauge buildKey(key), value, sampleRate key = this.sanitizeKey(key) - if !gauges[key]? - gauges[key] = new prom.Gauge({ + if !promMetrics[key] + promMetrics[key] = new prom.Gauge({ name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) - gauges[key].set({name: name, host: hostname}, this.sanitizeValue(value)) + promMetrics[key].set({name: name, host: hostname}, this.sanitizeValue(value)) globalGauge: (key, value, sampleRate = 1)-> statsd.gauge buildGlobalKey(key), value, sampleRate key = this.sanitizeKey(key) - if !gauges[key] - gauges[key] = new prom.Gauge({ - name: "#{name}_#{key}", + if !promMetrics[key] + promMetrics[key] = new prom.Gauge({ + name: "#{name}_#{key}", help: key, labelNames: ['name','host'] }) - gauges[key].set({name: name},this.sanitizeValue(value)) + promMetrics[key].set({name: name},this.sanitizeValue(value)) mongodb: require "./mongodb" http: require "./http" From 5599521c098ee4707386513fc7fb02f5a01503aa Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 20 Nov 2018 21:42:34 +0000 Subject: [PATCH 08/32] consolodate on metrics.timing --- libraries/metrics/metrics.coffee | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index fdc45cf4f1..0e54936429 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -56,6 +56,14 @@ module.exports = Metrics = timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) + if !promMetrics[this.key] + promMetrics[this.key] = new prom.Summary({ + name: "#{name}_timer_#{this.key}".replace(/\./g,"_"), + help: key, + maxAgeSeconds: 600, + ageBuckets: 10 + }) + promMetrics[this.key].observe(timeSpan) Timer : class constructor :(key, sampleRate = 1)-> @@ -65,15 +73,7 @@ module.exports = Metrics = done:-> timeSpan = new Date - this.start - statsd.timing(buildKey(this.key), timeSpan, this.sampleRate) - if !promMetrics[this.key] - promMetrics[this.key] = new prom.Summary({ - name: "#{name}_timer_#{this.key}".replace(/\./g,"_"), - help: this.key, - maxAgeSeconds: 600, - ageBuckets: 10 - }) - promMetrics[this.key].observe(timeSpan) + Metrics.timing(this.key, timeSpan, this.sampleRate) return timeSpan gauge : (key, value, sampleRate = 1)-> From cf920a86f747a98ebd1750e1eef42a0bdeeaf429 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 08:50:33 +0000 Subject: [PATCH 09/32] fix metrics.this --- libraries/metrics/metrics.coffee | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 0e54936429..56356ace27 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -56,14 +56,16 @@ module.exports = Metrics = timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) - if !promMetrics[this.key] - promMetrics[this.key] = new prom.Summary({ - name: "#{name}_timer_#{this.key}".replace(/\./g,"_"), + if !promMetrics[key] + k = "#{name}_timer_#{key}".replace(/\./g,"_") + console.log("sending timing", k) + promMetrics[key] = new prom.Summary({ + name: k, help: key, maxAgeSeconds: 600, ageBuckets: 10 }) - promMetrics[this.key].observe(timeSpan) + promMetrics[key].observe(timeSpan) Timer : class constructor :(key, sampleRate = 1)-> From c36e3d74b5908389ec053417ac555159cb0a3301 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 08:57:00 +0000 Subject: [PATCH 10/32] replace - with _ --- libraries/metrics/metrics.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 56356ace27..7814dcf9bd 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -57,7 +57,7 @@ module.exports = Metrics = timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) if !promMetrics[key] - k = "#{name}_timer_#{key}".replace(/\./g,"_") + k = "#{name}_timer_#{key}".replace(/\./g,"_").replace(/-/g,"_") console.log("sending timing", k) promMetrics[key] = new prom.Summary({ name: k, From 70a75113cb9bdf154bae898774461bf6fc913899 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 10:01:03 +0000 Subject: [PATCH 11/32] refactor key building --- libraries/metrics/metrics.coffee | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 7814dcf9bd..3e7dfc60af 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -11,6 +11,8 @@ hostname = require('os').hostname() buildKey = (key)-> "#{name}.#{hostname}.#{key}" buildGlobalKey = (key)-> "#{name}.global.#{key}" + + promMetrics = {} destructors = [] @@ -20,7 +22,7 @@ require "./uv_threadpool_size" module.exports = Metrics = initialize: (_name) -> name = _name - collectDefaultMetrics({ timeout: 5000, prefix: name+"_"}) + collectDefaultMetrics({ timeout: 5000, prefix: Metrics.buildPromKey()}) registerDestructor: (func) -> destructors.push func @@ -31,6 +33,9 @@ module.exports = Metrics = res.end(Register.metrics()) ) + buildPromKey: (key = "")-> + Metrics.sanitizeKey "#{name}_#{key}" + sanitizeKey: (key) -> key.replace /[^a-zA-Z0-9]/g, "_" @@ -42,10 +47,10 @@ module.exports = Metrics = inc : (key, sampleRate = 1)-> statsd.increment buildKey(key), sampleRate - key = this.sanitizeKey(key) + key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ - name: "#{name}_#{key}", + name: key, help: key, labelNames: ['name','host'] }) @@ -56,11 +61,10 @@ module.exports = Metrics = timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) + key = Metrics.buildPromKey("timer_#{key}") if !promMetrics[key] - k = "#{name}_timer_#{key}".replace(/\./g,"_").replace(/-/g,"_") - console.log("sending timing", k) promMetrics[key] = new prom.Summary({ - name: k, + name: key, help: key, maxAgeSeconds: 600, ageBuckets: 10 @@ -70,6 +74,7 @@ module.exports = Metrics = Timer : class constructor :(key, sampleRate = 1)-> this.start = new Date() + key = Metrics.sanitizeKey(key) this.key = key this.sampleRate = sampleRate @@ -80,10 +85,10 @@ module.exports = Metrics = gauge : (key, value, sampleRate = 1)-> statsd.gauge buildKey(key), value, sampleRate - key = this.sanitizeKey(key) + key = Metrics.buildPromKey(key) if !promMetrics[key] promMetrics[key] = new prom.Gauge({ - name: "#{name}_#{key}", + name: key, help: key, labelNames: ['name','host'] }) @@ -91,10 +96,10 @@ module.exports = Metrics = globalGauge: (key, value, sampleRate = 1)-> statsd.gauge buildGlobalKey(key), value, sampleRate - key = this.sanitizeKey(key) + key = Metrics.buildPromKey(key) if !promMetrics[key] promMetrics[key] = new prom.Gauge({ - name: "#{name}_#{key}", + name: key, help: key, labelNames: ['name','host'] }) From 32da84a1d7fbf6b5f40bae22ba028b0f97a76e01 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 10:20:33 +0000 Subject: [PATCH 12/32] proper null check --- libraries/metrics/metrics.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 3e7dfc60af..78a512d3b5 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -62,7 +62,7 @@ module.exports = Metrics = timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) key = Metrics.buildPromKey("timer_#{key}") - if !promMetrics[key] + if !promMetrics[key]? promMetrics[key] = new prom.Summary({ name: key, help: key, @@ -86,7 +86,7 @@ module.exports = Metrics = gauge : (key, value, sampleRate = 1)-> statsd.gauge buildKey(key), value, sampleRate key = Metrics.buildPromKey(key) - if !promMetrics[key] + if !promMetrics[key]? promMetrics[key] = new prom.Gauge({ name: key, help: key, @@ -97,7 +97,7 @@ module.exports = Metrics = globalGauge: (key, value, sampleRate = 1)-> statsd.gauge buildGlobalKey(key), value, sampleRate key = Metrics.buildPromKey(key) - if !promMetrics[key] + if !promMetrics[key]? promMetrics[key] = new prom.Gauge({ name: key, help: key, From d01ff95b466288eee3e108c03fb2f6b4f6a14fc4 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 10:20:58 +0000 Subject: [PATCH 13/32] bump to 1.9.1 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index cd47fc1466..720bb915b8 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "1.9.0", + "version": "1.9.1", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From f63b84983d0155cc696823e45b1bd53a72ffdf0a Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 21 Nov 2018 10:28:32 +0000 Subject: [PATCH 14/32] add count to prom --- libraries/metrics/metrics.coffee | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 78a512d3b5..03cce16c74 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -58,6 +58,14 @@ module.exports = Metrics = count : (key, count, sampleRate = 1)-> statsd.count buildKey(key), count, sampleRate + key = Metrics.buildPromKey(key) + if !promMetrics[key]? + promMetrics[key] = new prom.Counter({ + name: key, + help: key, + labelNames: ['name','host'] + }) + promMetrics[key].inc({name: name, host: hostname}, count) timing: (key, timeSpan, sampleRate)-> statsd.timing(buildKey(key), timeSpan, sampleRate) From 4806e6fd87cda661fd21e2fd0b37bb75efc36ea0 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 23 Nov 2018 16:24:31 +0000 Subject: [PATCH 15/32] use lables --- libraries/metrics/http.coffee | 2 +- libraries/metrics/metrics.coffee | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/libraries/metrics/http.coffee b/libraries/metrics/http.coffee index c175f26a14..1abaede902 100644 --- a/libraries/metrics/http.coffee +++ b/libraries/metrics/http.coffee @@ -12,7 +12,7 @@ module.exports.monitor = (logger) -> routePath = req.route.path.toString().replace(/\//g, '_').replace(/\:/g, '').slice(1) key = "http-requests.#{routePath}.#{req.method}.#{res.statusCode}" - Metrics.timing(key, responseTime) + Metrics.timing("http_request", responseTime, null, {method:req.method, status_code:res.status_code, path:routePath}) logger.log req: url: req.originalUrl || req.url diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 908efb6caf..60624e4ac2 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -78,17 +78,20 @@ module.exports = Metrics = }) promMetrics[key].inc({name: name, host: hostname}, count) - timing: (key, timeSpan, sampleRate)-> + timing: (key, timeSpan, sampleRate, opts = {})-> statsd.timing(buildKey(key), timeSpan, sampleRate) - key = Metrics.buildPromKey("timer_#{key}") + key = Metrics.sanitizeKey("timer_" + key) if !promMetrics[key]? promMetrics[key] = new prom.Summary({ name: key, help: key, maxAgeSeconds: 600, - ageBuckets: 10 + ageBuckets: 10, + labelNames: ['app', 'path', 'status_code', 'method'] }) - promMetrics[key].observe(timeSpan) + opts.app = name + console.log(key, opts, "timing key") + promMetrics[key].observe(opts, timeSpan) Timer : class constructor :(key, sampleRate = 1)-> @@ -109,9 +112,9 @@ module.exports = Metrics = promMetrics[key] = new prom.Gauge({ name: key, help: key, - labelNames: ['name','host'] + labelNames: ['app','host'] }) - promMetrics[key].set({name: name, host: hostname}, this.sanitizeValue(value)) + promMetrics[key].set({app: name, host: hostname}, this.sanitizeValue(value)) globalGauge: (key, value, sampleRate = 1)-> statsd.gauge buildGlobalKey(key), value, sampleRate @@ -120,9 +123,9 @@ module.exports = Metrics = promMetrics[key] = new prom.Gauge({ name: key, help: key, - labelNames: ['name','host'] + labelNames: ['app','host'] }) - promMetrics[key].set({name: name},this.sanitizeValue(value)) + promMetrics[key].set({app: name},this.sanitizeValue(value)) mongodb: require "./mongodb" http: require "./http" From b9f3a3f987c90ba063b5e6702c1ed4ed017a01e8 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 26 Nov 2018 11:08:29 +0000 Subject: [PATCH 16/32] improve mongo metrics to be more promethious like --- libraries/metrics/http.coffee | 2 +- libraries/metrics/metrics.coffee | 8 ++++---- libraries/metrics/mongodb.coffee | 2 +- libraries/metrics/timeAsyncMethod.coffee | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/metrics/http.coffee b/libraries/metrics/http.coffee index 1abaede902..45f72b7642 100644 --- a/libraries/metrics/http.coffee +++ b/libraries/metrics/http.coffee @@ -12,7 +12,7 @@ module.exports.monitor = (logger) -> routePath = req.route.path.toString().replace(/\//g, '_').replace(/\:/g, '').slice(1) key = "http-requests.#{routePath}.#{req.method}.#{res.statusCode}" - Metrics.timing("http_request", responseTime, null, {method:req.method, status_code:res.status_code, path:routePath}) + Metrics.timing("http_request", responseTime, null, {method:req.method, status_code: res.statusCode, path:routePath}) logger.log req: url: req.originalUrl || req.url diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 60624e4ac2..7e2927e7ae 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -87,22 +87,22 @@ module.exports = Metrics = help: key, maxAgeSeconds: 600, ageBuckets: 10, - labelNames: ['app', 'path', 'status_code', 'method'] + labelNames: ['app', 'path', 'status_code', 'method', 'collection', 'query'] }) opts.app = name - console.log(key, opts, "timing key") promMetrics[key].observe(opts, timeSpan) Timer : class - constructor :(key, sampleRate = 1)-> + constructor :(key, sampleRate = 1, opts)-> this.start = new Date() key = Metrics.sanitizeKey(key) this.key = key this.sampleRate = sampleRate + this.opts = opts done:-> timeSpan = new Date - this.start - Metrics.timing(this.key, timeSpan, this.sampleRate) + Metrics.timing(this.key, timeSpan, this.sampleRate, this.opts) return timeSpan gauge : (key, value, sampleRate = 1)-> diff --git a/libraries/metrics/mongodb.coffee b/libraries/metrics/mongodb.coffee index ce1ca71385..1f54cdde9a 100644 --- a/libraries/metrics/mongodb.coffee +++ b/libraries/metrics/mongodb.coffee @@ -34,7 +34,7 @@ module.exports = query = Object.keys(db_command.query).sort().join("_") key += "." + query - timer = new Metrics.Timer(key) + timer = new Metrics.Timer("mongo", {collection: collection, query:query}) start = new Date() _method.call this, db_command, options, () -> timer.done() diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 27e21e6e09..6986299390 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -17,7 +17,7 @@ module.exports = (obj, methodName, prefix, logger) -> logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback" return realMethod.apply this, originalArgs - timer = new metrics.Timer(key) + timer = new metrics.Timer(prefix, null, {method: methodName}) realMethod.call this, firstArgs..., (callbackArgs...) -> elapsedTime = timer.done() From fdd4db25a3ff742e95c3084c36371ff13c87c5df Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 10:01:33 +0000 Subject: [PATCH 17/32] big refactor remoing statsd and converting keys to prom keys --- libraries/metrics/metrics.coffee | 35 ++++++++++-------------- libraries/metrics/timeAsyncMethod.coffee | 11 ++++++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 7e2927e7ae..80dbf3cd03 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -1,6 +1,3 @@ -StatsD = require('lynx') -statsd = new StatsD(process.env["STATSD_HOST"] or "localhost", 8125, {on_error:->}) - traceAgent = require('@google-cloud/trace-agent') debugAgent = require('@google-cloud/debug-agent') @@ -8,7 +5,7 @@ prom = require('prom-client') Register = require('prom-client').register collectDefaultMetrics = prom.collectDefaultMetrics -name = "unknown" +appname = "unknown" hostname = require('os').hostname() buildKey = (key)-> "#{name}.#{hostname}.#{key}" @@ -24,13 +21,13 @@ require "./uv_threadpool_size" module.exports = Metrics = initialize: (_name) -> - name = _name + appname = _name collectDefaultMetrics({ timeout: 5000, prefix: Metrics.buildPromKey()}) traceAgent.start() debugAgent.start({ serviceContext: { allowExpressions: true, - service: name, + service: appname, version: '0.0.1' } }) @@ -45,7 +42,7 @@ module.exports = Metrics = ) buildPromKey: (key = "")-> - Metrics.sanitizeKey "#{name}_#{key}" + Metrics.sanitizeKey key sanitizeKey: (key) -> key.replace /[^a-zA-Z0-9]/g, "_" @@ -54,32 +51,29 @@ module.exports = Metrics = parseFloat(value) set : (key, value, sampleRate = 1)-> - statsd.set buildKey(key), value, sampleRate inc : (key, sampleRate = 1)-> - statsd.increment buildKey(key), sampleRate key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ name: key, help: key, - labelNames: ['name','host'] + labelNames: ['app','host'] }) - promMetrics[key].inc({name: name, host: hostname}) + console.log("doing inc", key, appname) + promMetrics[key].inc({app: appname, host: hostname}) count : (key, count, sampleRate = 1)-> - statsd.count buildKey(key), count, sampleRate key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ name: key, help: key, - labelNames: ['name','host'] + labelNames: ['app','host'] }) - promMetrics[key].inc({name: name, host: hostname}, count) + promMetrics[key].inc({app: appname, host: hostname}, count) timing: (key, timeSpan, sampleRate, opts = {})-> - statsd.timing(buildKey(key), timeSpan, sampleRate) key = Metrics.sanitizeKey("timer_" + key) if !promMetrics[key]? promMetrics[key] = new prom.Summary({ @@ -89,12 +83,14 @@ module.exports = Metrics = ageBuckets: 10, labelNames: ['app', 'path', 'status_code', 'method', 'collection', 'query'] }) - opts.app = name + opts.app = appname + console.log("doing timing", key, opts) promMetrics[key].observe(opts, timeSpan) Timer : class constructor :(key, sampleRate = 1, opts)-> this.start = new Date() + console.log("creating new timer", key) key = Metrics.sanitizeKey(key) this.key = key this.sampleRate = sampleRate @@ -106,7 +102,6 @@ module.exports = Metrics = return timeSpan gauge : (key, value, sampleRate = 1)-> - statsd.gauge buildKey(key), value, sampleRate key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Gauge({ @@ -114,10 +109,9 @@ module.exports = Metrics = help: key, labelNames: ['app','host'] }) - promMetrics[key].set({app: name, host: hostname}, this.sanitizeValue(value)) + promMetrics[key].set({app: appname, host: hostname}, this.sanitizeValue(value)) globalGauge: (key, value, sampleRate = 1)-> - statsd.gauge buildGlobalKey(key), value, sampleRate key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Gauge({ @@ -125,7 +119,7 @@ module.exports = Metrics = help: key, labelNames: ['app','host'] }) - promMetrics[key].set({app: name},this.sanitizeValue(value)) + promMetrics[key].set({app: appname},this.sanitizeValue(value)) mongodb: require "./mongodb" http: require "./http" @@ -138,4 +132,3 @@ module.exports = Metrics = close: () -> for func in destructors func() - statsd.close() diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 6986299390..75eb9a131f 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -6,8 +6,14 @@ module.exports = (obj, methodName, prefix, logger) -> throw new Error("[Metrics] expected object property '#{methodName}' to be a function") realMethod = obj[methodName] - key = "#{prefix}.#{methodName}" + keys = prefix.split(".") + key = keys[0].toLowerCase() + + if keys[1]? + methodName = "#{keys[1]}_#{methodName}" + + console.log "Async method", keys, methodName obj[methodName] = (originalArgs...) -> [firstArgs..., callback] = originalArgs @@ -17,7 +23,8 @@ module.exports = (obj, methodName, prefix, logger) -> logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback" return realMethod.apply this, originalArgs - timer = new metrics.Timer(prefix, null, {method: methodName}) + console.log("creating timer for async method") + timer = new metrics.Timer(key, null, {method: methodName}) realMethod.call this, firstArgs..., (callbackArgs...) -> elapsedTime = timer.done() From 2a6839f48cd3076a1c503a610ad4b1ec4f749867 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 10:46:54 +0000 Subject: [PATCH 18/32] roll back async method for moment --- libraries/metrics/timeAsyncMethod.coffee | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 75eb9a131f..5248c43f79 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -5,15 +5,14 @@ module.exports = (obj, methodName, prefix, logger) -> if typeof obj[methodName] != 'function' throw new Error("[Metrics] expected object property '#{methodName}' to be a function") + key = "#{prefix}.#{methodName}" + realMethod = obj[methodName] - keys = prefix.split(".") - key = keys[0].toLowerCase() + startPrefix = prefix.split(".")[0] - - if keys[1]? - methodName = "#{keys[1]}_#{methodName}" - - console.log "Async method", keys, methodName + endPrefix = prefix.split(".")[1] + modifedMethodName = "#{endPrefix}_methodName" + console.log "Async method", prefix, key, methodName, modifedMethodName obj[methodName] = (originalArgs...) -> [firstArgs..., callback] = originalArgs @@ -23,8 +22,11 @@ module.exports = (obj, methodName, prefix, logger) -> logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback" return realMethod.apply this, originalArgs - console.log("creating timer for async method") - timer = new metrics.Timer(key, null, {method: methodName}) + console.log("creating timer for async method", prefix, startPrefix, modifedMethodName) + + + timer = new metrics.Timer(prefix, null, {method: methodName}) + console.log("changed timer for async method", prefix, methodName, key) realMethod.call this, firstArgs..., (callbackArgs...) -> elapsedTime = timer.done() From a8cfa97463c193206ef2ff744ec7b4c54baa0753 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 11:33:45 +0000 Subject: [PATCH 19/32] use modified prefix and method name --- libraries/metrics/metrics.coffee | 8 +++++--- libraries/metrics/timeAsyncMethod.coffee | 9 ++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 80dbf3cd03..a603bc5327 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -52,16 +52,18 @@ module.exports = Metrics = set : (key, value, sampleRate = 1)-> - inc : (key, sampleRate = 1)-> + inc : (key, sampleRate = 1, opts)-> key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ name: key, help: key, - labelNames: ['app','host'] + labelNames: ['app','host','status','method'] }) console.log("doing inc", key, appname) - promMetrics[key].inc({app: appname, host: hostname}) + opts.app = appname + opts.host = hostname + promMetrics[key].inc(opts) count : (key, count, sampleRate = 1)-> key = Metrics.buildPromKey(key) diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 5248c43f79..0bf0108e56 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -11,7 +11,7 @@ module.exports = (obj, methodName, prefix, logger) -> startPrefix = prefix.split(".")[0] endPrefix = prefix.split(".")[1] - modifedMethodName = "#{endPrefix}_methodName" + modifedMethodName = "#{endPrefix}_#{methodName}" console.log "Async method", prefix, key, methodName, modifedMethodName obj[methodName] = (originalArgs...) -> @@ -25,16 +25,15 @@ module.exports = (obj, methodName, prefix, logger) -> console.log("creating timer for async method", prefix, startPrefix, modifedMethodName) - timer = new metrics.Timer(prefix, null, {method: methodName}) - console.log("changed timer for async method", prefix, methodName, key) + timer = new metrics.Timer(startPrefix, null, {method: modifedMethodName}) realMethod.call this, firstArgs..., (callbackArgs...) -> elapsedTime = timer.done() possibleError = callbackArgs[0] if possibleError? - metrics.inc "#{key}.failure" + metrics.inc "#{startPrefix}", null, {status:"success", method: modifedMethodName} else - metrics.inc "#{key}.success" + metrics.inc "#{startPrefix}", null, {status:"failed", method: modifedMethodName} if logger? loggableArgs = {} try From 4e370ef24ddd3a7bc4814cfd44aa9806b15bddc6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 12:04:50 +0000 Subject: [PATCH 20/32] clean up --- libraries/metrics/metrics.coffee | 2 +- libraries/metrics/timeAsyncMethod.coffee | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index a603bc5327..1c19fa05e9 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -60,7 +60,7 @@ module.exports = Metrics = help: key, labelNames: ['app','host','status','method'] }) - console.log("doing inc", key, appname) + console.log("doing inc", key, opts) opts.app = appname opts.host = hostname promMetrics[key].inc(opts) diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 0bf0108e56..e09e9bfa43 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -8,10 +8,14 @@ module.exports = (obj, methodName, prefix, logger) -> key = "#{prefix}.#{methodName}" realMethod = obj[methodName] - startPrefix = prefix.split(".")[0] - endPrefix = prefix.split(".")[1] - modifedMethodName = "#{endPrefix}_#{methodName}" + splitPrefix = prefix.split(".") + startPrefix = splitPrefix[0] + + if splitPrefix[1]? + modifedMethodName = "#{splitPrefix[1]}_#{methodName}" + else + modifedMethodName = methodName console.log "Async method", prefix, key, methodName, modifedMethodName obj[methodName] = (originalArgs...) -> @@ -31,9 +35,9 @@ module.exports = (obj, methodName, prefix, logger) -> elapsedTime = timer.done() possibleError = callbackArgs[0] if possibleError? - metrics.inc "#{startPrefix}", null, {status:"success", method: modifedMethodName} + metrics.inc "#{startPrefix}_result", null, {status:"failed", method: modifedMethodName} else - metrics.inc "#{startPrefix}", null, {status:"failed", method: modifedMethodName} + metrics.inc "#{startPrefix}_result", null, {status:"success", method: modifedMethodName} if logger? loggableArgs = {} try From 39f924f73cdf94f75731361984551a393ee6934c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 15:15:45 +0000 Subject: [PATCH 21/32] bump to 2.0.0 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index d51be1b107..63772e7ca1 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "1.9.1", + "version": "2.0.0", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From 8adfc49af7f863c589d80e158180fa958195455f Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 15:48:17 +0000 Subject: [PATCH 22/32] v 2.0.1 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index 63772e7ca1..0adf79ca3e 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.0", + "version": "2.0.1", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From 917afa3edc1ce66bce93a95ffa313f380f41eadf Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 27 Nov 2018 16:12:12 +0000 Subject: [PATCH 23/32] have default opts in inc --- libraries/metrics/metrics.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 1c19fa05e9..e5401d1c88 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -52,7 +52,7 @@ module.exports = Metrics = set : (key, value, sampleRate = 1)-> - inc : (key, sampleRate = 1, opts)-> + inc : (key, sampleRate = 1, opts = {})-> key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ From 9a737bef2fb4c9e94140cb3ab5a05918e2bb9c47 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 28 Nov 2018 10:45:11 +0000 Subject: [PATCH 24/32] bump to 2.0.3 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index 0adf79ca3e..02ee3101cf 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.1", + "version": "2.0.3", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From 752541b7f104d5fa2dc6cb9223973c4da547d9f9 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 29 Nov 2018 16:10:55 +0000 Subject: [PATCH 25/32] remove logging and bump to 2.0.4 --- libraries/metrics/metrics.coffee | 3 --- libraries/metrics/package.json | 2 +- libraries/metrics/timeAsyncMethod.coffee | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index e5401d1c88..3a778c052d 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -60,7 +60,6 @@ module.exports = Metrics = help: key, labelNames: ['app','host','status','method'] }) - console.log("doing inc", key, opts) opts.app = appname opts.host = hostname promMetrics[key].inc(opts) @@ -86,13 +85,11 @@ module.exports = Metrics = labelNames: ['app', 'path', 'status_code', 'method', 'collection', 'query'] }) opts.app = appname - console.log("doing timing", key, opts) promMetrics[key].observe(opts, timeSpan) Timer : class constructor :(key, sampleRate = 1, opts)-> this.start = new Date() - console.log("creating new timer", key) key = Metrics.sanitizeKey(key) this.key = key this.sampleRate = sampleRate diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index 02ee3101cf..c29e059b6a 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.3", + "version": "2.0.4", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index e09e9bfa43..73f69cc334 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -16,7 +16,6 @@ module.exports = (obj, methodName, prefix, logger) -> modifedMethodName = "#{splitPrefix[1]}_#{methodName}" else modifedMethodName = methodName - console.log "Async method", prefix, key, methodName, modifedMethodName obj[methodName] = (originalArgs...) -> [firstArgs..., callback] = originalArgs @@ -26,9 +25,6 @@ module.exports = (obj, methodName, prefix, logger) -> logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback" return realMethod.apply this, originalArgs - console.log("creating timer for async method", prefix, startPrefix, modifedMethodName) - - timer = new metrics.Timer(startPrefix, null, {method: modifedMethodName}) realMethod.call this, firstArgs..., (callbackArgs...) -> From 27e6db1e5130d9d40e500098f916255a337e9101 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 15:57:19 +0000 Subject: [PATCH 26/32] inc process_startup on init --- libraries/metrics/metrics.coffee | 1 + libraries/metrics/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 3a778c052d..068069a482 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -31,6 +31,7 @@ module.exports = Metrics = version: '0.0.1' } }) + Metrics.inc("process_startup") registerDestructor: (func) -> destructors.push func diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index c29e059b6a..07805f8f1e 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.4", + "version": "2.0.5", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From 31fa5cef517d2e39337610e0143e5c2545209cbf Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 16:09:09 +0000 Subject: [PATCH 27/32] add logging --- libraries/metrics/metrics.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 068069a482..a644a4a405 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -54,6 +54,7 @@ module.exports = Metrics = set : (key, value, sampleRate = 1)-> inc : (key, sampleRate = 1, opts = {})-> + console.log("doing inc", key) key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ From 2d4283fdf06aa369cbb12a88a6079121dbe0b9ee Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 16:20:52 +0000 Subject: [PATCH 28/32] add DEBUG_METRICS env var --- libraries/metrics/metrics.coffee | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index a644a4a405..77953b275b 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -54,7 +54,6 @@ module.exports = Metrics = set : (key, value, sampleRate = 1)-> inc : (key, sampleRate = 1, opts = {})-> - console.log("doing inc", key) key = Metrics.buildPromKey(key) if !promMetrics[key]? promMetrics[key] = new prom.Counter({ @@ -65,6 +64,8 @@ module.exports = Metrics = opts.app = appname opts.host = hostname promMetrics[key].inc(opts) + if process.env['DEBUG_METRICS'] + console.log("doing inc", key, opts) count : (key, count, sampleRate = 1)-> key = Metrics.buildPromKey(key) @@ -75,6 +76,8 @@ module.exports = Metrics = labelNames: ['app','host'] }) promMetrics[key].inc({app: appname, host: hostname}, count) + if process.env['DEBUG_METRICS'] + console.log("doing count/inc", key, opts) timing: (key, timeSpan, sampleRate, opts = {})-> key = Metrics.sanitizeKey("timer_" + key) @@ -88,6 +91,8 @@ module.exports = Metrics = }) opts.app = appname promMetrics[key].observe(opts, timeSpan) + if process.env['DEBUG_METRICS'] + console.log("doing timing", key, opts) Timer : class constructor :(key, sampleRate = 1, opts)-> @@ -111,7 +116,9 @@ module.exports = Metrics = labelNames: ['app','host'] }) promMetrics[key].set({app: appname, host: hostname}, this.sanitizeValue(value)) - + if process.env['DEBUG_METRICS'] + console.log("doing gauge", key, opts) + globalGauge: (key, value, sampleRate = 1)-> key = Metrics.buildPromKey(key) if !promMetrics[key]? From 85d4b03bcb3bb0f0bd6b53220e4e8bce4a101676 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 16:29:09 +0000 Subject: [PATCH 29/32] bump to 2.0.6 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index 07805f8f1e..c889b4c116 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.5", + "version": "2.0.6", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git", From 93f4a7eeaf05cdf9affc267027bb5c598d6757f9 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 17:01:30 +0000 Subject: [PATCH 30/32] small cleanup --- libraries/metrics/http.coffee | 2 -- libraries/metrics/metrics.coffee | 2 -- libraries/metrics/timeAsyncMethod.coffee | 6 +++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/libraries/metrics/http.coffee b/libraries/metrics/http.coffee index 45f72b7642..f5e3719d53 100644 --- a/libraries/metrics/http.coffee +++ b/libraries/metrics/http.coffee @@ -10,8 +10,6 @@ module.exports.monitor = (logger) -> responseTime = new Date() - startTime if req.route?.path? routePath = req.route.path.toString().replace(/\//g, '_').replace(/\:/g, '').slice(1) - key = "http-requests.#{routePath}.#{req.method}.#{res.statusCode}" - Metrics.timing("http_request", responseTime, null, {method:req.method, status_code: res.statusCode, path:routePath}) logger.log req: diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index 77953b275b..c96db21c67 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -11,8 +11,6 @@ hostname = require('os').hostname() buildKey = (key)-> "#{name}.#{hostname}.#{key}" buildGlobalKey = (key)-> "#{name}.global.#{key}" - - promMetrics = {} destructors = [] diff --git a/libraries/metrics/timeAsyncMethod.coffee b/libraries/metrics/timeAsyncMethod.coffee index 73f69cc334..8d6099d826 100644 --- a/libraries/metrics/timeAsyncMethod.coffee +++ b/libraries/metrics/timeAsyncMethod.coffee @@ -25,15 +25,15 @@ module.exports = (obj, methodName, prefix, logger) -> logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback" return realMethod.apply this, originalArgs - timer = new metrics.Timer(startPrefix, null, {method: modifedMethodName}) + timer = new metrics.Timer(startPrefix, 1, {method: modifedMethodName}) realMethod.call this, firstArgs..., (callbackArgs...) -> elapsedTime = timer.done() possibleError = callbackArgs[0] if possibleError? - metrics.inc "#{startPrefix}_result", null, {status:"failed", method: modifedMethodName} + metrics.inc "#{startPrefix}_result", 1, {status:"failed", method: modifedMethodName} else - metrics.inc "#{startPrefix}_result", null, {status:"success", method: modifedMethodName} + metrics.inc "#{startPrefix}_result", 1, {status:"success", method: modifedMethodName} if logger? loggableArgs = {} try From b2f49351c0df84917c99220ca29fb02c5524887a Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 11:03:40 +0000 Subject: [PATCH 31/32] few tidy up changes --- libraries/metrics/metrics.coffee | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index c96db21c67..9f8df741dd 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -41,15 +41,13 @@ module.exports = Metrics = ) buildPromKey: (key = "")-> - Metrics.sanitizeKey key - - sanitizeKey: (key) -> key.replace /[^a-zA-Z0-9]/g, "_" sanitizeValue: (value) -> parseFloat(value) set : (key, value, sampleRate = 1)-> + console.log("counts are not currently supported") inc : (key, sampleRate = 1, opts = {})-> key = Metrics.buildPromKey(key) @@ -78,7 +76,7 @@ module.exports = Metrics = console.log("doing count/inc", key, opts) timing: (key, timeSpan, sampleRate, opts = {})-> - key = Metrics.sanitizeKey("timer_" + key) + key = Metrics.buildPromKey("timer_" + key) if !promMetrics[key]? promMetrics[key] = new prom.Summary({ name: key, @@ -95,7 +93,7 @@ module.exports = Metrics = Timer : class constructor :(key, sampleRate = 1, opts)-> this.start = new Date() - key = Metrics.sanitizeKey(key) + key = Metrics.buildPromKey(key) this.key = key this.sampleRate = sampleRate this.opts = opts From 02071584ae84b06cb8351966e33eb98d03ac585e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 12:31:16 +0000 Subject: [PATCH 32/32] bump to 2.0.7 --- libraries/metrics/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metrics/package.json b/libraries/metrics/package.json index c889b4c116..c56b67ab3d 100644 --- a/libraries/metrics/package.json +++ b/libraries/metrics/package.json @@ -1,6 +1,6 @@ { "name": "metrics-sharelatex", - "version": "2.0.6", + "version": "2.0.7", "description": "A drop-in metrics and monitoring module for node.js apps", "repository": { "type": "git",