Add a response-time metric (#3948)

* Refactor existing metrics support into MetricHelper

This completes the refactor done at https://github.com/badges/shields/pull/3662#issuecomment-509011229 in anticipation of adding more metrics support, such as response size of an upstream service, or response time.

* Clean up

* Renames

* Add response time metrics

This adds around 30 new metrics to cover response times at a fairly granular level. We may be able to shrink the number of buckets with time, though I think using 30 metrics is probably okay given that I think may become our most important metric.

* Fix

core/server/prometheus-metrics.js

@@ -1,16 +1,59 @@
 'use strict'
 
+const decamelize = require('decamelize')
 const prometheus = require('prom-client')
 
 module.exports = class PrometheusMetrics {
   constructor() {
     this.register = new prometheus.Registry()
-    this.requestCounter = new prometheus.Counter({
-      name: 'service_requests_total',
-      help: 'Total service requests',
-      labelNames: ['category', 'family', 'service'],
-      registers: [this.register],
-    })
+    this.counters = {
+      numRequests: new prometheus.Counter({
+        name: 'service_requests_total',
+        help: 'Total service requests',
+        labelNames: ['category', 'family', 'service'],
+        registers: [this.register],
+      }),
+      responseTime: new prometheus.Histogram({
+        name: 'service_response_millis',
+        help: 'Service response time in milliseconds',
+        // 250 ms increments up to 2 seconds, then 500 ms increments up to 8
+        // seconds, then 1 second increments up to 15 seconds.
+        buckets: [
+          250,
+          500,
+          750,
+          1000,
+          1250,
+          1500,
+          1750,
+          2000,
+          2250,
+          2500,
+          2750,
+          3000,
+          3250,
+          3500,
+          3750,
+          4000,
+          4500,
+          5000,
+          5500,
+          6000,
+          6500,
+          7000,
+          7500,
+          8000,
+          9000,
+          10000,
+          11000,
+          12000,
+          13000,
+          14000,
+          15000,
+        ],
+        registers: [this.register],
+      }),
+    }
   }
 
   async initialize(server) {
@@ -30,4 +73,16 @@ module.exports = class PrometheusMetrics {
       this.interval = undefined
     }
   }
+
+  /**
+   * @returns {object} `{ inc() {} }`.
+   */
+  createNumRequestCounter({ category, serviceFamily, name }) {
+    const service = decamelize(name)
+    return this.counters.numRequests.labels(category, serviceFamily, service)
+  }
+
+  noteResponseTime(responseTime) {
+    return this.counters.responseTime.observe(responseTime)
+  }
 }