📈 PaaS-friendly metrics (#4874)

* prom-client JSON to InfluxDB line protocol converter

* Converts a metric with separate names

* prom-client JSON to InfluxDB line protocol (version 2) converter

* Server has instance id

* Read the instance id from an environment variable

* More unit tests for instance-metadata

* Log instance id

* Push influx metrics

* INSTANCE_ID with dyno metadata

* Prepare influx metrics in one place

* Influx metrics endpoint should return metrics

* More readable tests

* Env added to instance metadata

* hostname as an instance label value

* HEROKU_DYNO_ID as an instance id for heroku

* Instance env can be set by env variable

* HEROKU_APP_NAME as an instance env

* Log instance metadata as a JSON

* Typo fix

* Code refactoring in tests

* wait-for-expect dev dependency added

* Test for pushing metrics

* Test for pushing metrics

* Use basic authentication for pushing metrics

* intervalSeconds=2 for development env

* Using existing methods

* TODOs removed

* Schema for influx credentials

* Influx config removed from config files

* Require username and password when influx metrics are enabled

* Unused args removed

* pushing component should log errors

* Speed up tests

* should log error responses

* InstanceMetadata class replaces by simple object

* Influx metrics can be configuredd by env variables

* Use application label name instead of service

* Unused code removed

* Integration test for prom-client and converter

* metrics.influx.enabled configuration option added

* Improved influx configuration schema

* instanceMetadata validation

* Typo fix

* Default value for env

* metrics.infux.hostnameAsAInstanceId added

* should add hostname as an instance label when hostnameAsAInstanceId is enabled

* Default values for influx configuration

* flatMap is not available in Node.js 9.4

* Env vars removed from Procfile

* Better instance metadata values in tests

* Typo fix

* lodash.groupby added to prod dependencies

* Allow other keys in private config

* Missing test - should allow other private keys when influx metrics are enabled

* Missing test - should require private metrics config when influx configuration is enabled

* log.error instead of console.log

* metrics.influx.uri -> metrics.influx.url

* Unused arguments removed

* async removed

* promisify sendMetrics

* Allow to disable prometheus metrics

* Create test server with custom config

* 'metrics-influx' resource removed

* 'metrics-influx' resource removed

* Private config schema flattened out

* Extra code removed in Prometheus tests

* promisify moved outside of the class

* Do not throw errors from got in a specific test

* hostnameAliases added

* instanceIdFrom added

* instanceIdEnvVarName added

* envLabel added to schema

* instanceMetadata is not used by InfluxMetrics

* Instance metadata removed

* hostnameAsAnInstanceId removed

* A comment added

* waitForExpect removed

* Unused code removed

core/server/in-process-server-test-helpers.js

@@ -1,21 +1,16 @@
 'use strict'
 
+const merge = require('deepmerge')
 const config = require('config').util.toObject()
+const portfinder = require('portfinder')
 const Server = require('./server')
 
-function createTestServer({ port }) {
-  const serverConfig = {
-    ...config,
-    public: {
-      ...config.public,
-      bind: {
-        ...config.public.bind,
-        port,
-      },
-    },
+async function createTestServer(customConfig = {}) {
+  const mergedConfig = merge(config, customConfig)
+  if (!mergedConfig.public.bind.port) {
+    mergedConfig.public.bind.port = await portfinder.getPortPromise()
   }
-
-  return new Server(serverConfig)
+  return new Server(mergedConfig)
 }
 
 module.exports = {

core/server/influx-metrics.js

@@ -0,0 +1,86 @@
+'use strict'
+const os = require('os')
+const { promisify } = require('util')
+const { post } = require('request')
+const postAsync = promisify(post)
+const generateInstanceId = require('./instance-id-generator')
+const { promClientJsonToInfluxV2 } = require('./metrics/format-converters')
+const log = require('./log')
+
+module.exports = class InfluxMetrics {
+  constructor(metricInstance, config) {
+    this._metricInstance = metricInstance
+    this._config = config
+    this._instanceId = this.getInstanceId()
+  }
+
+  async sendMetrics() {
+    const auth = {
+      user: this._config.username,
+      pass: this._config.password,
+    }
+    const request = {
+      uri: this._config.url,
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: this.metrics(),
+      timeout: this._config.timeoutMillseconds,
+      auth,
+    }
+
+    let response
+    try {
+      response = await postAsync(request)
+    } catch (error) {
+      log.error(
+        new Error(`Cannot push metrics. Cause: ${error.name}: ${error.message}`)
+      )
+    }
+    if (response && response.statusCode >= 300) {
+      log.error(
+        new Error(
+          `Cannot push metrics. ${response.request.href} responded with status code ${response.statusCode}`
+        )
+      )
+    }
+  }
+
+  startPushingMetrics() {
+    this._intervalId = setInterval(
+      () => this.sendMetrics(),
+      this._config.intervalSeconds * 1000
+    )
+  }
+
+  metrics() {
+    return promClientJsonToInfluxV2(this._metricInstance.metrics(), {
+      env: this._config.envLabel,
+      application: 'shields',
+      instance: this._instanceId,
+    })
+  }
+
+  getInstanceId() {
+    const {
+      hostnameAliases = {},
+      instanceIdFrom,
+      instanceIdEnvVarName,
+    } = this._config
+    let instance
+    if (instanceIdFrom === 'env-var') {
+      instance = process.env[instanceIdEnvVarName]
+    } else if (instanceIdFrom === 'hostname') {
+      const hostname = os.hostname()
+      instance = hostnameAliases[hostname] || hostname
+    } else if (instanceIdFrom === 'random') {
+      instance = generateInstanceId()
+    }
+    return instance
+  }
+
+  stopPushingMetrics() {
+    if (this._intervalId) {
+      clearInterval(this._intervalId)
+      this._intervalId = undefined
+    }
+  }
+}

core/server/influx-metrics.spec.js

@@ -0,0 +1,177 @@
+'use strict'
+const os = require('os')
+const nock = require('nock')
+const sinon = require('sinon')
+const { expect } = require('chai')
+const log = require('./log')
+const InfluxMetrics = require('./influx-metrics')
+require('../register-chai-plugins.spec')
+describe('Influx metrics', function() {
+  const metricInstance = {
+    metrics() {
+      return [
+        {
+          help: 'counter 1 help',
+          name: 'counter1',
+          type: 'counter',
+          values: [{ value: 11, labels: {} }],
+          aggregator: 'sum',
+        },
+      ]
+    },
+  }
+  describe('"metrics" function', function() {
+    let osHostnameStub
+    afterEach(function() {
+      nock.enableNetConnect()
+      delete process.env.INSTANCE_ID
+      if (osHostnameStub) {
+        osHostnameStub.restore()
+      }
+    })
+    it('should use an environment variable value as an instance label', async function() {
+      process.env.INSTANCE_ID = 'instance3'
+      const influxMetrics = new InfluxMetrics(metricInstance, {
+        instanceIdFrom: 'env-var',
+        instanceIdEnvVarName: 'INSTANCE_ID',
+      })
+
+      expect(influxMetrics.metrics()).to.contain('instance=instance3')
+    })
+
+    it('should use a hostname as an instance label', async function() {
+      osHostnameStub = sinon.stub(os, 'hostname').returns('test-hostname')
+      const customConfig = {
+        instanceIdFrom: 'hostname',
+      }
+      const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+      expect(influxMetrics.metrics()).to.be.contain('instance=test-hostname')
+    })
+
+    it('should use a random string as an instance label', async function() {
+      const customConfig = {
+        instanceIdFrom: 'random',
+      }
+      const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+      expect(influxMetrics.metrics()).to.be.match(/instance=\w+ /)
+    })
+
+    it('should use a hostname alias as an instance label', async function() {
+      osHostnameStub = sinon.stub(os, 'hostname').returns('test-hostname')
+      const customConfig = {
+        instanceIdFrom: 'hostname',
+        hostnameAliases: { 'test-hostname': 'test-hostname-alias' },
+      }
+      const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+      expect(influxMetrics.metrics()).to.be.contain(
+        'instance=test-hostname-alias'
+      )
+    })
+  })
+
+  describe('startPushingMetrics', function() {
+    let influxMetrics, clock
+    beforeEach(function() {
+      clock = sinon.useFakeTimers()
+    })
+    afterEach(function() {
+      influxMetrics.stopPushingMetrics()
+      nock.cleanAll()
+      nock.enableNetConnect()
+      delete process.env.INSTANCE_ID
+      clock.restore()
+    })
+
+    it('should send metrics', async function() {
+      const scope = nock('http://shields-metrics.io/', {
+        reqheaders: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+        },
+      })
+        .persist()
+        .post(
+          '/metrics',
+          'prometheus,application=shields,env=test-env,instance=instance2 counter1=11'
+        )
+        .basicAuth({ user: 'metrics-username', pass: 'metrics-password' })
+        .reply(200)
+      process.env.INSTANCE_ID = 'instance2'
+      influxMetrics = new InfluxMetrics(metricInstance, {
+        url: 'http://shields-metrics.io/metrics',
+        timeoutMillseconds: 100,
+        intervalSeconds: 0.001,
+        username: 'metrics-username',
+        password: 'metrics-password',
+        instanceIdFrom: 'env-var',
+        instanceIdEnvVarName: 'INSTANCE_ID',
+        envLabel: 'test-env',
+      })
+
+      influxMetrics.startPushingMetrics()
+
+      await clock.tickAsync(10)
+      expect(scope.isDone()).to.be.equal(
+        true,
+        `pending mocks: ${scope.pendingMocks()}`
+      )
+    })
+  })
+
+  describe('sendMetrics', function() {
+    beforeEach(function() {
+      sinon.spy(log, 'error')
+    })
+    afterEach(function() {
+      log.error.restore()
+      nock.cleanAll()
+      nock.enableNetConnect()
+    })
+
+    const influxMetrics = new InfluxMetrics(metricInstance, {
+      url: 'http://shields-metrics.io/metrics',
+      timeoutMillseconds: 50,
+      intervalSeconds: 0,
+      username: 'metrics-username',
+      password: 'metrics-password',
+    })
+    it('should log errors', async function() {
+      nock.disableNetConnect()
+
+      await influxMetrics.sendMetrics()
+
+      expect(log.error).to.be.calledWith(
+        sinon.match
+          .instanceOf(Error)
+          .and(
+            sinon.match.has(
+              'message',
+              'Cannot push metrics. Cause: NetConnectNotAllowedError: Nock: Disallowed net connect for "shields-metrics.io:80/metrics"'
+            )
+          )
+      )
+    })
+
+    it('should log error responses', async function() {
+      nock('http://shields-metrics.io/')
+        .persist()
+        .post('/metrics')
+        .reply(400)
+
+      await influxMetrics.sendMetrics()
+
+      expect(log.error).to.be.calledWith(
+        sinon.match
+          .instanceOf(Error)
+          .and(
+            sinon.match.has(
+              'message',
+              'Cannot push metrics. http://shields-metrics.io/metrics responded with status code 400'
+            )
+          )
+      )
+    })
+  })
+})

core/server/instance-id-generator.js

@@ -0,0 +1,10 @@
+'use strict'
+
+function generateInstanceId() {
+  // from https://gist.github.com/gordonbrander/2230317
+  return Math.random()
+    .toString(36)
+    .substr(2, 9)
+}
+
+module.exports = generateInstanceId

core/server/metrics/format-converters.js

@@ -0,0 +1,27 @@
+'use strict'
+const groupBy = require('lodash.groupby')
+
+function promClientJsonToInfluxV2(metrics, extraLabels = {}) {
+  // TODO Replace with Array.prototype.flatMap() after migrating to Node.js >= 11
+  const flatMap = (f, arr) => arr.reduce((acc, x) => acc.concat(f(x)), [])
+  return flatMap(metric => {
+    const valuesByLabels = groupBy(metric.values, value =>
+      JSON.stringify(Object.entries(value.labels).sort())
+    )
+    return Object.values(valuesByLabels).map(metricsWithSameLabel => {
+      const labels = Object.entries(metricsWithSameLabel[0].labels)
+        .concat(Object.entries(extraLabels))
+        .sort((a, b) => a[0].localeCompare(b[0]))
+        .map(labelEntry => `${labelEntry[0]}=${labelEntry[1]}`)
+        .join(',')
+      const labelsFormatted = labels ? `,${labels}` : ''
+      const values = metricsWithSameLabel
+        .sort((a, b) => a.metricName.localeCompare(b.metricName))
+        .map(value => `${value.metricName || metric.name}=${value.value}`)
+        .join(',')
+      return `prometheus${labelsFormatted} ${values}`
+    })
+  }, metrics).join('\n')
+}
+
+module.exports = { promClientJsonToInfluxV2 }

core/server/metrics/format-converters.spec.js

@@ -0,0 +1,217 @@
+'use strict'
+
+const { expect } = require('chai')
+const prometheus = require('prom-client')
+const { promClientJsonToInfluxV2 } = require('./format-converters')
+
+describe('Metric format converters', function() {
+  describe('prom-client JSON to InfluxDB line protocol (version 2)', function() {
+    it('converts a counter', function() {
+      const json = [
+        {
+          help: 'counter 1 help',
+          name: 'counter1',
+          type: 'counter',
+          values: [{ value: 11, labels: {} }],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json)
+
+      expect(influx).to.be.equal('prometheus counter1=11')
+    })
+
+    it('converts a counter (from prometheus registry)', function() {
+      const register = new prometheus.Registry()
+      const counter = new prometheus.Counter({
+        name: 'counter1',
+        help: 'counter 1 help',
+        registers: [register],
+      })
+      counter.inc(11)
+
+      const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+      expect(influx).to.be.equal('prometheus counter1=11')
+    })
+
+    it('converts a gauge', function() {
+      const json = [
+        {
+          help: 'gause 1 help',
+          name: 'gauge1',
+          type: 'gauge',
+          values: [{ value: 20, labels: {} }],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json)
+
+      expect(influx).to.be.equal('prometheus gauge1=20')
+    })
+
+    it('converts a gauge (from prometheus registry)', function() {
+      const register = new prometheus.Registry()
+      const gauge = new prometheus.Gauge({
+        name: 'gauge1',
+        help: 'gauge 1 help',
+        registers: [register],
+      })
+      gauge.inc(20)
+
+      const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+      expect(influx).to.be.equal('prometheus gauge1=20')
+    })
+
+    const sortLines = text =>
+      text
+        .split('\n')
+        .sort()
+        .join('\n')
+
+    it('converts a histogram', function() {
+      const json = [
+        {
+          name: 'histogram1',
+          help: 'histogram 1 help',
+          type: 'histogram',
+          values: [
+            { labels: { le: 5 }, value: 1, metricName: 'histogram1_bucket' },
+            { labels: { le: 15 }, value: 2, metricName: 'histogram1_bucket' },
+            { labels: { le: 50 }, value: 2, metricName: 'histogram1_bucket' },
+            {
+              labels: { le: '+Inf' },
+              value: 3,
+              metricName: 'histogram1_bucket',
+            },
+            { labels: {}, value: 111, metricName: 'histogram1_sum' },
+            { labels: {}, value: 3, metricName: 'histogram1_count' },
+          ],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json)
+
+      expect(sortLines(influx)).to.be.equal(
+        sortLines(`prometheus,le=+Inf histogram1_bucket=3
+prometheus,le=50 histogram1_bucket=2
+prometheus,le=15 histogram1_bucket=2
+prometheus,le=5 histogram1_bucket=1
+prometheus histogram1_count=3,histogram1_sum=111`)
+      )
+    })
+
+    it('converts a histogram (from prometheus registry)', function() {
+      const register = new prometheus.Registry()
+      const histogram = new prometheus.Histogram({
+        name: 'histogram1',
+        help: 'histogram 1 help',
+        buckets: [5, 15, 50],
+        registers: [register],
+      })
+      histogram.observe(100)
+      histogram.observe(10)
+      histogram.observe(1)
+
+      const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+      expect(sortLines(influx)).to.be.equal(
+        sortLines(`prometheus,le=+Inf histogram1_bucket=3
+prometheus,le=50 histogram1_bucket=2
+prometheus,le=15 histogram1_bucket=2
+prometheus,le=5 histogram1_bucket=1
+prometheus histogram1_count=3,histogram1_sum=111`)
+      )
+    })
+
+    it('converts a summary', function() {
+      const json = [
+        {
+          name: 'summary1',
+          help: 'summary 1 help',
+          type: 'summary',
+          values: [
+            { labels: { quantile: 0.1 }, value: 1 },
+            { labels: { quantile: 0.9 }, value: 100 },
+            { labels: { quantile: 0.99 }, value: 100 },
+            { metricName: 'summary1_sum', labels: {}, value: 111 },
+            { metricName: 'summary1_count', labels: {}, value: 3 },
+          ],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json)
+
+      expect(sortLines(influx)).to.be.equal(
+        sortLines(`prometheus,quantile=0.99 summary1=100
+prometheus,quantile=0.9 summary1=100
+prometheus,quantile=0.1 summary1=1
+prometheus summary1_count=3,summary1_sum=111`)
+      )
+    })
+
+    it('converts a summary (from prometheus registry)', function() {
+      const register = new prometheus.Registry()
+      const summary = new prometheus.Summary({
+        name: 'summary1',
+        help: 'summary 1 help',
+        percentiles: [0.1, 0.9, 0.99],
+        registers: [register],
+      })
+      summary.observe(100)
+      summary.observe(10)
+      summary.observe(1)
+
+      const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+      expect(sortLines(influx)).to.be.equal(
+        sortLines(`prometheus,quantile=0.99 summary1=100
+prometheus,quantile=0.9 summary1=100
+prometheus,quantile=0.1 summary1=1
+prometheus summary1_count=3,summary1_sum=111`)
+      )
+    })
+
+    it('converts a counter and skip a timestamp', function() {
+      const json = [
+        {
+          help: 'counter 4 help',
+          name: 'counter4',
+          type: 'counter',
+          values: [{ value: 11, labels: {}, timestamp: 1581850552292 }],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json)
+
+      expect(influx).to.be.equal('prometheus counter4=11')
+    })
+
+    it('converts a counter and adds extra labels', function() {
+      const json = [
+        {
+          help: 'counter 1 help',
+          name: 'counter1',
+          type: 'counter',
+          values: [{ value: 11, labels: {} }],
+          aggregator: 'sum',
+        },
+      ]
+
+      const influx = promClientJsonToInfluxV2(json, {
+        instance: 'instance1',
+        env: 'production',
+      })
+
+      expect(influx).to.be.equal(
+        'prometheus,env=production,instance=instance1 counter1=11'
+      )
+    })
+  })
+})

core/server/prometheus-metrics.js

@@ -68,11 +68,13 @@ module.exports = class PrometheusMetrics {
         registers: [this.register],
       }),
     }
+    this.interval = prometheus.collectDefaultMetrics({
+      register: this.register,
+    })
   }
 
-  async initialize(server) {
+  async registerMetricsEndpoint(server) {
     const { register } = this
-    this.interval = prometheus.collectDefaultMetrics({ register })
 
     server.route(/^\/metrics$/, (data, match, end, ask) => {
       ask.res.setHeader('Content-Type', register.contentType)
@@ -88,6 +90,10 @@ module.exports = class PrometheusMetrics {
     }
   }
 
+  metrics() {
+    return this.register.getMetricsAsJSON()
+  }
+
   /**
    * @returns {object} `{ inc() {} }`.
    */

core/server/prometheus-metrics.spec.js

@@ -7,26 +7,26 @@ const got = require('../got-test-client')
 const Metrics = require('./prometheus-metrics')
 
 describe('Prometheus metrics route', function() {
-  let port, baseUrl
+  let port, baseUrl, camp, metrics
   beforeEach(async function() {
     port = await portfinder.getPortPromise()
     baseUrl = `http://127.0.0.1:${port}`
-  })
-
-  let camp
-  beforeEach(async function() {
     camp = Camp.start({ port, hostname: '::' })
     await new Promise(resolve => camp.on('listening', () => resolve()))
   })
   afterEach(async function() {
+    if (metrics) {
+      metrics.stop()
+    }
     if (camp) {
       await new Promise(resolve => camp.close(resolve))
       camp = undefined
     }
   })
 
-  it('returns metrics', async function() {
-    new Metrics({ enabled: true }).initialize(camp)
+  it('returns default metrics', async function() {
+    metrics = new Metrics()
+    metrics.registerMetricsEndpoint(camp)
 
     const { statusCode, body } = await got(`${baseUrl}/metrics`)
 

core/server/server.js

@@ -23,6 +23,7 @@ const { rasterRedirectUrl } = require('../badge-urls/make-badge-url')
 const log = require('./log')
 const sysMonitor = require('./monitor')
 const PrometheusMetrics = require('./prometheus-metrics')
+const InfluxMetrics = require('./influx-metrics')
 
 const Joi = originalJoi
   .extend(base => ({
@@ -81,6 +82,33 @@ const publicConfigSchema = Joi.object({
   metrics: {
     prometheus: {
       enabled: Joi.boolean().required(),
+      endpointEnabled: Joi.boolean().required(),
+    },
+    influx: {
+      enabled: Joi.boolean().required(),
+      url: Joi.string()
+        .uri()
+        .when('enabled', { is: true, then: Joi.required() }),
+      timeoutMilliseconds: Joi.number()
+        .integer()
+        .min(1)
+        .when('enabled', { is: true, then: Joi.required() }),
+      intervalSeconds: Joi.number()
+        .integer()
+        .min(1)
+        .when('enabled', { is: true, then: Joi.required() }),
+      instanceIdFrom: Joi.string()
+        .equal('hostname', 'env-var', 'random')
+        .when('enabled', { is: true, then: Joi.required() }),
+      instanceIdEnvVarName: Joi.string().when('instanceIdFrom', {
+        is: 'env-var',
+        then: Joi.required(),
+      }),
+      envLabel: Joi.string().when('enabled', {
+        is: true,
+        then: Joi.required(),
+      }),
+      hostnameAliases: Joi.object(),
     },
   },
   ssl: {
@@ -160,8 +188,13 @@ const privateConfigSchema = Joi.object({
   twitch_client_id: Joi.string(),
   twitch_client_secret: Joi.string(),
   wheelmap_token: Joi.string(),
+  influx_username: Joi.string(),
+  influx_password: Joi.string(),
 }).required()
-
+const privateMetricsInfluxConfigSchema = privateConfigSchema.append({
+  influx_username: Joi.string().required(),
+  influx_password: Joi.string().required(),
+})
 /**
  * The Server is based on the web framework Scoutcamp. It creates
  * an http server, sets up helpers for token persistence and monitoring.
@@ -173,22 +206,25 @@ class Server {
    * Badge Server Constructor
    *
    * @param {object} config Configuration object read from config yaml files
-   *    by https://www.npmjs.com/package/config and validated against
-   *    publicConfigSchema and privateConfigSchema
+   * by https://www.npmjs.com/package/config and validated against
+   * publicConfigSchema and privateConfigSchema
    * @see https://github.com/badges/shields/blob/master/doc/production-hosting.md#configuration
    * @see https://github.com/badges/shields/blob/master/doc/server-secrets.md
    */
   constructor(config) {
     const publicConfig = Joi.attempt(config.public, publicConfigSchema)
-    let privateConfig
-    try {
-      privateConfig = Joi.attempt(config.private, privateConfigSchema)
-    } catch (e) {
-      const badPaths = e.details.map(({ path }) => path)
-      throw Error(
-        `Private configuration is invalid. Check these paths: ${badPaths.join(
-          ','
-        )}`
+    const privateConfig = this.validatePrivateConfig(
+      config.private,
+      privateConfigSchema
+    )
+    // We want to require an username and a password for the influx metrics
+    // only if the influx metrics are enabled. The private config schema
+    // and the public config schema are two separate schemas so we have to run
+    // validation manually.
+    if (publicConfig.metrics.influx && publicConfig.metrics.influx.enabled) {
+      this.validatePrivateConfig(
+        config.private,
+        privateMetricsInfluxConfigSchema
       )
     }
     this.config = {
@@ -201,8 +237,31 @@ class Server {
       service: publicConfig.services.github,
       private: privateConfig,
     })
+
     if (publicConfig.metrics.prometheus.enabled) {
       this.metricInstance = new PrometheusMetrics()
+      if (publicConfig.metrics.influx.enabled) {
+        this.influxMetrics = new InfluxMetrics(
+          this.metricInstance,
+          Object.assign({}, publicConfig.metrics.influx, {
+            username: privateConfig.influx_username,
+            password: privateConfig.influx_password,
+          })
+        )
+      }
+    }
+  }
+
+  validatePrivateConfig(privateConfig, privateConfigSchema) {
+    try {
+      return Joi.attempt(privateConfig, privateConfigSchema)
+    } catch (e) {
+      const badPaths = e.details.map(({ path }) => path)
+      throw Error(
+        `Private configuration is invalid. Check these paths: ${badPaths.join(
+          ','
+        )}`
+      )
     }
   }
 
@@ -381,7 +440,12 @@ class Server {
     const { githubConstellation } = this
     githubConstellation.initialize(camp)
     if (metricInstance) {
-      metricInstance.initialize(camp)
+      if (this.config.public.metrics.prometheus.endpointEnabled) {
+        metricInstance.registerMetricsEndpoint(camp)
+      }
+      if (this.influxMetrics) {
+        this.influxMetrics.startPushingMetrics()
+      }
     }
 
     const { apiProvider: githubApiProvider } = this.githubConstellation
@@ -425,6 +489,9 @@ class Server {
     }
 
     if (this.metricInstance) {
+      if (this.influxMetrics) {
+        this.influxMetrics.stopPushingMetrics()
+      }
       this.metricInstance.stop()
     }
   }

core/server/server.spec.js

@@ -2,163 +2,333 @@
 
 const { expect } = require('chai')
 const isSvg = require('is-svg')
-const portfinder = require('portfinder')
+const config = require('config')
 const got = require('../got-test-client')
+const Server = require('./server')
 const { createTestServer } = require('./in-process-server-test-helpers')
 
 describe('The server', function() {
-  let server, baseUrl
-  before('Start the server', async function() {
-    // Fixes https://github.com/badges/shields/issues/2611
-    this.timeout(10000)
-    const port = await portfinder.getPortPromise()
-    server = createTestServer({ port })
-    baseUrl = server.baseUrl
-    await server.start()
-  })
-  after('Shut down the server', async function() {
-    if (server) {
-      await server.stop()
-    }
-    server = undefined
-  })
-
-  it('should allow strings for port', async function() {
-    // fixes #4391 - This allows the app to be run using iisnode, which uses a named pipe for the port.
-    const pipeServer = createTestServer({
-      port: '\\\\.\\pipe\\9c137306-7c4d-461e-b7cf-5213a3939ad6',
+  describe('running', function() {
+    let server, baseUrl
+    before('Start the server', async function() {
+      // Fixes https://github.com/badges/shields/issues/2611
+      this.timeout(10000)
+      server = await createTestServer()
+      baseUrl = server.baseUrl
+      await server.start()
+    })
+    after('Shut down the server', async function() {
+      if (server) {
+        await server.stop()
+      }
+      server = undefined
     })
-    expect(pipeServer).to.not.be.undefined
-  })
 
-  it('should produce colorscheme badges', async function() {
-    const { statusCode, body } = await got(`${baseUrl}:fruit-apple-green.svg`)
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('fruit')
-      .and.to.include('apple')
-  })
+    it('should allow strings for port', async function() {
+      // fixes #4391 - This allows the app to be run using iisnode, which uses a named pipe for the port.
+      const pipeServer = await createTestServer({
+        public: {
+          bind: {
+            port: '\\\\.\\pipe\\9c137306-7c4d-461e-b7cf-5213a3939ad6',
+          },
+        },
+      })
+      expect(pipeServer).to.not.be.undefined
+    })
 
-  it('should redirect colorscheme PNG badges as configured', async function() {
-    const { statusCode, headers } = await got(
-      `${baseUrl}:fruit-apple-green.png`,
-      {
+    it('should produce colorscheme badges', async function() {
+      const { statusCode, body } = await got(`${baseUrl}:fruit-apple-green.svg`)
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('fruit')
+        .and.to.include('apple')
+    })
+
+    it('should redirect colorscheme PNG badges as configured', async function() {
+      const { statusCode, headers } = await got(
+        `${baseUrl}:fruit-apple-green.png`,
+        {
+          followRedirect: false,
+        }
+      )
+      expect(statusCode).to.equal(301)
+      expect(headers.location).to.equal(
+        'http://raster.example.test/:fruit-apple-green.png'
+      )
+    })
+
+    it('should redirect modern PNG badges as configured', async function() {
+      const { statusCode, headers } = await got(`${baseUrl}npm/v/express.png`, {
         followRedirect: false,
-      }
-    )
-    expect(statusCode).to.equal(301)
-    expect(headers.location).to.equal(
-      'http://raster.example.test/:fruit-apple-green.png'
-    )
-  })
-
-  it('should redirect modern PNG badges as configured', async function() {
-    const { statusCode, headers } = await got(`${baseUrl}npm/v/express.png`, {
-      followRedirect: false,
+      })
+      expect(statusCode).to.equal(301)
+      expect(headers.location).to.equal(
+        'http://raster.example.test/npm/v/express.png'
+      )
     })
-    expect(statusCode).to.equal(301)
-    expect(headers.location).to.equal(
-      'http://raster.example.test/npm/v/express.png'
-    )
-  })
 
-  it('should produce json badges', async function() {
-    const { statusCode, body, headers } = await got(
-      `${baseUrl}twitter/follow/_Pyves.json`
-    )
-    expect(statusCode).to.equal(200)
-    expect(headers['content-type']).to.equal('application/json')
-    expect(() => JSON.parse(body)).not.to.throw()
-  })
+    it('should produce json badges', async function() {
+      const { statusCode, body, headers } = await got(
+        `${baseUrl}twitter/follow/_Pyves.json`
+      )
+      expect(statusCode).to.equal(200)
+      expect(headers['content-type']).to.equal('application/json')
+      expect(() => JSON.parse(body)).not.to.throw()
+    })
 
-  it('should preserve label case', async function() {
-    const { statusCode, body } = await got(`${baseUrl}:fRuiT-apple-green.svg`)
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('fRuiT')
-  })
+    it('should preserve label case', async function() {
+      const { statusCode, body } = await got(`${baseUrl}:fRuiT-apple-green.svg`)
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('fRuiT')
+    })
 
-  // https://github.com/badges/shields/pull/1319
-  it('should not crash with a numeric logo', async function() {
-    const { statusCode, body } = await got(
-      `${baseUrl}:fruit-apple-green.svg?logo=1`
-    )
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('fruit')
-      .and.to.include('apple')
-  })
+    // https://github.com/badges/shields/pull/1319
+    it('should not crash with a numeric logo', async function() {
+      const { statusCode, body } = await got(
+        `${baseUrl}:fruit-apple-green.svg?logo=1`
+      )
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('fruit')
+        .and.to.include('apple')
+    })
 
-  it('should not crash with a numeric link', async function() {
-    const { statusCode, body } = await got(
-      `${baseUrl}:fruit-apple-green.svg?link=1`
-    )
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('fruit')
-      .and.to.include('apple')
-  })
+    it('should not crash with a numeric link', async function() {
+      const { statusCode, body } = await got(
+        `${baseUrl}:fruit-apple-green.svg?link=1`
+      )
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('fruit')
+        .and.to.include('apple')
+    })
 
-  it('should not crash with a boolean link', async function() {
-    const { statusCode, body } = await got(
-      `${baseUrl}:fruit-apple-green.svg?link=true`
-    )
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('fruit')
-      .and.to.include('apple')
-  })
+    it('should not crash with a boolean link', async function() {
+      const { statusCode, body } = await got(
+        `${baseUrl}:fruit-apple-green.svg?link=true`
+      )
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('fruit')
+        .and.to.include('apple')
+    })
 
-  it('should return the 404 badge for unknown badges', async function() {
-    const { statusCode, body } = await got(
-      `${baseUrl}this/is/not/a/badge.svg`,
-      { throwHttpErrors: false }
-    )
-    expect(statusCode).to.equal(404)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('404')
-      .and.to.include('badge not found')
-  })
+    it('should return the 404 badge for unknown badges', async function() {
+      const { statusCode, body } = await got(
+        `${baseUrl}this/is/not/a/badge.svg`,
+        {
+          throwHttpErrors: false,
+        }
+      )
+      expect(statusCode).to.equal(404)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('404')
+        .and.to.include('badge not found')
+    })
 
-  it('should return the 404 badge page for rando links', async function() {
-    const { statusCode, body } = await got(
-      `${baseUrl}this/is/most/definitely/not/a/badge.js`,
-      {
+    it('should return the 404 badge page for rando links', async function() {
+      const { statusCode, body } = await got(
+        `${baseUrl}this/is/most/definitely/not/a/badge.js`,
+        {
+          throwHttpErrors: false,
+        }
+      )
+      expect(statusCode).to.equal(404)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('404')
+        .and.to.include('badge not found')
+    })
+
+    it('should redirect the root as configured', async function() {
+      const { statusCode, headers } = await got(baseUrl, {
+        followRedirect: false,
+      })
+
+      expect(statusCode).to.equal(302)
+      // This value is set in `config/test.yml`
+      expect(headers.location).to.equal('http://frontend.example.test')
+    })
+
+    it('should return the 410 badge for obsolete formats', async function() {
+      const { statusCode, body } = await got(`${baseUrl}npm/v/express.jpg`, {
         throwHttpErrors: false,
+      })
+      // TODO It would be nice if this were 404 or 410.
+      expect(statusCode).to.equal(200)
+      expect(body)
+        .to.satisfy(isSvg)
+        .and.to.include('410')
+        .and.to.include('jpg no longer available')
+    })
+  })
+
+  describe('configuration', function() {
+    let server
+    afterEach(async function() {
+      if (server) {
+        server.stop()
       }
-    )
-    expect(statusCode).to.equal(404)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('404')
-      .and.to.include('badge not found')
-  })
-
-  it('should redirect the root as configured', async function() {
-    const { statusCode, headers } = await got(baseUrl, {
-      followRedirect: false,
     })
 
-    expect(statusCode).to.equal(302)
-    // This value is set in `config/test.yml`
-    expect(headers.location).to.equal('http://frontend.example.test')
+    it('should allow to enable prometheus metrics', async function() {
+      // Fixes https://github.com/badges/shields/issues/2611
+      this.timeout(10000)
+      server = await createTestServer({
+        public: {
+          metrics: { prometheus: { enabled: true, endpointEnabled: true } },
+        },
+      })
+      await server.start()
+
+      const { statusCode } = await got(`${server.baseUrl}metrics`)
+
+      expect(statusCode).to.be.equal(200)
+    })
+
+    it('should allow to disable prometheus metrics', async function() {
+      // Fixes https://github.com/badges/shields/issues/2611
+      this.timeout(10000)
+      server = await createTestServer({
+        public: {
+          metrics: { prometheus: { enabled: true, endpointEnabled: false } },
+        },
+      })
+      await server.start()
+
+      const { statusCode } = await got(`${server.baseUrl}metrics`, {
+        throwHttpErrors: false,
+      })
+
+      expect(statusCode).to.be.equal(404)
+    })
   })
 
-  it('should return the 410 badge for obsolete formats', async function() {
-    const { statusCode, body } = await got(`${baseUrl}npm/v/express.jpg`, {
-      throwHttpErrors: false,
+  describe('configuration validation', function() {
+    describe('influx', function() {
+      let customConfig
+      beforeEach(function() {
+        customConfig = config.util.toObject()
+        customConfig.public.metrics.influx = {
+          enabled: true,
+          url: 'http://localhost:8081/telegraf',
+          timeoutMilliseconds: 1000,
+          intervalSeconds: 2,
+          instanceIdFrom: 'random',
+          instanceIdEnvVarName: 'INSTANCE_ID',
+          hostnameAliases: { 'metrics-hostname': 'metrics-hostname-alias' },
+          envLabel: 'test-env',
+        }
+        customConfig.private = {
+          influx_username: 'telegraf',
+          influx_password: 'telegrafpass',
+        }
+      })
+
+      it('should not require influx configuration', function() {
+        delete customConfig.public.metrics.influx
+        expect(() => new Server(config.util.toObject())).to.not.throw()
+      })
+
+      it('should require url when influx configuration is enabled', function() {
+        delete customConfig.public.metrics.influx.url
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.url" is required'
+        )
+      })
+
+      it('should not require url when influx configuration is disabled', function() {
+        customConfig.public.metrics.influx.enabled = false
+        delete customConfig.public.metrics.influx.url
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should require timeoutMilliseconds when influx configuration is enabled', function() {
+        delete customConfig.public.metrics.influx.timeoutMilliseconds
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.timeoutMilliseconds" is required'
+        )
+      })
+
+      it('should require intervalSeconds when influx configuration is enabled', function() {
+        delete customConfig.public.metrics.influx.intervalSeconds
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.intervalSeconds" is required'
+        )
+      })
+
+      it('should require instanceIdFrom when influx configuration is enabled', function() {
+        delete customConfig.public.metrics.influx.instanceIdFrom
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.instanceIdFrom" is required'
+        )
+      })
+
+      it('should require instanceIdEnvVarName when instanceIdFrom is env-var', function() {
+        customConfig.public.metrics.influx.instanceIdFrom = 'env-var'
+        delete customConfig.public.metrics.influx.instanceIdEnvVarName
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.instanceIdEnvVarName" is required'
+        )
+      })
+
+      it('should allow instanceIdFrom = hostname', function() {
+        customConfig.public.metrics.influx.instanceIdFrom = 'hostname'
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should allow instanceIdFrom = env-var', function() {
+        customConfig.public.metrics.influx.instanceIdFrom = 'env-var'
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should allow instanceIdFrom = random', function() {
+        customConfig.public.metrics.influx.instanceIdFrom = 'random'
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should require envLabel when influx configuration is enabled', function() {
+        delete customConfig.public.metrics.influx.envLabel
+        expect(() => new Server(customConfig)).to.throw(
+          '"metrics.influx.envLabel" is required'
+        )
+      })
+
+      it('should not require hostnameAliases', function() {
+        delete customConfig.public.metrics.influx.hostnameAliases
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should allow empty hostnameAliases', function() {
+        customConfig.public.metrics.influx.hostnameAliases = {}
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
+
+      it('should require username when influx configuration is enabled', function() {
+        delete customConfig.private.influx_username
+        expect(() => new Server(customConfig)).to.throw(
+          'Private configuration is invalid. Check these paths: influx_username'
+        )
+      })
+
+      it('should require password when influx configuration is enabled', function() {
+        delete customConfig.private.influx_password
+        expect(() => new Server(customConfig)).to.throw(
+          'Private configuration is invalid. Check these paths: influx_password'
+        )
+      })
+
+      it('should allow other private keys', function() {
+        customConfig.private.gh_token = 'my-token'
+        expect(() => new Server(customConfig)).to.not.throw()
+      })
     })
-    // TODO It would be nice if this were 404 or 410.
-    expect(statusCode).to.equal(200)
-    expect(body)
-      .to.satisfy(isSvg)
-      .and.to.include('410')
-      .and.to.include('jpg no longer available')
   })
 })