Require default values

config/custom-environment-variables.yml

@@ -62,7 +62,9 @@ public:
 
   rateLimit: 'RATE_LIMIT'
 
-  fetchLimit: 'FETCH_LIMIT'
+  integrations:
+    default:
+      fetchLimit: 'FETCH_LIMIT'
 
 private:
   azure_devops_token: 'AZURE_DEVOPS_TOKEN'

config/default.yml

@@ -34,6 +34,17 @@ public:
 
   handleInternalErrors: true
 
-  fetchLimit: '10MB'
+  integrations:
+    default:
+      fetchLimit: '10MB'
+
+    DynamicJson:
+      fetchLimit: '2MB'
+
+    DynamicXml:
+      fetchLimit: '128KB'
+
+    DynamicYaml:
+      fetchLimit: '64KB'
 
 private: {}

core/base-service/base.js

@@ -3,6 +3,7 @@
  * @module
  */
 
+const bytes = require('bytes')
 // See available emoji at http://emoji.muan.co/
 const emojic = require('emojic')
 const Joi = require('@hapi/joi')
@@ -425,7 +426,8 @@ class BaseService {
     { camp, handleRequest, githubApiProvider, metricInstance },
     serviceConfig
   ) {
-    const { cacheHeaders: cacheHeaderConfig, fetchLimitBytes } = serviceConfig
+    const { cacheHeaders: cacheHeaderConfig, fetchLimit } = serviceConfig
+    const fetchLimitBytes = bytes.parse(fetchLimit)
     const { regex, captureNames } = prepareRoute(this.route)
     const queryParams = getQueryParamNames(this.route)
 

core/server/config.js

@@ -0,0 +1,38 @@
+'use strict'
+const deepmerge = require('deepmerge')
+
+class RedundantCustomConfiguration extends Error {
+  constructor(message) {
+    super(message)
+    this.name = 'RedundantCustomConfiguration'
+  }
+}
+
+function merge(_default, custom) {
+  // Overwrites the existing array values completely rather than concatenating them
+  // recipe from https://github.com/TehShrike/deepmerge#arraymerge
+  const overwriteMerge = (destinationArray, sourceArray, options) => sourceArray
+  return deepmerge(_default, custom, {
+    arrayMerge: overwriteMerge,
+  })
+}
+
+function checkCustomIntegrationConfiguration(config, serviceClasses) {
+  const serviceNames = new Set(
+    serviceClasses.map(serviceClass => serviceClass.name)
+  )
+  const redundantConfigurations = Object.keys(config.public.integrations)
+    .filter(configName => configName !== 'default')
+    .filter(configName => !serviceNames.has(configName))
+  if (redundantConfigurations.length) {
+    throw new RedundantCustomConfiguration(
+      `Custom configurations found without a corresponding service: ${redundantConfigurations}`
+    )
+  }
+}
+
+module.exports = {
+  RedundantCustomConfiguration,
+  merge,
+  checkCustomIntegrationConfiguration,
+}

core/server/config.spec.js

@@ -0,0 +1,61 @@
+'use strict'
+
+const { test, given } = require('sazerac')
+const { expect } = require('chai')
+const {
+  RedundantCustomConfiguration,
+  merge,
+  checkCustomIntegrationConfiguration,
+} = require('./config')
+
+describe('configuration', function() {
+  test(merge, function() {
+    given({ a: 2 }, {})
+      .describe('copies the default value')
+      .expect({ a: 2 })
+    given({ a: 2 }, { a: 3 })
+      .describe('overrides a primitive value')
+      .expect({ a: 3 })
+    given({ a: { a1: 1, a2: 2 } }, { a: { a1: 2 } })
+      .describe('merges objects')
+      .expect({ a: { a1: 2, a2: 2 } })
+    given({ a: { a1: 1, a2: 2 } }, { a: 3 })
+      .describe('overrides an object with a primitive')
+      .expect({ a: 3 })
+    given({ a: { a1: 1, a2: 2 } }, { a: {} })
+      .describe('does not override an object with an empty object')
+      .expect({ a: { a1: 1, a2: 2 } })
+    given({ a: [2, 3, 4] }, { a: [5, 6] })
+      .describe('overrides array')
+      .expect({ a: [5, 6] })
+  })
+
+  describe('checkCustomIntegrationConfiguration function', function() {
+    it('accepts the default configuration', function() {
+      const config = { public: { integrations: { default: {} } } }
+      const serviceClasses = [{ name: 'SomeService' }]
+
+      expect(() =>
+        checkCustomIntegrationConfiguration(config, serviceClasses)
+      ).not.throw()
+    })
+
+    it('accepts a configuration for an existing service', function() {
+      const config = { public: { integrations: { SomeService: {} } } }
+      const serviceClasses = [{ name: 'SomeService' }]
+
+      expect(() =>
+        checkCustomIntegrationConfiguration(config, serviceClasses)
+      ).not.throw()
+    })
+
+    it('throws an error if a custom config does not have a corresponding service', function() {
+      const config = { public: { integrations: { UnknownService: {} } } }
+      const serviceClasses = [{ name: 'KnownService' }]
+
+      expect(() =>
+        checkCustomIntegrationConfiguration(config, serviceClasses)
+      ).to.throw(RedundantCustomConfiguration)
+    })
+  })
+})

core/server/server.js

@@ -6,7 +6,6 @@
 const path = require('path')
 const url = require('url')
 const { URL } = url
-const bytes = require('bytes')
 const Camp = require('camp')
 const originalJoi = require('@hapi/joi')
 const makeBadge = require('../../gh-badges/lib/make-badge')
@@ -20,6 +19,7 @@ const {
 } = require('../base-service/legacy-request-handler')
 const { clearRegularUpdateCache } = require('../legacy/regular-update')
 const { rasterRedirectUrl } = require('../badge-urls/make-badge-url')
+const { merge, checkCustomIntegrationConfiguration } = require('./config')
 const log = require('./log')
 const sysMonitor = require('./monitor')
 const PrometheusMetrics = require('./prometheus-metrics')
@@ -59,6 +59,13 @@ const Joi = originalJoi
 
 const optionalUrl = Joi.string().uri({ scheme: ['http', 'https'] })
 const requiredUrl = optionalUrl.required()
+const bytes = Joi.string().regex(/^[0-9]+(b|kb|mb|gb|tb)$/i)
+const requireFields = {
+  required: schema => schema.required(),
+}
+const integrationSchema = Joi.object({
+  fetchLimit: bytes.alter(requireFields),
+})
 const origins = Joi.arrayFromString().items(Joi.string().origin())
 const defaultService = Joi.object({ authorizedOrigins: origins }).default({
   authorizedOrigins: [],
@@ -158,7 +165,9 @@ const publicConfigSchema = Joi.object({
   },
   rateLimit: Joi.boolean().required(),
   handleInternalErrors: Joi.boolean().required(),
-  fetchLimit: Joi.string().regex(/^[0-9]+(b|kb|mb|gb|tb)$/i),
+  integrations: Joi.object({
+    default: integrationSchema.tailor('required').required(),
+  }).pattern(Joi.string(), integrationSchema),
 }).required()
 
 const privateConfigSchema = Joi.object({
@@ -391,19 +400,25 @@ class Server {
     const { config, camp, metricInstance } = this
     const { apiProvider: githubApiProvider } = this.githubConstellation
 
-    loadServiceClasses().forEach(serviceClass =>
+    const serviceClasses = loadServiceClasses()
+    checkCustomIntegrationConfiguration(config, serviceClasses)
+    serviceClasses.forEach(serviceClass => {
+      const serviceConfig = merge(
+        config.public.integrations.default,
+        config.public.integrations[serviceClass.name] || {}
+      )
       serviceClass.register(
         { camp, handleRequest, githubApiProvider, metricInstance },
         {
           handleInternalErrors: config.public.handleInternalErrors,
           cacheHeaders: config.public.cacheHeaders,
-          fetchLimitBytes: bytes(config.public.fetchLimit),
           rasterUrl: config.public.rasterUrl,
           private: config.private,
           public: config.public,
+          ...serviceConfig,
         }
       )
-    )
+    })
   }
 
   /**

package-lock.json

@@ -9908,8 +9908,7 @@
     "deepmerge": {
       "version": "4.2.2",
       "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
-      "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
+      "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg=="
-      "dev": true
     },
     "default-gateway": {
       "version": "4.2.0",

package.json

@@ -34,6 +34,7 @@
     "config": "^3.3.1",
     "cross-env": "^6.0.3",
     "decamelize": "^3.2.0",
+    "deepmerge": "^4.2.2",
     "dotenv": "^8.2.0",
     "emojic": "^1.1.15",
     "escape-string-regexp": "^2.0.0",

services/dynamic/dynamic-xml.tester.js

@@ -153,7 +153,7 @@ t.create('XPath parse error')
 
 t.create('XML from url | invalid url')
   .get(
-    '.json?url=https://github.com/badges/shields/raw/master/notafile.xml&query=//version'
+    '.json?url=https://raw.githubusercontent.com/badges/shields/master/notafile.xml&query=//version'
   )
   .expectBadge({
     label: 'custom badge',