docs: start collating info on integration patterns

Co-authored-by: repo-ranger[bot] <39074581+repo-ranger[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -149,8 +149,6 @@ jobs:
   main@node-17:
     docker:
       - image: cimg/node:17.9
-    environment:
-      NPM_CONFIG_ENGINE_STRICT: 'false'
 
     <<: *main_steps
 
@@ -165,8 +163,6 @@ jobs:
     docker:
       - image: cimg/node:17.9
       - image: redis
-    environment:
-      NPM_CONFIG_ENGINE_STRICT: 'false'
 
     <<: *integration_steps
 
@@ -243,8 +239,6 @@ jobs:
   services@node-17:
     docker:
       - image: cimg/node:17.9
-    environment:
-      NPM_CONFIG_ENGINE_STRICT: 'false'
 
     <<: *services_steps
 

app.json

@@ -35,16 +35,6 @@
     "WEBLATE_API_KEY": {
       "description": "Configure the API key to be used for the Weblate service.",
       "required": false
-    },
-    "METRICS_INFLUX_ENABLED": {
-      "description": "Disable influx metrics",
-      "value": "0",
-      "required": false
-    },
-    "REQUIRE_CLOUDFLARE": {
-      "description": "Allow direct traffic",
-      "value": "0",
-      "required": false
     }
   },
   "formation": {

doc/service-integrations.md

@@ -0,0 +1,75 @@
+# Integration with upstream services
+
+## Overview
+
+In a nutshell, the Shields Badge Server handles the responsibilities of accepting requests for badges, and then serving those badges back to users.
+
+A grossly oversimplified visualization would probably look something like this:
+
+```mermaid
+sequenceDiagram
+    actor User
+    participant B as Badge Server
+    participant P as Data Provider
+    User->>+B: I'd like a badge for my project please!
+    B->>+P: Get data for project
+    P-->>-B: Data
+    B->>B: Make badge with data
+    B-->>-User: Here's your badge!
+```
+
+Shields is not a system of record (we're not the package registry, pipeline tool, etc.) so when Shields receives a request for a badge, the badge server will first have to reach out to the system of record in order to get the data points it needs to create your badge.
+
+For example, if you ask Shields for a build status badge for your CircleCI pipeline, then Shields has to reach out to CircleCI to figure out what the status of your pipeline is (CircleCI would be the "Data Provider" actor in the above diagram). Similarly if you want a badge that shows the count of downloads of your npm package, Shields has to reach out to the npm.js registry.
+
+That covers the gist, but the actual story is a bit more involved and complicated than that of course. There's a number of other components along the way, ranging from the browser on your local machine to extra services and actors we have deployed as part of the Shields.io runtime ecosystem which help ensure we can provide a stable and reliable deployment of the badge server-as-a-service. Additionally, badges rendered in GitHub have some additional factors at play that impose some additional constraints, detailed in the next section.
+
+### GitHub Badge Rendering
+
+A common usage pattern for badges is to embed them in your project's README files so relevant information is conveyed to the project's users. This means badges are often utilized and rendered in source control management platforms, like GitHub.
+
+[GitHub utilizes a proxy service, called camo][camo], for handling the images that you see when you browse project pages on github.com, and this is utilized for badges too (both svg and png formats). GitHub does this for a number of reasons, including to anonymize requests and protect your privacy. However, this also requires the upstream images (including badges) to be returned quickly in order for those images to show up on your screen, with a rough ceiling of 3-4 seconds. If the upstream image provider is too slow to respond, then camo will timeout and the image won't be displayed.
+
+This imposes constraints on Shields, as we need to ensure that the badge server completes the entire request/response workflow and returns the final badge within a few seconds.
+
+[camo]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-anonymized-urls
+
+## Interaction patterns
+
+Because of the time limits around the full badge flow discussed above, the badge server is somewhat constrained in terms of patterns it can employ to fetch data from the upstream provider. When we incorporate new badges we typically need to ensure when we receive a badge request and need to get data to serve that badge request, that we can get that data by making a single call to the upstream provider to get the data points needed for the badge.
+
+The nature of the call Shields makes to upstream data providers is obviously dependent upon the nature of the upstream endpoints. The overwhelming majority of the time this occurs via a GET request, though there are a few occasions where the upstream endpoint requires a POST in order for us to retrieve data, and a few others where we issue a HEAD request because the data points we need for the badge actually reside in the response headers.
+
+There are a couple other exceptions, but as a general pattern we strive to integrate with services via a stateless, single call manner.
+
+### Authentication
+
+Shields typically integrates with upstream data providers anonymously, largely because the data targets we need are anonymously available (open source packages, repositories, pipelines, etc.)
+
+The badge server can be configured to make authenticated requests for certain supported services. This exists so that users who are [self-hosting] their own instance of the badge server can get badges for their private content, and also as part of agreements we've made with certain upstream data providers for the main Shields.io deployment.
+
+[self-hosting]: ./self-hosting.md
+
+### Rate Limits
+
+Many upstream data providers employ common techniques to protect the availability and integrity of their service, and one common technique is [rate limiting].
+
+Typically, when clients/consumers of rate limited endpoints will employ client-side techniques to avoid running afoul of those limits and potentially getting blocked or having their requests throttled. Unfortunately, those techniques aren't really viable for the Shields.io environment due to the workflow and constraints discussed above.
+
+As such, we instead try to ensure that Shields.io never makes more calls to an upstream provider than their rate limits allow.
+
+In cases where Shields.io may run close to or exceed those limits, we typically consider:
+
+- increasing the cache periods we set (to reduce the number of badge requests we receive)
+- collaborating with the vendor or maintainers of the upstream data provider to explore options for an increased rate limit for Shields.io
+- decline to provide badges for that upstream data provider
+
+[rate limiting]: https://en.wikipedia.org/wiki/Rate_limiting
+
+### Denial of Service
+
+coming soon...
+
+### Considerations for new upstream integrations
+
+coming soon...

package.json

@@ -241,7 +241,7 @@
     "url": "^0.11.0"
   },
   "engines": {
-    "node": "^16.13.0",
+    "node": ">=16.13.0",
     "npm": ">=8.0.0"
   },
   "type": "module",

services/conda/conda-version.service.js

@@ -31,7 +31,7 @@ export default class CondaVersion extends BaseCondaService {
 
   static render({ variant, channel, version }) {
     return {
-      label: variant === 'vn' ? channel : `conda|${channel}`,
+      label: variant === 'vn' ? channel : `conda | ${channel}`,
       message: versionText(version),
       color: versionColor(version),
     }

services/conda/conda-version.tester.js

@@ -3,7 +3,7 @@ import { createServiceTester } from '../tester.js'
 export const t = await createServiceTester()
 
 t.create('version').get('/v/conda-forge/zlib.json').expectBadge({
-  label: 'conda|conda-forge',
+  label: 'conda | conda-forge',
   message: isVPlusTripleDottedVersion,
 })
 

services/ros/ros-version.service.js

@@ -0,0 +1,154 @@
+import gql from 'graphql-tag'
+import Joi from 'joi'
+import yaml from 'js-yaml'
+import { renderVersionBadge } from '../version.js'
+import { GithubAuthV4Service } from '../github/github-auth-service.js'
+import { NotFound, InvalidResponse } from '../index.js'
+
+const tagsSchema = Joi.object({
+  data: Joi.object({
+    repository: Joi.object({
+      refs: Joi.object({
+        edges: Joi.array()
+          .items({
+            node: Joi.object({
+              name: Joi.string().required(),
+            }).required(),
+          })
+          .required(),
+      }).required(),
+    }).required(),
+  }).required(),
+}).required()
+
+const contentSchema = Joi.object({
+  data: Joi.object({
+    repository: Joi.object({
+      object: Joi.object({
+        text: Joi.string().required(),
+      }).allow(null),
+    }).required(),
+  }).required(),
+}).required()
+
+const distroSchema = Joi.object({
+  repositories: Joi.object().required(),
+})
+const packageSchema = Joi.object({
+  release: Joi.object({
+    version: Joi.string().required(),
+  }).required(),
+})
+
+export default class RosVersion extends GithubAuthV4Service {
+  static category = 'version'
+
+  static route = { base: 'ros/v', pattern: ':distro/:packageName' }
+
+  static examples = [
+    {
+      title: 'ROS Package Index',
+      namedParams: { distro: 'humble', packageName: 'vision_msgs' },
+      staticPreview: {
+        ...renderVersionBadge({ version: '4.0.0' }),
+        label: 'ros | humble',
+      },
+    },
+  ]
+
+  static defaultBadgeData = { label: 'ros' }
+
+  async handle({ distro, packageName }) {
+    const tagsJson = await this._requestGraphql({
+      query: gql`
+        query ($refPrefix: String!) {
+          repository(owner: "ros", name: "rosdistro") {
+            refs(
+              refPrefix: $refPrefix
+              first: 30
+              orderBy: { field: TAG_COMMIT_DATE, direction: DESC }
+            ) {
+              edges {
+                node {
+                  name
+                }
+              }
+            }
+          }
+        }
+      `,
+      variables: { refPrefix: `refs/tags/${distro}/` },
+      schema: tagsSchema,
+    })
+
+    // Filter for tags that look like dates: humble/2022-06-10
+    const tags = tagsJson.data.repository.refs.edges
+      .map(edge => edge.node.name)
+      .filter(tag => /^\d+-\d+-\d+$/.test(tag))
+      .sort()
+      .reverse()
+
+    const ref = tags[0] ? `refs/tags/${distro}/${tags[0]}` : 'refs/heads/master'
+    const prettyRef = tags[0] ? `${distro}/${tags[0]}` : 'master'
+
+    const contentJson = await this._requestGraphql({
+      query: gql`
+        query ($expression: String!) {
+          repository(owner: "ros", name: "rosdistro") {
+            object(expression: $expression) {
+              ... on Blob {
+                text
+              }
+            }
+          }
+        }
+      `,
+      variables: {
+        expression: `${ref}:${distro}/distribution.yaml`,
+      },
+      schema: contentSchema,
+    })
+
+    if (!contentJson.data.repository.object) {
+      throw new NotFound({
+        prettyMessage: `distribution.yaml not found: ${distro}@${prettyRef}`,
+      })
+    }
+    const version = this.constructor._parseReleaseVersionFromDistro(
+      contentJson.data.repository.object.text,
+      packageName
+    )
+
+    return { ...renderVersionBadge({ version }), label: `ros | ${distro}` }
+  }
+
+  static _parseReleaseVersionFromDistro(distroYaml, packageName) {
+    let distro
+    try {
+      distro = yaml.load(distroYaml)
+    } catch (err) {
+      throw new InvalidResponse({
+        prettyMessage: 'unparseable distribution.yml',
+        underlyingError: err,
+      })
+    }
+
+    const validatedDistro = this._validate(distro, distroSchema, {
+      prettyErrorMessage: 'invalid distribution.yml',
+    })
+    if (!validatedDistro.repositories[packageName]) {
+      throw new NotFound({ prettyMessage: `package not found: ${packageName}` })
+    }
+
+    const packageInfo = this._validate(
+      validatedDistro.repositories[packageName],
+      packageSchema,
+      {
+        prettyErrorMessage: `invalid section for ${packageName} in distribution.yml`,
+      }
+    )
+
+    // Strip off "release inc" suffix
+    return packageInfo.release.version.replace(/-\d+$/, '')
+  }
+}

services/ros/ros-version.service.spec.js

@@ -0,0 +1,44 @@
+import { expect } from 'chai'
+import RosVersion from './ros-version.service.js'
+
+describe('parseReleaseVersionFromDistro', function () {
+  it('returns correct version', function () {
+    expect(
+      RosVersion._parseReleaseVersionFromDistro(
+        `
+%YAML 1.1
+# ROS distribution file
+# see REP 143: http://ros.org/reps/rep-0143.html
+---
+release_platforms:
+  debian:
+  - bullseye
+  rhel:
+  - '8'
+  ubuntu:
+  - jammy
+repositories:
+  vision_msgs:
+    doc:
+      type: git
+      url: https://github.com/ros-perception/vision_msgs.git
+      version: ros2
+    release:
+      tags:
+        release: release/humble/{package}/{version}
+      url: https://github.com/ros2-gbp/vision_msgs-release.git
+      version: 4.0.0-2
+    source:
+      test_pull_requests: true
+      type: git
+      url: https://github.com/ros-perception/vision_msgs.git
+      version: ros2
+    status: developed
+type: distribution
+version: 2
+      `,
+        'vision_msgs'
+      )
+    ).to.equal('4.0.0')
+  })
+})

services/ros/ros-version.tester.js

@@ -0,0 +1,28 @@
+import { isSemver } from '../test-validators.js'
+import { createServiceTester } from '../tester.js'
+
+export const t = await createServiceTester()
+
+t.create('gets the package version of vision_msgs in active distro')
+  .get('/humble/vision_msgs.json')
+  .expectBadge({ label: 'ros | humble', message: isSemver })
+
+t.create('gets the package version of vision_msgs in EOL distro')
+  .get('/lunar/vision_msgs.json')
+  .expectBadge({ label: 'ros | lunar', message: isSemver })
+
+t.create('returns not found for invalid package')
+  .get('/humble/this package does not exist - ros test.json')
+  .expectBadge({
+    label: 'ros',
+    color: 'red',
+    message: 'package not found: this package does not exist - ros test',
+  })
+
+t.create('returns error for invalid distro')
+  .get('/xxxxxx/vision_msgs.json')
+  .expectBadge({
+    label: 'ros',
+    color: 'red',
+    message: 'distribution.yaml not found: xxxxxx@master',
+  })