Service definition export format (#2397)

Three main goals:

1. In the front end:
      a. Show form fields and automatically assemble badge URLs (#701)
      c. Group together examples for the same service
      b. Show deprecated services
2. Make it easy to changing the schema of `examples`, thanks to 100% validation. One challenge with frameworks is that when there are typos things fail silently which is pretty unfriendly to developers. The validation should really help with that. (This caught one bug in AUR, though I fixed it in #2405 which landed first.)
3. Produce a service definition export for external tool builders. (#776)
4. Build toward harmony between the front-end data structure and the `examples` key in the service classes. I aliased `staticPreview` to `staticExample` which starts this process.

The old format:

- Lacked a consistent machine-readable representation of the fields.
- Flattened multiple examples for the same service were flattened.
- Excluded deprecated services.

The new format improves a few things, too:

- It cleans up the naming. Since this file evolved over time, the names were a bit muddled (i.e. what was an example vs a preview).
- It duplicated information (like `.svg`). (I can imagine dropping the `.svg` from our badge URLs someday, which would make the URLs easier to read and maintain.)
- For a human reading the YAML file, providing the static example as a deconstructed object is more readable.

Here are a couple snippets:

```yml
  - category: build
    name: AppVeyorCi
    isDeprecated: false
    route:
      format: '([^/]+/[^/]+)(?:/(.+))?'
      queryParams: []
    examples:
      - title: AppVeyor
        example: {path: /appveyor/ci/gruntjs/grunt, queryParams: {}}
        preview: {label: build, message: passing, color: brightgreen}
        keywords: []
      - title: AppVeyor branch
        example: {path: /appveyor/ci/gruntjs/grunt/master, queryParams: {}}
        preview: {label: build, message: passing, color: brightgreen}
        keywords: []
  - category: downloads
    name: AmoDownloads
    isDeprecated: false
    examples:
      - title: Mozilla Add-on
        example: {path: /amo/d/dustman, queryParams: {}}
        preview: {path: /amo/d/dustman, queryParams: {}}
        keywords: [amo, firefox]
```
This commit is contained in:
Paul Melnikow
2018-12-02 11:21:30 -05:00
committed by GitHub
parent 658086bc46
commit 2045489019
25 changed files with 517 additions and 10 deletions

1
.gitignore vendored
View File

@@ -92,3 +92,4 @@ typings/
.next
badge-examples.json
supported-features.json
service-definitions.yml

View File

@@ -85,13 +85,14 @@
"postinstall": "npm run depcheck",
"prebuild": "npm run depcheck",
"features": "node scripts/export-supported-features-cli.js > supported-features.json",
"defs": "node scripts/export-service-definitions-cli.js > service-definitions.yml",
"examples": "node scripts/export-badge-examples-cli.js > badge-examples.json",
"build": "npm run examples && npm run features && next build && next export -o build/",
"build": "npm run examples && npm run defs && npm run features && next build && next export -o build/",
"heroku-postbuild": "npm run build",
"analyze": "ANALYZE=true LONG_CACHE=false BASE_URL=https://img.shields.io npm run build",
"start:server": "HANDLE_INTERNAL_ERRORS=false RATE_LIMIT=false node server 8080 ::",
"now-start": "node server",
"prestart": "npm run depcheck && npm run examples && npm run features",
"prestart": "npm run depcheck && npm run examples && npm run defs && npm run features",
"start": "concurrently --names server,frontend \"ALLOWED_ORIGIN=http://localhost:3000 npm run start:server\" \"BASE_URL=http://[::]:8080 next dev\"",
"refactoring-report": "node scripts/refactoring-cli.js"
},

View File

@@ -0,0 +1,13 @@
'use strict'
const yaml = require('js-yaml')
const { collectDefinitions } = require('../services')
const definitions = collectDefinitions()
// Omit undefined
// https://github.com/nodeca/js-yaml/issues/356#issuecomment-312430599
const cleaned = JSON.parse(JSON.stringify(definitions))
process.stdout.write(yaml.safeDump(cleaned, { flowLevel: 5 }))

View File

@@ -22,7 +22,10 @@ const {
} = require('../lib/badge-data')
const { staticBadgeUrl } = require('../lib/make-badge-url')
const trace = require('./trace')
const validateExample = require('./validate-example')
const oldValidateExample = require('./validate-example')
const { validateExample, transformExample } = require('./transform-example')
const { assertValidCategory } = require('./categories')
const { assertValidServiceDefinition } = require('./service-definitions')
class BaseService {
constructor({ sendAndCacheRequest }, { handleInternalErrors }) {
@@ -69,6 +72,10 @@ class BaseService {
throw new Error(`Route not defined for ${this.name}`)
}
static get isDeprecated() {
return false
}
/**
* Default data for the badge. Can include things such as default logo, color,
* etc. These defaults will be used if the value is not explicitly overridden
@@ -94,17 +101,20 @@ class BaseService {
* is to use the service class name, which probably is not what you want.
* namedParams: An object containing the values of named parameters to
* substitute into the compiled route pattern.
* query: An object containing query parameters to include in the example URLs.
* queryParams: An object containing query parameters to include in the
* example URLs.
* query: Deprecated. An alias for `queryParams`.
* pattern: The route pattern to compile. Defaults to `this.route.pattern`.
* urlPattern: Deprecated. An alias for `pattern`.
* staticExample: A rendered badge of the sort returned by `handle()` or
* staticPreview: A rendered badge of the sort returned by `handle()` or
* `render()`: an object containing `message` and optional `label` and
* `color`. This is usually generated by invoking `this.render()` with some
* explicit props.
* staticExample: Deprecated. An alias for `staticPreview`.
* previewUrl: Deprecated. An explicit example which is rendered as part of
* the badge listing.
* exampleUrl: Deprecated. An explicit example which will be displayed to
* the user, but not rendered.
* exampleUrl: Deprecated. An explicit example which will _not_ be rendered.
* Only the URL itself is shown to the user.
* keywords: Additional keywords, other than words in the title. This helps
* users locate relevant badges.
* documentation: An HTML string that is included in the badge popup.
@@ -163,7 +173,7 @@ class BaseService {
staticExample,
documentation,
keywords,
} = validateExample(example, index, this)
} = oldValidateExample(example, index, this)
const stringified = queryString.stringify(query)
const suffix = stringified ? `?${stringified}` : ''
@@ -203,6 +213,44 @@ class BaseService {
})
}
static validateDefinition() {
assertValidCategory(this.category, `Category for ${this.name}`)
this.examples.forEach((example, index) =>
validateExample(example, index, this)
)
}
static getDefinition() {
const { category, name, isDeprecated } = this
let format, pattern, queryParams
try {
;({ format, pattern, query: queryParams = [] } = this.route)
} catch (e) {
// Legacy services do not have a route.
}
const examples = this.examples.map((example, index) =>
transformExample(example, index, this)
)
let route
if (pattern) {
route = { pattern, queryParams }
} else if (format) {
route = { format, queryParams }
} else {
route = undefined
}
const result = { category, name, isDeprecated, route, examples }
assertValidServiceDefinition(result, `getDefinition() for ${this.name}`)
return result
}
static get _regexFromPath() {
const { pattern } = this.route
const fullPattern = `${this._makeFullUrl(

View File

@@ -518,6 +518,86 @@ describe('BaseService', function() {
})
})
describe('getDefinition', function() {
it('returns the expected result', function() {
const {
examples: [first, second, third, fourth, fifth, sixth],
} = DummyService.getDefinition()
expect(first).to.deep.equal({
title: 'DummyService',
example: {
path: '/foo/World',
queryParams: {},
},
preview: {
path: '/foo/World',
queryParams: {},
},
keywords: [],
documentation: undefined,
})
expect(second).to.deep.equal({
title: 'DummyService',
example: {
path: '/foo/World',
queryParams: { queryParamA: '!!!' },
},
preview: {
path: '/foo/World',
queryParams: { queryParamA: '!!!' },
},
keywords: [],
documentation: undefined,
})
const expectedDefinition = {
title: 'DummyService',
example: {
path: '/foo/World',
queryParams: {},
},
preview: {
label: 'cat',
message: 'Hello namedParamA: foo with queryParamA: bar',
color: 'lightgrey',
},
keywords: ['hello'],
documentation: undefined,
}
expect(third).to.deep.equal(expectedDefinition)
expect(fourth).to.deep.equal(expectedDefinition)
expect(fifth).to.deep.equal({
title: 'DummyService',
example: {
pattern: '/foo/:world',
namedParams: { world: 'World' },
queryParams: {},
},
preview: {
label: 'cat',
message: 'Hello namedParamA: foo with queryParamA: bar',
color: 'lightgrey',
},
keywords: ['hello'],
documentation: undefined,
})
expect(sixth).to.deep.equal({
title: 'DummyService',
example: {
pattern: '/foo/:world',
namedParams: { world: 'World' },
queryParams: { queryParamA: '!!!' },
},
preview: {
color: 'lightgrey',
label: 'cat',
message: 'Hello namedParamA: foo with queryParamA: bar',
},
keywords: ['hello'],
documentation: undefined,
})
})
})
describe('validate', function() {
const dummySchema = Joi.object({
requiredString: Joi.string().required(),

View File

@@ -4,6 +4,7 @@ const deprecatedService = require('../deprecated-service')
// bitHound integration - deprecated as of July 2018
module.exports = deprecatedService({
category: 'dependencies',
url: {
base: 'bithound',
format: '(?:code/|dependencies/|devDependencies/)?(?:.+?)',

32
services/categories.js Normal file
View File

@@ -0,0 +1,32 @@
'use strict'
const Joi = require('joi')
const categories = [
{ id: 'build', name: 'Build' },
{ id: 'chat', name: 'Chat' },
{ id: 'dependencies', name: 'Dependencies' },
{ id: 'size', name: 'Size' },
{ id: 'downloads', name: 'Downloads' },
{ id: 'funding', name: 'Funding' },
{ id: 'issue-tracking', name: 'Issue Tracking' },
{ id: 'license', name: 'License' },
{ id: 'rating', name: 'Rating' },
{ id: 'social', name: 'Social' },
{ id: 'version', name: 'Version' },
{ id: 'platform-support', name: 'Platform & Version Support' },
{ id: 'monitoring', name: 'Monitoring' },
{ id: 'other', name: 'Other' },
]
const isValidCategory = Joi.equal(categories.map(({ id }) => id)).required()
function assertValidCategory(category, message = undefined) {
Joi.assert(category, isValidCategory, message)
}
module.exports = {
categories,
isValidCategory,
assertValidCategory,
}

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'other',
url: {
base: 'cauditor',
format: '(?:mi|ccn|npath|hi|i|ca|ce|dit)/(?:[^/]+)/(?:[^/]+)/(?:.+)',

View File

@@ -14,6 +14,10 @@ function deprecatedService({ url, label, category, examples = [] }) {
return url
}
static get isDeprecated() {
return true
}
static get defaultBadgeData() {
return { label }
}

View File

@@ -4,6 +4,7 @@ const deprecatedService = require('../deprecated-service')
// dotnet-status integration - deprecated as of April 2018.
module.exports = deprecatedService({
category: 'dependencies',
url: {
base: 'dotnetstatus',
format: '(?:.+)',

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'dependencies',
url: {
base: 'gemnasium',
format: '(?:.+)',

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'funding',
url: {
format: '(?:gittip|gratipay(?:/user|/team|/project)?)/(?:.*)',
},

View File

@@ -4,6 +4,7 @@ const deprecatedService = require('../deprecated-service')
// image layers integration - deprecated as of November 2018.
module.exports = deprecatedService({
category: 'size',
url: {
base: 'imagelayers',
format: '(?:.+)',

View File

@@ -2,6 +2,8 @@
const glob = require('glob')
const BaseService = require('./base')
const { categories } = require('./categories')
const { assertValidServiceDefinitionExport } = require('./service-definitions')
class InvalidService extends Error {
constructor(message) {
@@ -49,6 +51,19 @@ function loadServiceClasses(servicePaths) {
return serviceClasses
}
function collectDefinitions() {
const services = loadServiceClasses()
// flatMap.
.map(ServiceClass => ServiceClass.getDefinition())
.reduce((accum, these) => accum.concat(these), [])
const result = { schemaVersion: '0', categories, services }
assertValidServiceDefinitionExport(result)
return result
}
function loadTesters() {
return glob.sync(`${__dirname}/**/*.tester.js`).map(path => require(path))
}
@@ -57,4 +72,5 @@ module.exports = {
InvalidService,
loadServiceClasses,
loadTesters,
collectDefinitions,
}

View File

@@ -1,7 +1,11 @@
'use strict'
const { expect } = require('chai')
const { loadServiceClasses, InvalidService } = require('./index')
const {
loadServiceClasses,
InvalidService,
collectDefinitions,
} = require('./index')
describe('loadServiceClasses function', function() {
it('throws if module exports empty', function() {
@@ -54,4 +58,8 @@ describe('loadServiceClasses function', function() {
])
).to.have.length(5)
})
it('can collect the service definitions', function() {
expect(() => collectDefinitions()).not.to.throw()
})
})

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'issue-tracking',
url: {
base: 'issuestats',
format: '(?:[^/]+)(?:/long)?/(?:[^/]+)/(?:.+)',

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'rating',
url: {
base: 'libscore',
format: 's/(?:.+)',

View File

@@ -4,6 +4,7 @@ const deprecatedService = require('../deprecated-service')
// Magnum CI integration - deprecated as of July 2018
module.exports = deprecatedService({
category: 'build',
url: {
base: 'magnumci/ci',
format: '(?:[^/]+)(?:/(?:.+))?',

View File

@@ -0,0 +1,95 @@
'use strict'
const Joi = require('joi')
const arrayOfStrings = Joi.array()
.items(Joi.string())
.allow([])
.required()
const objectOfKeyValues = Joi.object()
.pattern(/./, Joi.string().allow(null))
.required()
const staticBadgeContent = Joi.object({
label: Joi.string(),
message: Joi.string().required(),
color: Joi.string().required(),
})
const serviceDefinition = Joi.object({
category: Joi.string().required(),
name: Joi.string().required(),
isDeprecated: Joi.boolean().required(),
route: Joi.alternatives().try(
Joi.object({
pattern: Joi.string().required(),
queryParams: arrayOfStrings,
}),
Joi.object({
format: Joi.string().required(),
queryParams: arrayOfStrings,
})
),
examples: Joi.array()
.items(
Joi.object({
title: Joi.string().required(),
example: Joi.alternatives()
.try(
Joi.object({
pattern: Joi.string(),
namedParams: objectOfKeyValues,
queryParams: objectOfKeyValues,
}),
Joi.object({
path: Joi.string().required(), // URL convertible.
queryParams: objectOfKeyValues,
})
)
.required(),
preview: Joi.alternatives()
.try(
staticBadgeContent,
Joi.object({
path: Joi.string().required(), // URL convertible.
queryParams: objectOfKeyValues,
})
)
.required(),
keywords: arrayOfStrings,
documentation: Joi.string(), // Valid HTML.
})
)
.default([]),
}).required()
function assertValidServiceDefinition(example, message = undefined) {
Joi.assert(example, serviceDefinition, message)
}
const serviceDefinitionExport = Joi.object({
schemaVersion: Joi.equal('0').required(),
categories: Joi.array()
.items(
Joi.object({
id: Joi.string().required(),
name: Joi.string().required(),
})
)
.required(),
services: Joi.array()
.items(serviceDefinition)
.required(),
}).required()
function assertValidServiceDefinitionExport(examples, message = undefined) {
Joi.assert(examples, serviceDefinitionExport, message)
}
module.exports = {
serviceDefinition,
assertValidServiceDefinition,
serviceDefinitionExport,
assertValidServiceDefinitionExport,
}

View File

@@ -3,6 +3,7 @@
const deprecatedService = require('../deprecated-service')
module.exports = deprecatedService({
category: 'build',
url: {
format: 'snap(?:-ci?)/(?:[^/]+/[^/]+)(?:/(?:.+))',
},

View File

@@ -0,0 +1,141 @@
'use strict'
const Joi = require('joi')
const optionalObjectOfKeyValues = Joi.object().pattern(
/./,
Joi.string().allow(null)
)
const optionalServiceData = Joi.object({
label: Joi.string(),
message: Joi.alternatives()
.try(
Joi.string()
.allow('')
.required(),
Joi.number()
)
.required(),
color: Joi.string(),
})
const schema = Joi.object({
// This should be:
// title: Joi.string().required(),
title: Joi.string(),
namedParams: optionalObjectOfKeyValues,
queryParams: optionalObjectOfKeyValues.default({}),
pattern: Joi.string(),
staticPreview: optionalServiceData,
previewUrl: Joi.string(),
exampleUrl: Joi.string(),
keywords: Joi.array()
.items(Joi.string())
.default([]),
documentation: Joi.string(), // Valid HTML.
})
.rename('query', 'queryParams', { ignoreUndefined: true })
.rename('staticExample', 'staticPreview', { ignoreUndefined: true })
.rename('urlPattern', 'pattern', { ignoreUndefined: true })
.required()
function validateExample(example, index, ServiceClass) {
const result = Joi.attempt(
example,
schema,
`Example for ${ServiceClass.name} at index ${index}`
)
const { namedParams, pattern, staticPreview, previewUrl, exampleUrl } = result
if (staticPreview) {
if (!pattern && !ServiceClass.route.pattern) {
throw new Error(
`Static preview for ${
ServiceClass.name
} at index ${index} does not declare a pattern`
)
}
if (namedParams && exampleUrl) {
throw new Error(
`Static preview for ${
ServiceClass.name
} at index ${index} declares both namedParams and exampleUrl`
)
} else if (!namedParams && !exampleUrl) {
throw new Error(
`Static preview for ${
ServiceClass.name
} at index ${index} does not declare namedParams nor exampleUrl`
)
}
if (previewUrl) {
throw new Error(
`Static preview for ${
ServiceClass.name
} at index ${index} also declares a dynamic previewUrl, which is not allowed`
)
}
} else if (!previewUrl) {
throw Error(
`Example for ${
ServiceClass.name
} at index ${index} is missing required previewUrl or staticPreview`
)
}
return result
}
function transformExample(inExample, index, ServiceClass) {
const {
// We should get rid of this transform, since the class name is never what
// we want to see.
title = ServiceClass.name,
namedParams,
queryParams,
pattern,
staticPreview,
previewUrl,
exampleUrl,
keywords,
documentation,
} = validateExample(inExample, index, ServiceClass)
let example
if (namedParams) {
example = {
pattern: ServiceClass._makeFullUrl(pattern),
namedParams,
queryParams,
}
} else {
example = {
path: ServiceClass._makeFullUrl(exampleUrl || previewUrl),
queryParams,
}
}
let preview
if (staticPreview) {
const badgeData = ServiceClass._makeBadgeData({}, staticPreview)
preview = {
label: badgeData.text[0],
message: `${badgeData.text[1]}`,
color: badgeData.colorscheme || badgeData.colorB,
}
} else {
preview = {
path: ServiceClass._makeFullUrl(previewUrl),
queryParams,
}
}
return { title, example, preview, keywords, documentation }
}
module.exports = {
validateExample,
transformExample,
}

View File

@@ -0,0 +1,53 @@
'use strict'
const { expect } = require('chai')
const { validateExample } = require('./transform-example')
describe('validateExample function', function() {
it('passes valid examples', function() {
const validExamples = [
{
staticExample: { message: '123' },
pattern: 'dt/:package',
exampleUrl: 'dt/mypackage',
},
{
staticExample: { message: '123' },
pattern: 'dt/:package',
namedParams: { package: 'mypackage' },
},
{ previewUrl: 'dt/mypackage' },
]
validExamples.forEach(example => {
expect(() =>
validateExample(example, 0, { route: {}, name: 'mockService' })
).not.to.throw(Error)
})
})
it('rejects invalid examples', function() {
const invalidExamples = [
{},
{ staticExample: { message: '123' } },
{
staticExample: { message: '123' },
pattern: 'dt/:package',
namedParams: { package: 'mypackage' },
exampleUrl: 'dt/mypackage',
},
{ staticExample: { message: '123' }, pattern: 'dt/:package' },
{
staticExample: { message: '123' },
pattern: 'dt/:package',
previewUrl: 'dt/mypackage',
},
]
invalidExamples.forEach(example => {
expect(() =>
validateExample(example, 0, { route: {}, name: 'mockService' })
).to.throw(Error)
})
})
})

View File

@@ -51,7 +51,7 @@ module.exports = class TravisBuild extends LegacyService {
]
}
static staticExample() {
static get staticExample() {
return { message: 'passing', color: 'brightgreen' }
}

View File

@@ -4,12 +4,14 @@ module.exports = function validateExample(
{
title,
query,
queryParams,
namedParams,
exampleUrl,
previewUrl,
pattern,
urlPattern,
staticExample,
staticPreview,
documentation,
keywords,
},
@@ -17,6 +19,8 @@ module.exports = function validateExample(
ServiceClass
) {
pattern = pattern || urlPattern || ServiceClass.route.pattern
staticExample = staticExample || staticPreview
query = query || queryParams
if (staticExample) {
if (!pattern) {

View File

@@ -4,6 +4,7 @@ const deprecatedService = require('../deprecated-service')
// VersionEye integration - deprecated as of August 2018.
module.exports = deprecatedService({
category: 'downloads',
url: {
base: 'versioneye',
format: 'd/(?:.+)',