Add authentication for Libraries.io-based badges, run [Libraries Bower] (#7080)

* feat: support authentication on Libraries.io requests

* feat: wire up libraries.io config and api provider instantiation

* feat: create libraries.io and bower base classes

* refactor: tweak libraries/bower service classes and tests

* rename request fetcher function/arg

* throw exception when no tokens available

* cleanup old value

Co-authored-by: repo-ranger[bot] <39074581+repo-ranger[bot]@users.noreply.github.com>
This commit is contained in:
Caleb Cartwright
2021-10-28 19:21:24 -05:00
committed by GitHub
parent 6e100bf274
commit ae58e4a211
21 changed files with 510 additions and 81 deletions

View File

@@ -86,6 +86,7 @@ private:
jenkins_pass: 'JENKINS_PASS'
jira_user: 'JIRA_USER'
jira_pass: 'JIRA_PASS'
librariesio_tokens: 'LIBRARIESIO_TOKENS'
nexus_user: 'NEXUS_USER'
nexus_pass: 'NEXUS_PASS'
npm_token: 'NPM_TOKEN'

View File

@@ -420,7 +420,13 @@ class BaseService {
}
static register(
{ camp, handleRequest, githubApiProvider, metricInstance },
{
camp,
handleRequest,
githubApiProvider,
librariesIoApiProvider,
metricInstance,
},
serviceConfig
) {
const { cacheHeaders: cacheHeaderConfig, fetchLimitBytes } = serviceConfig
@@ -447,6 +453,7 @@ class BaseService {
sendAndCacheRequest: fetcher,
sendAndCacheRequestWithCallbacks: request,
githubApiProvider,
librariesIoApiProvider,
metricHelper,
},
serviceConfig,

View File

@@ -13,6 +13,7 @@ import {
Inaccessible,
InvalidParameter,
Deprecated,
ImproperlyConfigured,
} from './errors.js'
export {
@@ -29,5 +30,6 @@ export {
InvalidResponse,
Inaccessible,
InvalidParameter,
ImproperlyConfigured,
Deprecated,
}

View File

@@ -11,6 +11,7 @@ import Camp from '@shields_io/camp'
import originalJoi from 'joi'
import makeBadge from '../../badge-maker/lib/make-badge.js'
import GithubConstellation from '../../services/github/github-constellation.js'
import LibrariesIoConstellation from '../../services/librariesio/librariesio-constellation.js'
import { setRoutes } from '../../services/suggest.js'
import { loadServiceClasses } from '../base-service/loader.js'
import { makeSend } from '../base-service/legacy-result-sender.js'
@@ -170,6 +171,7 @@ const privateConfigSchema = Joi.object({
jira_pass: Joi.string(),
bitbucket_server_username: Joi.string(),
bitbucket_server_password: Joi.string(),
librariesio_tokens: Joi.arrayFromString().items(Joi.string()),
nexus_user: Joi.string(),
nexus_pass: Joi.string(),
npm_token: Joi.string(),
@@ -241,6 +243,10 @@ class Server {
private: privateConfig,
})
this.librariesioConstellation = new LibrariesIoConstellation({
private: privateConfig,
})
if (publicConfig.metrics.prometheus.enabled) {
this.metricInstance = new PrometheusMetrics()
if (publicConfig.metrics.influx.enabled) {
@@ -413,10 +419,17 @@ class Server {
async registerServices() {
const { config, camp, metricInstance } = this
const { apiProvider: githubApiProvider } = this.githubConstellation
const { apiProvider: librariesIoApiProvider } =
this.librariesioConstellation
;(await loadServiceClasses()).forEach(serviceClass =>
serviceClass.register(
{ camp, handleRequest, githubApiProvider, metricInstance },
{
camp,
handleRequest,
githubApiProvider,
librariesIoApiProvider,
metricInstance,
},
{
handleInternalErrors: config.public.handleInternalErrors,
cacheHeaders: config.public.cacheHeaders,

View File

@@ -80,6 +80,10 @@ class Token {
return this.usesRemaining <= 0 && !this.hasReset
}
get decrementedUsesRemaining() {
return this._usesRemaining - 1
}
/**
* Update the uses remaining and next reset time for a token.
*

View File

@@ -174,6 +174,24 @@ access to a private Jenkins CI instance.
Provide a username and password to give your self-hosted Shields installation
access to a private JIRA instance.
### Libraries.io/Bower
- `LIBRARIESIO_TOKENS` (yml: `private.librariesio_tokens`)
Note that the Bower badges utilize the Libraries.io API, so use this secret for both Libraries.io badges and/or Bower badges.
Just like the `*_ORIGINS` type secrets, this value can accept a single token as a string, or a group of tokens provided as an array of strings. For example:
```yaml
private:
librariesio_tokens: my-token
## Or
private:
librariesio_tokens: [my-token some-other-token]
```
When using the environment variable with multiple tokens, be sure to use a space to separate the tokens, e.g. `LIBRARIESIO_TOKENS="my-token some-other-token"`
### Nexus
- `NEXUS_ORIGINS` (yml: `public.services.nexus.authorizedOrigins`)

View File

@@ -1,5 +1,5 @@
import Joi from 'joi'
import { BaseJsonService } from '../index.js'
import LibrariesIoBase from '../librariesio/librariesio-base.js'
const schema = Joi.object()
.keys({
@@ -17,11 +17,11 @@ const schema = Joi.object()
})
.required()
export default class BaseBowerService extends BaseJsonService {
export default class BaseBowerService extends LibrariesIoBase {
async fetch({ packageName }) {
return this._requestJson({
schema,
url: `https://libraries.io/api/bower/${packageName}`,
url: `/bower/${packageName}`,
errorMessages: {
404: 'package not found',
},

View File

@@ -6,15 +6,6 @@ t.create('licence')
.get('/bootstrap.json')
.expectBadge({ label: 'license', message: 'MIT' })
t.create('license not declared')
.get('/bootstrap.json')
.intercept(nock =>
nock('https://libraries.io')
.get('/api/bower/bootstrap')
.reply(200, { normalized_licenses: [] })
)
.expectBadge({ label: 'license', message: 'missing' })
t.create('licence for Invalid Package')
.timeout(10000)
.get('/it-is-a-invalid-package-should-error.json')

View File

@@ -27,9 +27,7 @@ class BowerVersion extends BaseBowerService {
static defaultBadgeData = { label: 'bower' }
async handle({ packageName }, queryParams) {
const data = await this.fetch({ packageName })
const includePrereleases = queryParams.include_prereleases !== undefined
static transform(data, includePrereleases) {
const version = includePrereleases
? data.latest_release_number
: data.latest_stable_release_number
@@ -38,6 +36,14 @@ class BowerVersion extends BaseBowerService {
throw new InvalidResponse({ prettyMessage: 'no releases' })
}
return version
}
async handle({ packageName }, queryParams) {
const data = await this.fetch({ packageName })
const includePrereleases = queryParams.include_prereleases !== undefined
const version = this.constructor.transform(data, includePrereleases)
return renderVersionBadge({ version })
}
}

View File

@@ -0,0 +1,92 @@
import { expect } from 'chai'
import { test, given } from 'sazerac'
import nock from 'nock'
import { cleanUpNockAfterEach, defaultContext } from '../test-helpers.js'
import { InvalidResponse } from '../index.js'
import LibrariesIoApiProvider from '../librariesio/librariesio-api-provider.js'
import { BowerVersion } from './bower-version.service.js'
describe('BowerVersion', function () {
test(BowerVersion.transform, () => {
given(
{
latest_release_number: '2.0.0-beta',
latest_stable_release_number: '1.8.3',
},
false
).expect('1.8.3')
given(
{
latest_release_number: '2.0.0-beta',
latest_stable_release_number: '1.8.3',
},
true
).expect('2.0.0-beta')
})
it('throws `no releases` InvalidResponse if no stable version', function () {
expect(() =>
BowerVersion.transform({ latest_release_number: 'panda' }, false)
)
.to.throw(InvalidResponse)
.with.property('prettyMessage', 'no releases')
})
it('throws `no releases` InvalidResponse if no prereleases', function () {
expect(() =>
BowerVersion.transform({ latest_stable_release_number: 'penguin' }, true)
)
.to.throw(InvalidResponse)
.with.property('prettyMessage', 'no releases')
})
context('auth', function () {
cleanUpNockAfterEach()
const fakeApiKey = 'fakeness'
const response = {
normalized_licenses: [],
latest_release_number: '2.0.0-beta',
latest_stable_release_number: '1.8.3',
}
const config = {
private: {
librariesio_tokens: fakeApiKey,
},
}
const librariesIoApiProvider = new LibrariesIoApiProvider({
baseUrl: 'https://libraries.io/api',
tokens: [fakeApiKey],
})
it('sends the auth information as configured', async function () {
const scope = nock('https://libraries.io/api')
// This ensures that the expected credentials are actually being sent with the HTTP request.
// Without this the request wouldn't match and the test would fail.
.get(`/bower/bootstrap?api_key=${fakeApiKey}`)
.reply(200, response)
expect(
await BowerVersion.invoke(
{
...defaultContext,
librariesIoApiProvider,
},
config,
{
platform: 'bower',
packageName: 'bootstrap',
},
{
include_prereleases: '',
}
)
).to.deep.equal({
message: 'v2.0.0-beta',
color: 'orange',
label: undefined,
})
scope.done()
})
})
})

View File

@@ -34,24 +34,6 @@ t.create('Pre Version for Invalid Package')
.get('/v/it-is-a-invalid-package-should-error.json?include_prereleases')
.expectBadge({ label: 'bower', message: 'package not found' })
t.create('Version label should be `no releases` if no stable version')
.get('/v/bootstrap.json')
.intercept(nock =>
nock('https://libraries.io')
.get('/api/bower/bootstrap')
.reply(200, { normalized_licenses: [], latest_stable_release: null })
)
.expectBadge({ label: 'bower', message: 'no releases' })
t.create('Version label should be `no releases` if no pre-release')
.get('/v/bootstrap.json?include_prereleases')
.intercept(nock =>
nock('https://libraries.io')
.get('/api/bower/bootstrap')
.reply(200, { normalized_licenses: [], latest_release_number: null })
)
.expectBadge({ label: 'bower', message: 'no releases' })
t.create('Version (legacy redirect: vpre)')
.get('/vpre/bootstrap.svg')
.expectRedirect('/bower/v/bootstrap.svg?include_prereleases')

View File

@@ -0,0 +1,108 @@
import { ImproperlyConfigured } from '../index.js'
import log from '../../core/server/log.js'
import { TokenPool } from '../../core/token-pooling/token-pool.js'
import { userAgent } from '../../core/base-service/legacy-request-handler.js'
// Provides an interface to the Libraries.io API.
export default class LibrariesIoApiProvider {
constructor({ baseUrl, tokens = [], defaultRateLimit = 60 }) {
const withPooling = tokens.length > 1
Object.assign(this, {
baseUrl,
withPooling,
globalToken: tokens[0],
defaultRateLimit,
})
if (this.withPooling) {
this.standardTokens = new TokenPool({ batchSize: 45 })
tokens.forEach(t => this.standardTokens.add(t, {}, defaultRateLimit))
}
}
getRateLimitFromHeaders({ headers, token }) {
// The Libraries.io API does not consistently provide the rate limiting headers.
// In some cases (e.g. package/version not founds) it won't include any of these headers,
// and the `retry-after` header is only provided _after_ the rate limit has been exceeded
// and requests are throttled.
//
// https://github.com/librariesio/libraries.io/issues/2860
// The standard rate limit is 60/requests/minute, so fallback to that default
// if the header isn't present.
// https://libraries.io/api#rate-limit
const rateLimit = headers['x-ratelimit-limit'] || this.defaultRateLimit
// If the remaining header is missing, then we're in the 404 response phase, and simply
// subtract one from the `usesRemaining` count on the token, since the 404 responses do count
// against the rate limits.
const totalUsesRemaining =
headers['x-ratelimit-remaining'] || token.decrementedUsesRemaining
// The `retry-after` header is only present post-rate limit excess, and contains the value in
// seconds the client needs to wait before the limits are reset.
// Our token pools internally use UTC-based milliseconds, so we perform the conversion
// if the header is present to ensure the token pool has the correct value.
// If the header is absent, we just use the current timestamp to
// advance the value to _something_
const retryAfter = headers['retry-after']
const nextReset = Date.now() + (retryAfter ? retryAfter * 1000 : 0)
return {
rateLimit,
totalUsesRemaining,
nextReset,
}
}
updateToken({ token, res }) {
const { totalUsesRemaining, nextReset } = this.getRateLimitFromHeaders({
headers: res.headers,
token,
})
token.update(totalUsesRemaining, nextReset)
}
async fetch(requestFetcher, url, options = {}) {
const { baseUrl } = this
let token
let tokenString
if (this.withPooling) {
try {
token = this.standardTokens.next()
} catch (e) {
log.error(e)
throw new ImproperlyConfigured({
prettyMessage: 'Unable to select next Libraries.io token from pool',
})
}
tokenString = token.id
} else {
tokenString = this.globalToken
}
const mergedOptions = {
...options,
...{
headers: {
'User-Agent': userAgent,
...options.headers,
},
qs: {
api_key: tokenString,
...options.qs,
},
},
}
const response = await requestFetcher(`${baseUrl}${url}`, mergedOptions)
if (this.withPooling) {
if (response.res.statusCode === 401) {
this.invalidateToken(token)
} else if (response.res.statusCode < 500) {
this.updateToken({ token, url, res: response.res })
}
}
return response
}
}

View File

@@ -0,0 +1,132 @@
import { expect } from 'chai'
import sinon from 'sinon'
import { ImproperlyConfigured } from '../index.js'
import log from '../../core/server/log.js'
import LibrariesIoApiProvider from './librariesio-api-provider.js'
describe('LibrariesIoApiProvider', function () {
const baseUrl = 'https://libraries.io/api'
const tokens = ['abc123', 'def456']
const rateLimit = 60
const remaining = 57
const nextReset = 60
const mockResponse = {
res: {
statusCode: 200,
headers: {
'x-ratelimit-limit': rateLimit,
'x-ratelimit-remaining': remaining,
'retry-after': nextReset,
},
},
buffer: {},
}
let token, provider, nextTokenStub
beforeEach(function () {
provider = new LibrariesIoApiProvider({ baseUrl, tokens })
token = {
update: sinon.spy(),
invalidate: sinon.spy(),
decrementedUsesRemaining: remaining - 1,
}
nextTokenStub = sinon.stub(provider.standardTokens, 'next').returns(token)
})
afterEach(function () {
sinon.restore()
})
context('a core API request', function () {
const mockResponse = { res: { headers: {} } }
const mockRequest = sinon.stub().resolves(mockResponse)
it('should obtain an appropriate token', async function () {
await provider.fetch(mockRequest, '/npm/badge-maker')
expect(provider.standardTokens.next).to.have.been.calledOnce
})
it('should throw an error when the next token fails', async function () {
nextTokenStub.throws(Error)
sinon.stub(log, 'error')
try {
await provider.fetch(mockRequest, '/npm/badge-maker')
expect.fail('Expected to throw')
} catch (e) {
expect(e).to.be.an.instanceof(ImproperlyConfigured)
expect(e.prettyMessage).to.equal(
'Unable to select next Libraries.io token from pool'
)
}
})
})
context('a valid API response', function () {
const mockRequest = sinon.stub().resolves(mockResponse)
const tickTime = 123456789
beforeEach(function () {
const clock = sinon.useFakeTimers()
clock.tick(tickTime)
})
it('should return the response', async function () {
const res = await provider.fetch(mockRequest, '/npm/badge-maker')
expect(Object.is(res, mockResponse)).to.be.true
})
it('should update the token with the expected values when headers are present', async function () {
await provider.fetch(mockRequest, '/npm/badge-maker')
expect(token.update).to.have.been.calledWith(
remaining,
nextReset * 1000 + tickTime
)
expect(token.invalidate).not.to.have.been.called
})
it('should update the token with the expected values when throttling not applied', async function () {
const response = {
res: {
statusCode: 200,
headers: {
'x-ratelimit-limit': rateLimit,
'x-ratelimit-remaining': remaining,
},
},
}
const mockRequest = sinon.stub().resolves(response)
await provider.fetch(mockRequest, '/npm/badge-maker')
expect(token.update).to.have.been.calledWith(remaining, tickTime)
expect(token.invalidate).not.to.have.been.called
})
it('should update the token with the expected values in 404 case', async function () {
const response = {
res: { statusCode: 200, headers: {} },
}
const mockRequest = sinon.stub().resolves(response)
await provider.fetch(mockRequest, '/npm/badge-maker')
expect(token.update).to.have.been.calledWith(remaining - 1, tickTime)
expect(token.invalidate).not.to.have.been.called
})
})
context('a connection error', function () {
const msg = 'connection timeout'
const requestError = new Error(msg)
const mockRequest = sinon.stub().rejects(requestError)
it('should pass the error to the callback', async function () {
try {
await provider.fetch(mockRequest, '/npm/badge-maker')
expect(false).to.be.true
} catch (err) {
expect(err).to.be.an.instanceof(Error)
expect(err.message).to.equal(msg)
}
})
})
})

View File

@@ -0,0 +1,35 @@
import Joi from 'joi'
import { anyInteger, nonNegativeInteger } from '../validators.js'
import { BaseJsonService } from '../index.js'
// API doc: https://libraries.io/api#project
const projectSchema = Joi.object({
platform: Joi.string().required(),
dependents_count: nonNegativeInteger,
dependent_repos_count: nonNegativeInteger,
rank: anyInteger,
}).required()
function createRequestFetcher(context, config) {
const { sendAndCacheRequest, librariesIoApiProvider } = context
return async (url, options) =>
await librariesIoApiProvider.fetch(sendAndCacheRequest, url, options)
}
export default class LibrariesIoBase extends BaseJsonService {
constructor(context, config) {
super(context, config)
this._requestFetcher = createRequestFetcher(context, config)
}
async fetchProject({ platform, scope, packageName }) {
return this._requestJson({
schema: projectSchema,
url: `/${encodeURIComponent(platform)}/${
scope ? encodeURIComponent(`${scope}/`) : ''
}${encodeURIComponent(packageName)}`,
errorMessages: { 404: 'package not found' },
})
}
}

View File

@@ -1,22 +0,0 @@
import Joi from 'joi'
import { nonNegativeInteger, anyInteger } from '../validators.js'
// API doc: https://libraries.io/api#project
const projectSchema = Joi.object({
platform: Joi.string().required(),
dependents_count: nonNegativeInteger,
dependent_repos_count: nonNegativeInteger,
rank: anyInteger,
}).required()
async function fetchProject(serviceInstance, { platform, scope, packageName }) {
return serviceInstance._requestJson({
schema: projectSchema,
url: `https://libraries.io/api/${encodeURIComponent(platform)}/${
scope ? encodeURIComponent(`${scope}/`) : ''
}${encodeURIComponent(packageName)}`,
errorMessages: { 404: 'package not found' },
})
}
export { fetchProject }

View File

@@ -0,0 +1,13 @@
import LibrariesIoApiProvider from './librariesio-api-provider.js'
// Convenience class with all the stuff related to the Libraries.io API and its
// authorization tokens, to simplify server initialization.
export default class LibrariesIoConstellation {
constructor({ private: { librariesio_tokens: tokens } }) {
this.apiProvider = new LibrariesIoApiProvider({
baseUrl: 'https://libraries.io/api',
tokens,
defaultRateLimit: 60,
})
}
}

View File

@@ -1,5 +1,5 @@
import Joi from 'joi'
import { BaseJsonService } from '../index.js'
import LibrariesIoBase from './librariesio-base.js'
import {
transform,
renderDependenciesBadge,
@@ -16,7 +16,7 @@ const schema = Joi.object({
.default([]),
}).required()
class LibrariesIoProjectDependencies extends BaseJsonService {
class LibrariesIoProjectDependencies extends LibrariesIoBase {
static category = 'dependencies'
static route = {
@@ -82,7 +82,7 @@ class LibrariesIoProjectDependencies extends BaseJsonService {
]
async handle({ platform, scope, packageName, version = 'latest' }) {
const url = `https://libraries.io/api/${encodeURIComponent(platform)}/${
const url = `/${encodeURIComponent(platform)}/${
scope ? encodeURIComponent(`${scope}/`) : ''
}${encodeURIComponent(packageName)}/${encodeURIComponent(
version
@@ -97,7 +97,7 @@ class LibrariesIoProjectDependencies extends BaseJsonService {
}
}
class LibrariesIoRepoDependencies extends BaseJsonService {
class LibrariesIoRepoDependencies extends LibrariesIoBase {
static category = 'dependencies'
static route = {
@@ -117,9 +117,9 @@ class LibrariesIoRepoDependencies extends BaseJsonService {
]
async handle({ user, repo }) {
const url = `https://libraries.io/api/github/${encodeURIComponent(
user
)}/${encodeURIComponent(repo)}/dependencies`
const url = `/github/${encodeURIComponent(user)}/${encodeURIComponent(
repo
)}/dependencies`
const json = await this._requestJson({
url,
schema,

View File

@@ -1,9 +1,8 @@
import { metric } from '../text-formatters.js'
import { BaseJsonService } from '../index.js'
import { fetchProject } from './librariesio-common.js'
import LibrariesIoBase from './librariesio-base.js'
// https://libraries.io/api#project-dependent-repositories
export default class LibrariesIoDependentRepos extends BaseJsonService {
export default class LibrariesIoDependentRepos extends LibrariesIoBase {
static category = 'other'
static route = {
@@ -45,14 +44,12 @@ export default class LibrariesIoDependentRepos extends BaseJsonService {
}
async handle({ platform, scope, packageName }) {
const { dependent_repos_count: dependentReposCount } = await fetchProject(
this,
{
const { dependent_repos_count: dependentReposCount } =
await this.fetchProject({
platform,
scope,
packageName,
}
)
})
return this.constructor.render({ dependentReposCount })
}
}

View File

@@ -1,9 +1,8 @@
import { metric } from '../text-formatters.js'
import { BaseJsonService } from '../index.js'
import { fetchProject } from './librariesio-common.js'
import LibrariesIoBase from './librariesio-base.js'
// https://libraries.io/api#project-dependents
export default class LibrariesIoDependents extends BaseJsonService {
export default class LibrariesIoDependents extends LibrariesIoBase {
static category = 'other'
static route = {
@@ -45,7 +44,7 @@ export default class LibrariesIoDependents extends BaseJsonService {
}
async handle({ platform, scope, packageName }) {
const { dependents_count: dependentCount } = await fetchProject(this, {
const { dependents_count: dependentCount } = await this.fetchProject({
platform,
scope,
packageName,

View File

@@ -1,10 +1,9 @@
import { colorScale } from '../color-formatters.js'
import { BaseJsonService } from '../index.js'
import { fetchProject } from './librariesio-common.js'
import LibrariesIoBase from './librariesio-base.js'
const sourceRankColor = colorScale([10, 15, 20, 25, 30])
export default class LibrariesIoSourcerank extends BaseJsonService {
export default class LibrariesIoSourcerank extends LibrariesIoBase {
static category = 'rating'
static route = {
@@ -46,7 +45,7 @@ export default class LibrariesIoSourcerank extends BaseJsonService {
}
async handle({ platform, scope, packageName }) {
const { rank } = await fetchProject(this, {
const { rank } = await this.fetchProject({
platform,
scope,
packageName,

View File

@@ -0,0 +1,52 @@
import { expect } from 'chai'
import nock from 'nock'
import { cleanUpNockAfterEach, defaultContext } from '../test-helpers.js'
import LibrariesIoSourcerank from './librariesio-sourcerank.service.js'
import LibrariesIoApiProvider from './librariesio-api-provider.js'
describe('LibrariesIoSourcerank', function () {
cleanUpNockAfterEach()
const fakeApiKey = 'fakeness'
const response = {
platform: 'npm',
dependents_count: 150,
dependent_repos_count: 191,
rank: 100,
}
const config = {
private: {
librariesio_tokens: fakeApiKey,
},
}
const librariesIoApiProvider = new LibrariesIoApiProvider({
baseUrl: 'https://libraries.io/api',
tokens: [fakeApiKey],
})
it('sends the auth information as configured', async function () {
const scope = nock('https://libraries.io/api')
// This ensures that the expected credentials are actually being sent with the HTTP request.
// Without this the request wouldn't match and the test would fail.
.get(`/npm/badge-maker?api_key=${fakeApiKey}`)
.reply(200, response)
expect(
await LibrariesIoSourcerank.invoke(
{
...defaultContext,
librariesIoApiProvider,
},
config,
{
platform: 'npm',
packageName: 'badge-maker',
}
)
).to.deep.equal({
message: 100,
color: 'brightgreen',
})
scope.done()
})
})