Move legacy request helpers (#2829)

In #2698 we decided to put legacy helper functions in `core/legacy`. I think that’s a fine idea, though if we’re going to have a bunch of badge helper functions in there, it seems like it is probably better to keep these two important but esoteric helper functions with the core code to which they are most coupled. So I added `legacy-` to the name, and put them in `core/base-service`.
This commit is contained in:
Paul Melnikow
2019-01-21 22:14:22 -05:00
committed by GitHub
parent eb390a30c9
commit aac2a4d5ad
9 changed files with 19 additions and 19 deletions

View File

@@ -1,308 +0,0 @@
'use strict'
// eslint-disable-next-line node/no-deprecated-api
const domain = require('domain')
const request = require('request')
const queryString = require('query-string')
const log = require('../core/server/log')
const analytics = require('../core/server/analytics')
const LruCache = require('../gh-badges/lib/lru-cache')
const makeBadge = require('../gh-badges/lib/make-badge')
const {
Inaccessible,
InvalidResponse,
ShieldsRuntimeError,
} = require('../services/errors')
const { setCacheHeaders } = require('../services/cache-headers')
const { makeBadgeData: getBadgeData } = require('./badge-data')
const { makeSend } = require('./result-sender')
// We avoid calling the vendor's server for computation of the information in a
// number of badges.
const minAccuracy = 0.75
// The quotient of (vendor) data change frequency by badge request frequency
// must be lower than this to trigger sending the cached data *before*
// updating our data from the vendor's server.
// Indeed, the accuracy of our badges are:
// A(Δt) = 1 - min(# data change over Δt, # requests over Δt)
// / (# requests over Δt)
// = 1 - max(1, df) / rf
const freqRatioMax = 1 - minAccuracy
// Request cache size of 5MB (~5000 bytes/image).
const requestCache = new LruCache(1000)
// Deep error handling for vendor hooks.
const vendorDomain = domain.create()
vendorDomain.on('error', err => {
log.error('Vendor hook error:', err.stack)
})
// These query parameters are available to any badge. For the most part they
// are used by makeBadgeData (see `lib/badge-data.js`) and related functions.
const globalQueryParams = new Set([
'label',
'style',
'link',
'logo',
'logoColor',
'logoPosition',
'logoWidth',
'link',
'colorA',
'colorB',
])
function flattenQueryParams(queryParams) {
const union = new Set(globalQueryParams)
;(queryParams || []).forEach(name => {
union.add(name)
})
return Array.from(union).sort()
}
// handlerOptions can contain:
// - handler: The service's request handler function
// - queryParams: An array of the field names of any custom query parameters
// the service uses
// - cacheLength: An optional badge or category-specific cache length
// (in number of seconds) to be used in preference to the default
// - fetchLimitBytes: A limit on the response size we're willing to parse
//
// For safety, the service must declare the query parameters it wants to use.
// Only the declared parameters (and the global parameters) are provided to
// the service. Consequently, failure to declare a parameter results in the
// parameter not working at all (which is undesirable, but easy to debug)
// rather than indeterminate behavior that depends on the cache state
// (undesirable and hard to debug).
//
// Pass just the handler function as shorthand.
function handleRequest(cacheHeaderConfig, handlerOptions) {
if (!cacheHeaderConfig) {
throw Error('cacheHeaderConfig is required')
}
if (typeof handlerOptions === 'function') {
handlerOptions = { handler: handlerOptions }
}
const allowedKeys = flattenQueryParams(handlerOptions.queryParams)
const {
cacheLength: serviceDefaultCacheLengthSeconds,
fetchLimitBytes,
} = handlerOptions
return (queryParams, match, end, ask) => {
const reqTime = new Date()
// `defaultCacheLengthSeconds` can be overridden by
// `serviceDefaultCacheLengthSeconds` (either by category or on a badge-
// by-badge basis). Then in turn that can be overridden by
// `serviceOverrideCacheLengthSeconds` (which we expect to be used only in
// the dynamic badge) but only if `serviceOverrideCacheLengthSeconds` is
// longer than `serviceDefaultCacheLengthSeconds` and then the `maxAge`
// query param can also override both of those but again only if `maxAge`
// is longer.
//
// When the legacy services have been rewritten, all the code in here
// will go away, which should achieve this goal in a simpler way.
//
// Ref: https://github.com/badges/shields/pull/2755
function setCacheHeadersOnResponse(res, serviceOverrideCacheLengthSeconds) {
setCacheHeaders({
cacheHeaderConfig,
serviceDefaultCacheLengthSeconds,
serviceOverrideCacheLengthSeconds,
queryParams,
res,
})
}
analytics.noteRequest(queryParams, match)
const filteredQueryParams = {}
allowedKeys.forEach(key => {
filteredQueryParams[key] = queryParams[key]
})
// Use sindresorhus query-string because it sorts the keys, whereas the
// builtin querystring module relies on the iteration order.
const stringified = queryString.stringify(filteredQueryParams)
const cacheIndex = `${match[0]}?${stringified}`
// Should we return the data right away?
const cached = requestCache.get(cacheIndex)
let cachedVersionSent = false
if (cached !== undefined) {
// A request was made not long ago.
const tooSoon = +reqTime - cached.time < cached.interval
if (tooSoon || cached.dataChange / cached.reqs <= freqRatioMax) {
const svg = makeBadge(cached.data.badgeData)
setCacheHeadersOnResponse(
ask.res,
cached.data.badgeData.cacheLengthSeconds
)
makeSend(cached.data.format, ask.res, end)(svg)
cachedVersionSent = true
// We do not wish to call the vendor servers.
if (tooSoon) {
return
}
}
}
// In case our vendor servers are unresponsive.
let serverUnresponsive = false
const serverResponsive = setTimeout(() => {
serverUnresponsive = true
if (cachedVersionSent) {
return
}
if (requestCache.has(cacheIndex)) {
const cached = requestCache.get(cacheIndex)
const svg = makeBadge(cached.data.badgeData)
setCacheHeadersOnResponse(
ask.res,
cached.data.badgeData.cacheLengthSeconds
)
makeSend(cached.data.format, ask.res, end)(svg)
return
}
ask.res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
const badgeData = getBadgeData('vendor', filteredQueryParams)
badgeData.text[1] = 'unresponsive'
let extension
try {
extension = match[0].split('.').pop()
} catch (e) {
extension = 'svg'
}
const svg = makeBadge(badgeData)
setCacheHeadersOnResponse(ask.res)
makeSend(extension, ask.res, end)(svg)
}, 25000)
// Only call vendor servers when last request is older than…
let cacheInterval = 5000 // milliseconds
function cachingRequest(uri, options, callback) {
if (typeof options === 'function' && !callback) {
callback = options
}
if (options && typeof options === 'object') {
options.uri = uri
} else if (typeof uri === 'string') {
options = { uri }
} else {
options = uri
}
options.headers = options.headers || {}
options.headers['User-Agent'] =
options.headers['User-Agent'] || 'Shields.io'
let bufferLength = 0
const r = request(options, (err, res, body) => {
if (res != null && res.headers != null) {
const cacheControl = res.headers['cache-control']
if (cacheControl != null) {
const age = cacheControl.match(/max-age=([0-9]+)/)
// Would like to get some more test coverage on this before changing it.
// eslint-disable-next-line no-self-compare
if (age != null && +age[1] === +age[1]) {
cacheInterval = +age[1] * 1000
}
}
}
callback(err, res, body)
})
r.on('data', chunk => {
bufferLength += chunk.length
if (bufferLength > fetchLimitBytes) {
r.abort()
r.emit(
'error',
new InvalidResponse({
prettyMessage: 'Maximum response size exceeded',
})
)
}
})
}
// Wrapper around `cachingRequest` that returns a promise rather than
// needing to pass a callback.
cachingRequest.asPromise = (uri, options) =>
new Promise((resolve, reject) => {
cachingRequest(uri, options, (err, res, buffer) => {
if (err) {
if (err instanceof ShieldsRuntimeError) {
reject(err)
} else {
// Wrap the error in an Inaccessible so it can be identified
// by the BaseService handler.
reject(new Inaccessible({ underlyingError: err }))
}
} else {
resolve({ res, buffer })
}
})
})
vendorDomain.run(() => {
const result = handlerOptions.handler(
filteredQueryParams,
match,
// eslint-disable-next-line mocha/prefer-arrow-callback
function sendBadge(format, badgeData) {
if (serverUnresponsive) {
return
}
clearTimeout(serverResponsive)
// Check for a change in the data.
let dataHasChanged = false
if (
cached !== undefined &&
cached.data.badgeData.text[1] !== badgeData.text[1]
) {
dataHasChanged = true
}
// Add format to badge data.
badgeData.format = format
// Update information in the cache.
const updatedCache = {
reqs: cached ? cached.reqs + 1 : 1,
dataChange: cached
? cached.dataChange + (dataHasChanged ? 1 : 0)
: 1,
time: +reqTime,
interval: cacheInterval,
data: { format, badgeData },
}
requestCache.set(cacheIndex, updatedCache)
if (!cachedVersionSent) {
const svg = makeBadge(badgeData)
setCacheHeadersOnResponse(ask.res, badgeData.cacheLengthSeconds)
makeSend(format, ask.res, end)(svg)
}
},
cachingRequest
)
if (result && result.catch) {
result.catch(err => {
throw err
})
}
})
}
}
function clearRequestCache() {
requestCache.clear()
}
module.exports = {
handleRequest,
clearRequestCache,
// Expose for testing.
_requestCache: requestCache,
}

View File

@@ -1,344 +0,0 @@
'use strict'
const { expect } = require('chai')
const fetch = require('node-fetch')
const nock = require('nock')
const portfinder = require('portfinder')
const Camp = require('camp')
const analytics = require('../core/server/analytics')
const { makeBadgeData: getBadgeData } = require('./badge-data')
const {
handleRequest,
clearRequestCache,
_requestCache,
} = require('./request-handler')
async function performTwoRequests(baseUrl, first, second) {
expect((await fetch(`${baseUrl}${first}`)).ok).to.be.true
expect((await fetch(`${baseUrl}${second}`)).ok).to.be.true
}
function fakeHandler(queryParams, match, sendBadge, request) {
const [, someValue, format] = match
const badgeData = getBadgeData('testing', queryParams)
badgeData.text[1] = someValue
sendBadge(format, badgeData)
}
function createFakeHandlerWithCacheLength(cacheLengthSeconds) {
return function fakeHandler(queryParams, match, sendBadge, request) {
const [, someValue, format] = match
const badgeData = getBadgeData('testing', queryParams)
badgeData.text[1] = someValue
badgeData.cacheLengthSeconds = cacheLengthSeconds
sendBadge(format, badgeData)
}
}
function fakeHandlerWithNetworkIo(queryParams, match, sendBadge, request) {
const [, someValue, format] = match
request('https://www.google.com/foo/bar', (err, res, buffer) => {
const badgeData = getBadgeData('testing', queryParams)
if (err) {
badgeData.text[1] = err.prettyMessage
sendBadge(format, badgeData)
return
}
badgeData.text[1] = someValue
sendBadge(format, badgeData)
})
}
describe('The request handler', function() {
before(analytics.load)
let port, baseUrl
beforeEach(async function() {
port = await portfinder.getPortPromise()
baseUrl = `http://127.0.0.1:${port}`
})
let camp
beforeEach(function(done) {
camp = Camp.start({ port, hostname: '::' })
camp.on('listening', () => done())
})
afterEach(function(done) {
clearRequestCache()
if (camp) {
camp.close(() => done())
camp = null
}
})
const standardCacheHeaders = { defaultCacheLengthSeconds: 120 }
describe('the options object calling style', function() {
beforeEach(function() {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(standardCacheHeaders, { handler: fakeHandler })
)
})
it('should return the expected response', async function() {
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.ok).to.be.true
expect(await res.json()).to.deep.equal({ name: 'testing', value: '123' })
})
})
describe('the function shorthand calling style', function() {
beforeEach(function() {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(standardCacheHeaders, fakeHandler)
)
})
it('should return the expected response', async function() {
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.ok).to.be.true
expect(await res.json()).to.deep.equal({ name: 'testing', value: '123' })
})
})
describe('the response size limit', function() {
beforeEach(function() {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(standardCacheHeaders, {
handler: fakeHandlerWithNetworkIo,
fetchLimitBytes: 100,
})
)
})
it('should not throw an error if the response <= fetchLimitBytes', async function() {
nock('https://www.google.com')
.get('/foo/bar')
.once()
.reply(200, 'x'.repeat(100))
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.ok).to.be.true
expect(await res.json()).to.deep.equal({
name: 'testing',
value: '123',
})
})
it('should throw an error if the response is > fetchLimitBytes', async function() {
nock('https://www.google.com')
.get('/foo/bar')
.once()
.reply(200, 'x'.repeat(101))
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.ok).to.be.true
expect(await res.json()).to.deep.equal({
name: 'testing',
value: 'Maximum response size exceeded',
})
})
afterEach(function() {
nock.cleanAll()
})
})
describe('caching', function() {
describe('standard query parameters', function() {
let handlerCallCount
beforeEach(function() {
handlerCallCount = 0
})
function register({ cacheHeaderConfig }) {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(
cacheHeaderConfig,
(queryParams, match, sendBadge, request) => {
++handlerCallCount
fakeHandler(queryParams, match, sendBadge, request)
}
)
)
}
context('With standard cache settings', function() {
beforeEach(function() {
register({ cacheHeaderConfig: standardCacheHeaders })
})
it('should cache identical requests', async function() {
await performTwoRequests(
baseUrl,
'/testing/123.svg',
'/testing/123.svg'
)
expect(handlerCallCount).to.equal(1)
})
it('should differentiate known query parameters', async function() {
await performTwoRequests(
baseUrl,
'/testing/123.svg?label=foo',
'/testing/123.svg?label=bar'
)
expect(handlerCallCount).to.equal(2)
})
it('should ignore unknown query parameters', async function() {
await performTwoRequests(
baseUrl,
'/testing/123.svg?foo=1',
'/testing/123.svg?foo=2'
)
expect(handlerCallCount).to.equal(1)
})
})
it('should set the expires header to current time + defaultCacheLengthSeconds', async function() {
register({ cacheHeaderConfig: { defaultCacheLengthSeconds: 900 } })
const res = await fetch(`${baseUrl}/testing/123.json`)
const expectedExpiry = new Date(
+new Date(res.headers.get('date')) + 900000
).toGMTString()
expect(res.headers.get('expires')).to.equal(expectedExpiry)
expect(res.headers.get('cache-control')).to.equal('max-age=900')
})
it('should set the expected cache headers on cached responses', async function() {
register({ cacheHeaderConfig: { defaultCacheLengthSeconds: 900 } })
// Make first request.
await fetch(`${baseUrl}/testing/123.json`)
const res = await fetch(`${baseUrl}/testing/123.json`)
const expectedExpiry = new Date(
+new Date(res.headers.get('date')) + 900000
).toGMTString()
expect(res.headers.get('expires')).to.equal(expectedExpiry)
expect(res.headers.get('cache-control')).to.equal('max-age=900')
})
it('should let live service data override the default cache headers with longer value', async function() {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(
{ defaultCacheLengthSeconds: 300 },
(queryParams, match, sendBadge, request) => {
++handlerCallCount
createFakeHandlerWithCacheLength(400)(
queryParams,
match,
sendBadge,
request
)
}
)
)
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.headers.get('cache-control')).to.equal('max-age=400')
})
it('should not let live service data override the default cache headers with shorter value', async function() {
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(
{ defaultCacheLengthSeconds: 300 },
(queryParams, match, sendBadge, request) => {
++handlerCallCount
createFakeHandlerWithCacheLength(200)(
queryParams,
match,
sendBadge,
request
)
}
)
)
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.headers.get('cache-control')).to.equal('max-age=300')
})
it('should set the expires header to current time + maxAge', async function() {
register({ cacheHeaderConfig: { defaultCacheLengthSeconds: 0 } })
const res = await fetch(`${baseUrl}/testing/123.json?maxAge=3600`)
const expectedExpiry = new Date(
+new Date(res.headers.get('date')) + 3600000
).toGMTString()
expect(res.headers.get('expires')).to.equal(expectedExpiry)
expect(res.headers.get('cache-control')).to.equal('max-age=3600')
})
it('should ignore maxAge if maxAge < defaultCacheLengthSeconds', async function() {
register({ cacheHeaderConfig: { defaultCacheLengthSeconds: 600 } })
const res = await fetch(`${baseUrl}/testing/123.json?maxAge=300`)
const expectedExpiry = new Date(
+new Date(res.headers.get('date')) + 600000
).toGMTString()
expect(res.headers.get('expires')).to.equal(expectedExpiry)
expect(res.headers.get('cache-control')).to.equal('max-age=600')
})
it('should set Cache-Control: no-cache, no-store, must-revalidate if maxAge=0', async function() {
register({ cacheHeaderConfig: { defaultCacheLengthSeconds: 0 } })
const res = await fetch(`${baseUrl}/testing/123.json`)
expect(res.headers.get('expires')).to.equal(res.headers.get('date'))
expect(res.headers.get('cache-control')).to.equal(
'no-cache, no-store, must-revalidate'
)
})
describe('the cache key', function() {
beforeEach(function() {
register({ cacheHeaderConfig: standardCacheHeaders })
})
const expectedCacheKey = '/testing/123.json?colorB=123&label=foo'
it('should match expected and use canonical order - 1', async function() {
const res = await fetch(
`${baseUrl}/testing/123.json?colorB=123&label=foo`
)
expect(res.ok).to.be.true
expect(_requestCache.cache).to.have.keys(expectedCacheKey)
})
it('should match expected and use canonical order - 2', async function() {
const res = await fetch(
`${baseUrl}/testing/123.json?label=foo&colorB=123`
)
expect(res.ok).to.be.true
expect(_requestCache.cache).to.have.keys(expectedCacheKey)
})
})
})
describe('custom query parameters', function() {
let handlerCallCount
beforeEach(function() {
handlerCallCount = 0
camp.route(
/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(standardCacheHeaders, {
queryParams: ['foo'],
handler: (queryParams, match, sendBadge, request) => {
++handlerCallCount
fakeHandler(queryParams, match, sendBadge, request)
},
})
)
})
it('should differentiate them', async function() {
await performTwoRequests(
baseUrl,
'/testing/123.svg?foo=1',
'/testing/123.svg?foo=2'
)
expect(handlerCallCount).to.equal(2)
})
})
})
})

View File

@@ -1,54 +0,0 @@
'use strict'
const stream = require('stream')
const log = require('../core/server/log')
const svg2img = require('../gh-badges/lib/svg-to-img')
function streamFromString(str) {
const newStream = new stream.Readable()
newStream._read = () => {
newStream.push(str)
newStream.push(null)
}
return newStream
}
function makeSend(format, askres, end) {
if (format === 'svg') {
return res => sendSVG(res, askres, end)
} else if (format === 'json') {
return res => sendJSON(res, askres, end)
} else {
return res => sendOther(format, res, askres, end)
}
}
function sendSVG(res, askres, end) {
askres.setHeader('Content-Type', 'image/svg+xml;charset=utf-8')
end(null, { template: streamFromString(res) })
}
function sendOther(format, res, askres, end) {
askres.setHeader('Content-Type', `image/${format}`)
svg2img(res, format)
// This interacts with callback code and can't use async/await.
// eslint-disable-next-line promise/prefer-await-to-then
.then(data => {
end(null, { template: streamFromString(data) })
})
.catch(err => {
// This emits status code 200, though 500 would be preferable.
log.error('svg2img error', err)
end(null, { template: '500.html' })
})
}
function sendJSON(res, askres, end) {
askres.setHeader('Content-Type', 'application/json')
askres.setHeader('Access-Control-Allow-Origin', '*')
end(null, { template: streamFromString(res) })
}
module.exports = {
makeSend,
}