ComputerSurge/shields
3
0
Fork 0
Code Pull Requests 1 Activity

Correct cache behavior for custom query parameters (#1186)

Browse Source 
I developed this for #820 to provide the correct cache behavior when a service wants to use custom parameters from the query string.

For safety, only the declared parameters (and the global parameters) are provided to the service. Consequently, failure to declare a parameter results in the parameter not working at all (undesirable, but easy to debug) rather than indeterminate behavior that depends on the cache state (undesirable, but hard to debug).
This commit is contained in:
Paul Melnikow
2017-10-24 23:00:23 -04:00
committed by GitHub
parent 7153490ef9
commit 446d4ce21e
7 changed files with 239 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CONTRIBUTING.md
View File

@@ -64,9 +64,8 @@ Badge guidelines
- The left-hand side of a badge should not advertize. It should be a noun
describing succinctly the meaning of the right-hand-side data.
- New query parameters (such as `?label=` or `?style=`) should apply to any
requested badge. They must be registered in the cache (see `LruCache` in
`server.js`).
- New query parameters must be declared by the service. See
`request-handler.js`.
- The format of new badges should be of the form
`/VENDOR/SUBVENDOR-BADGE-SPECIFIC/PARAMETERS.format`. For instance,
`https://img.shields.io/gitter/room/nwjs/nw.js.svg`. The vendor is gitter, the

10
lib/github-auth.js
View File

@@ -1,7 +1,7 @@
'use strict';
const log = require('./log');
const querystring = require('querystring');
const queryString = require('query-string');
const request = require('request');
const autosave = require('json-autosave');
let serverSecrets;
@@ -48,7 +48,7 @@ function setRoutes(server) {
if (!(serverSecrets && serverSecrets.gh_client_id)) {
return end('This server is missing GitHub client secrets.');
}
const query = querystring.stringify({
const query = queryString.stringify({
client_id: serverSecrets.gh_client_id,
redirect_uri: baseUrl + '/github-auth/done',
});
@@ -71,7 +71,7 @@ function setRoutes(server) {
'Content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
'User-Agent': 'Shields.io',
},
form: querystring.stringify({
form: queryString.stringify({
client_id: serverSecrets.gh_client_id,
client_secret: serverSecrets.gh_client_secret,
code: data.code,
@@ -82,7 +82,7 @@ function setRoutes(server) {
if (err != null) { return end('The connection to GitHub failed.'); }
let content;
try {
content = querystring.parse(body);
content = queryString.parse(body);
} catch(e) { return end('The GitHub OAuth token could not be parsed.'); }
const token = content.access_token;
if (!token) {
@@ -251,7 +251,7 @@ function githubRequest(request, url, query, cb) {
query.client_secret = serverSecrets.gh_client_secret;
}
const qs = querystring.stringify(query);
const qs = queryString.stringify(query);
if (qs) { url += '?' + qs; }
request(url, {headers: headers}, function(err, res, buffer) {
if (githubToken != null && err === null) {

61
lib/request-handler.js
View File

@@ -9,6 +9,7 @@ const log = require('./log');
const LruCache = require('./lru-cache');
const analytics = require('./analytics');
const { makeSend } = require('./result-sender');
const queryString = require('query-string');
// We avoid calling the vendor's server for computation of the information in a
// number of badges.
@@ -32,7 +33,47 @@ vendorDomain.on('error', err => {
log.error('Vendor hook error:', err.stack);
});
function handleRequest (vendorRequestHandler) {
// These query parameters are available to any badge. For the most part they
// are used by makeBadgeData (see `lib/badge-data.js`) and related functions.
const globalQueryParams = new Set([
'label',
'style',
'link',
'logo',
'logoWidth',
'link',
'colorA',
'colorB',
]);
function flattenQueryParams(queryParams) {
const union = new Set(globalQueryParams);
(queryParams || []).forEach(name => {
union.add(name);
});
return Array.from(union).sort();
}
// handlerOptions can contain:
// - handler: The service's request handler function
// - queryParams: An array of the field names of any custom query parameters
// the service uses
//
// For safety, the service must declare the query parameters it wants to use.
// Only the declared parameters (and the global parameters) are provided to
// the service. Consequently, failure to declare a parameter results in the
// parameter not working at all (which is undesirable, but easy to debug)
// rather than indeterminate behavior that depends on the cache state
// (undesirable and hard to debug).
//
// Pass just the handler function as shorthand.
function handleRequest (handlerOptions) {
if (typeof handlerOptions === 'function') {
handlerOptions = { handler: handlerOptions }
}
const allowedKeys = flattenQueryParams(handlerOptions.queryParams);
return (queryParams, match, end, ask) => {
if (queryParams.maxAge !== undefined && /^[0-9]+$/.test(queryParams.maxAge)) {
ask.res.setHeader('Cache-Control', 'max-age=' + queryParams.maxAge);
@@ -48,10 +89,16 @@ function handleRequest (vendorRequestHandler) {
analytics.noteRequest(queryParams, match);
const cacheIndex = match[0] + '?label=' + queryParams.label + '&style=' + queryParams.style
+ '&logo=' + queryParams.logo + '&logoWidth=' + queryParams.logoWidth
+ '&link=' + JSON.stringify(queryParams.link) + '&colorA=' + queryParams.colorA
+ '&colorB=' + queryParams.colorB;
const filteredQueryParams = {};
allowedKeys.forEach(key => {
filteredQueryParams[key] = queryParams[key];
});
// Use sindresorhus query-string because it sorts the keys, whereas the
// builtin querystring module relies on the iteration order.
const stringified = queryString.stringify(filteredQueryParams);
const cacheIndex = `match[0]?${stringified}`;
// Should we return the data right away?
const cached = requestCache.get(cacheIndex);
let cachedVersionSent = false;
@@ -77,7 +124,7 @@ function handleRequest (vendorRequestHandler) {
return;
}
ask.res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
const badgeData = getBadgeData('vendor', queryParams);
const badgeData = getBadgeData('vendor', filteredQueryParams);
badgeData.text[1] = 'unresponsive';
let extension;
try {
@@ -117,7 +164,7 @@ function handleRequest (vendorRequestHandler) {
}
vendorDomain.run(() => {
vendorRequestHandler(queryParams, match, function sendBadge(format, badgeData) {
handlerOptions.handler(filteredQueryParams, match, function sendBadge(format, badgeData) {
if (serverUnresponsive) { return; }
clearTimeout(serverResponsive);
// Check for a change in the data.

115
lib/request-handler.spec.js
View File

@@ -24,6 +24,13 @@ function performTwoRequests (first, second) {
});
}
function fakeHandler(queryParams, match, sendBadge, request) {
const [, someValue, format] = match;
const badgeData = getBadgeData('testing', queryParams);
badgeData.text[1] = someValue;
sendBadge(format, badgeData);
}
describe('The request handler', function() {
before(analytics.load);
@@ -40,36 +47,94 @@ describe('The request handler', function() {
}
});
let handlerCallCount;
beforeEach(function () {
handlerCallCount = 0;
camp.route(/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest((queryParams, match, sendBadge, request) => {
++handlerCallCount;
const [, someValue, format] = match;
const badgeData = getBadgeData('testing', queryParams);
badgeData.text[1] = someValue;
sendBadge(format, badgeData);
}));
});
describe('the options object calling style', function() {
beforeEach(function () {
camp.route(/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest({ handler: fakeHandler }));
});
it('should cache identical requests', function () {
return performTwoRequests('/testing/123.svg', '/testing/123.svg').then(() => {
assert.equal(handlerCallCount, 1);
it('should return the expected response', function () {
return fetch(`${baseUri}/testing/123.json`)
.then(res => {
assert.ok(res.ok);
return res.json();
}).then(json => {
assert.deepEqual(json, { name: 'testing', value: '123' });
});
});
});
it('should differentiate known query parameters', function () {
return performTwoRequests(
'/testing/123.svg?label=foo',
'/testing/123.svg?label=bar'
).then(() => { assert.equal(handlerCallCount, 2); });
describe('the function shorthand calling style', function() {
beforeEach(function () {
camp.route(/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest(fakeHandler));
});
it('should return the expected response', function () {
return fetch(`${baseUri}/testing/123.json`)
.then(res => {
assert.ok(res.ok);
return res.json();
}).then(json => {
assert.deepEqual(json, { name: 'testing', value: '123' });
});
});
});
it('should ignore unknown query parameters', function () {
return performTwoRequests(
'/testing/123.svg?foo=1',
'/testing/123.svg?foo=2'
).then(() => { assert.equal(handlerCallCount, 1); });
describe('caching', function () {
describe('standard query parameters', function () {
let handlerCallCount;
beforeEach(function () {
handlerCallCount = 0;
camp.route(/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest((queryParams, match, sendBadge, request) => {
++handlerCallCount;
fakeHandler(queryParams, match, sendBadge, request);
}));
});
it('should cache identical requests', function () {
return performTwoRequests('/testing/123.svg', '/testing/123.svg').then(() => {
assert.equal(handlerCallCount, 1);
});
});
it('should differentiate known query parameters', function () {
return performTwoRequests(
'/testing/123.svg?label=foo',
'/testing/123.svg?label=bar'
).then(() => { assert.equal(handlerCallCount, 2); });
});
it('should ignore unknown query parameters', function () {
return performTwoRequests(
'/testing/123.svg?foo=1',
'/testing/123.svg?foo=2'
).then(() => { assert.equal(handlerCallCount, 1); });
});
});
describe('custom query parameters', function() {
let handlerCallCount;
beforeEach(function () {
handlerCallCount = 0;
camp.route(/^\/testing\/([^/]+)\.(svg|png|gif|jpg|json)$/,
handleRequest({
queryParams: ['foo'],
handler: (queryParams, match, sendBadge, request) => {
++handlerCallCount;
fakeHandler(queryParams, match, sendBadge, request);
},
}))
});
it('should differentiate them', function () {
return performTwoRequests(
'/testing/123.svg?foo=1',
'/testing/123.svg?foo=2'
).then(() => { assert.equal(handlerCallCount, 2); });
});
});
});
});

20
package-lock.json generated
View File

@@ -1665,6 +1665,11 @@
"integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
"dev": true
},
"decode-uri-component": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
},
"deep-eql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz",
@@ -7015,6 +7020,16 @@
"integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=",
"dev": true
},
"query-string": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/query-string/-/query-string-5.0.1.tgz",
"integrity": "sha512-aM+MkQClojlNiKkO09tiN2Fv8jM/L7GWIjG2liWeKljlOdOPNWr+bW3KQ+w5V/uKprpezC7fAsAMsJtJ+2rLKA==",
"requires": {
"decode-uri-component": "0.2.0",
"object-assign": "4.1.1",
"strict-uri-encode": "1.1.0"
}
},
"quote-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/quote-stream/-/quote-stream-1.0.2.tgz",
@@ -7763,6 +7778,11 @@
}
}
},
"strict-uri-encode": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
"integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM="
},
"string-width": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",

1
package.json
View File

@@ -31,6 +31,7 @@
"moment": "^2.18.1",
"pdfkit": "~0.8.0",
"pretty-bytes": "^4.0.2",
"query-string": "^5.0.0",
"redis": "~2.6.2",
"request": "~2.83.0",
"semver": "~5.4.1",

128
server.js
View File

@@ -24,7 +24,7 @@ var tryUrl = require('url').format({
var log = require('./lib/log.js');
var badge = require('./lib/badge.js');
var githubAuth = require('./lib/github-auth');
var querystring = require('querystring');
var queryString = require('query-string');
var prettyBytes = require('pretty-bytes');
var xml2js = require('xml2js');
var serverSecrets = require('./lib/server-secrets');
@@ -2458,7 +2458,7 @@ cache(function(data, match, sendBadge, request) {
apiUrl = `https://codecov.io/${userRepo}/graphs/badge.txt`;
}
if (token) {
apiUrl += '?' + querystring.stringify({ token });
apiUrl += '?' + queryString.stringify({ token });
}
var badgeData = getBadgeData('coverage', data);
request(apiUrl, function(err, res, body) {
@@ -2927,7 +2927,7 @@ cache(function(data, match, sendBadge, request) {
if (branch) {
queryParams.branch = branch;
}
var query = querystring.stringify(queryParams);
var query = queryString.stringify(queryParams);
var url = 'https://www.codacy.com/project/badge/grade/' + projectId + '?' + query;
var badgeData = getBadgeData('code quality', data);
fetchFromSvg(request, url, function(err, res) {
@@ -3012,7 +3012,7 @@ cache(function(data, match, sendBadge, request) {
if (branch) {
queryParams.branch = branch;
}
var query = querystring.stringify(queryParams);
var query = queryString.stringify(queryParams);
var url = 'https://www.codacy.com/project/badge/coverage/' + projectId + '?' + query;
var badgeData = getBadgeData('coverage', data);
fetchFromSvg(request, url, function(err, res) {
@@ -4568,7 +4568,7 @@ cache(function(data, match, sendBadge, request) {
};
var badgeData = getBadgeData('role', data);
request(options, function(err, res, json) {
if (res && (res.statusCode === 404 || data.state === null)) {
if (res && (res.statusCode === 404 || json.state === null)) {
badgeData.text[1] = 'not found';
sendBadge(format, badgeData);
return;
@@ -5101,7 +5101,7 @@ cache(function(data, match, sendBadge, request) {
'request[slug]': match[1] // eg, `hestia`.
};
var format = match[2];
var apiUrl = 'https://api.wordpress.org/themes/info/1.1/?' + querystring.stringify(queryParams);
var apiUrl = 'https://api.wordpress.org/themes/info/1.1/?' + queryString.stringify(queryParams);
var badgeData = getBadgeData('rating', data);
request(apiUrl, function(err, res, buffer) {
if (err != null) {
@@ -5141,7 +5141,7 @@ cache(function(data, match, sendBadge, request) {
'request[slug]': match[1] // eg, `hestia`.
};
var format = match[2];
var apiUrl = 'https://api.wordpress.org/themes/info/1.1/?' + querystring.stringify(queryParams);
var apiUrl = 'https://api.wordpress.org/themes/info/1.1/?' + queryString.stringify(queryParams);
var badgeData = getBadgeData('downloads', data);
request(apiUrl, function(err, res, buffer) {
if (err != null) {
@@ -5449,48 +5449,51 @@ cache(function(data, match, sendBadge, request) {
}));
camp.route(/^\/dockbit\/([A-Za-z0-9-_]+)\/([A-Za-z0-9-_]+)\.(svg|png|gif|jpg|json)$/,
cache(function(data, match, sendBadge, request) {
const org = match[1];
const pipeline = match[2];
const format = match[3];
cache({
queryParams: ['token'],
handler: (data, match, sendBadge, request) => {
const org = match[1];
const pipeline = match[2];
const format = match[3];
const token = data.token;
const badgeData = getBadgeData('deploy', data);
const apiUrl = `https://dockbit.com/${org}/${pipeline}/status/${token}`;
const token = data.token;
const badgeData = getBadgeData('deploy', data);
const apiUrl = `https://dockbit.com/${org}/${pipeline}/status/${token}`;
var dockbitStates = {
success: '#72BC37',
failure: '#F55C51',
error: '#F55C51',
working: '#FCBC41',
pending: '#CFD0D7',
rejected: '#CFD0D7'
};
var dockbitStates = {
success: '#72BC37',
failure: '#F55C51',
error: '#F55C51',
working: '#FCBC41',
pending: '#CFD0D7',
rejected: '#CFD0D7'
};
request(apiUrl, {json: true}, function(err, res, data) {
try {
if (res && (res.statusCode === 404 || data.state === null)) {
badgeData.text[1] = 'not found';
sendBadge(format, badgeData);
return;
}
if (!res || err !== null || res.statusCode !== 200) {
badgeData.text[1] = 'inaccessible';
sendBadge(format, badgeData);
return;
}
badgeData.text[1] = data.state;
badgeData.colorB = dockbitStates[data.state];
request(apiUrl, {json: true}, function(err, res, data) {
try {
if (res && (res.statusCode === 404 || data.state === null)) {
badgeData.text[1] = 'not found';
sendBadge(format, badgeData);
return;
}
if (!res || err !== null || res.statusCode !== 200) {
badgeData.text[1] = 'inaccessible';
catch(e) {
badgeData.text[1] = 'invalid';
sendBadge(format, badgeData);
return;
}
badgeData.text[1] = data.state;
badgeData.colorB = dockbitStates[data.state];
sendBadge(format, badgeData);
}
catch(e) {
badgeData.text[1] = 'invalid';
sendBadge(format, badgeData);
}
});
});
},
}));
// CircleCI build integration.
@@ -5522,7 +5525,7 @@ cache(function(data, match, sendBadge, request) {
}
// Apprend query params to API URL
apiUrl += '?' + querystring.stringify(queryParams);
apiUrl += '?' + queryString.stringify(queryParams);
var badgeData = getBadgeData('build', data);
request(apiUrl, {json:true}, function(err, res, data) {
@@ -6275,26 +6278,29 @@ cache(function(data, match, sendBadge, request) {
// bitHound integration
camp.route(/^\/bithound\/(code\/|dependencies\/|devDependencies\/)?(.+?)\.(svg|png|gif|jpg|json)$/,
cache(function(data, match, sendBadge, request) {
var type = match[1].slice(0, -1);
var userRepo = match[2]; // eg, `github/rexxars/sse-channel`.
var format = match[3];
var apiUrl = 'https://www.bithound.io/api/' + userRepo + '/badge/' + type;
var badgeData = getBadgeData(type === 'devDependencies' ? 'dev dependencies' : type, data);
cache({
queryParams: ['color'], // argh.
handler: (data, match, sendBadge, request) => {
var type = match[1].slice(0, -1);
var userRepo = match[2]; // eg, `github/rexxars/sse-channel`.
var format = match[3];
var apiUrl = 'https://www.bithound.io/api/' + userRepo + '/badge/' + type;
var badgeData = getBadgeData(type === 'devDependencies' ? 'dev dependencies' : type, data);
request(apiUrl, { headers: { 'Accept': 'application/json' } }, function(err, res, buffer) {
try {
var data = JSON.parse(buffer);
badgeData.text[1] = data.label;
badgeData.colorscheme = null;
badgeData.colorB = '#' + data.color;
sendBadge(format, badgeData);
request(apiUrl, { headers: { 'Accept': 'application/json' } }, function(err, res, buffer) {
try {
var data = JSON.parse(buffer);
badgeData.text[1] = data.label;
badgeData.colorscheme = null;
badgeData.colorB = '#' + data.color;
sendBadge(format, badgeData);
} catch(e) {
badgeData.text[1] = 'invalid';
sendBadge(format, badgeData);
}
});
} catch(e) {
badgeData.text[1] = 'invalid';
sendBadge(format, badgeData);
}
});
},
}));
// Waffle.io integration