'use strict'; const crypto = require('crypto'); const log = require('./log'); const queryString = require('query-string'); const request = require('request'); const autosave = require('json-autosave'); const serverSecrets = require('./server-secrets'); const mapKeys = require('lodash.mapkeys'); // This is an initial value which makes the code work while the initial data // is loaded. In the then() callback of scheduleAutosaving(), it's reassigned // with a JsonSave object. let githubUserTokens = {data: []}; function scheduleAutosaving() { const githubUserTokensFile = './private/github-user-tokens.json'; autosave(githubUserTokensFile, {data: []}).then(save => { githubUserTokens = save; for (let i = 0; i < githubUserTokens.data.length; i++) { addGithubToken(githubUserTokens.data[i]); } // Personal tokens allow access to GitHub private repositories. // You can manage your personal GitHub token at // . if (serverSecrets && serverSecrets.gh_token) { addGithubToken(serverSecrets.gh_token); } }).catch(e => { console.error('Could not create ' + githubUserTokensFile); }); } function cancelAutosaving() { if (githubUserTokens.stop) { githubUserTokens.stop(); githubUserTokens.save(); githubUserTokens = {data: []}; } } function setRoutes(server) { const baseUrl = process.env.BASE_URL || 'https://img.shields.io'; server.route(/^\/github-auth$/, function(data, match, end, ask) { if (!(serverSecrets && serverSecrets.gh_client_id)) { return end('This server is missing GitHub client secrets.'); } const query = queryString.stringify({ client_id: serverSecrets.gh_client_id, redirect_uri: baseUrl + '/github-auth/done', }); ask.res.statusCode = 302; // Found. ask.res.setHeader('Location', 'https://github.com/login/oauth/authorize?' + query); end(''); }); server.route(/^\/github-auth\/done$/, function(data, match, end, ask) { if (!(serverSecrets && serverSecrets.gh_client_id && serverSecrets.gh_client_secret)) { return end('This server is missing GitHub client secrets.'); } if (!data.code) { log(`GitHub OAuth data.code: ${JSON.stringify(data)}`); return end('GitHub OAuth authentication failed to provide a code.'); } const options = { url: 'https://github.com/login/oauth/access_token', headers: { 'Content-type': 'application/x-www-form-urlencoded;charset=UTF-8', 'User-Agent': 'Shields.io', }, form: queryString.stringify({ client_id: serverSecrets.gh_client_id, client_secret: serverSecrets.gh_client_secret, code: data.code, }), method: 'POST', }; request(options, function(err, res, body) { if (err != null) { return end('The connection to GitHub failed.'); } let content; try { content = queryString.parse(body); } catch(e) { return end('The GitHub OAuth token could not be parsed.'); } const token = content.access_token; if (!token) { return end('The GitHub OAuth process did not return a user token.'); } ask.res.setHeader('Content-Type', 'text/html'); end('

Shields.io has received your app-specific GitHub user token. ' + 'You can revoke it by going to ' + 'GitHub.

' + '

Until you do, you have now increased the rate limit for GitHub ' + 'requests going through Shields.io. GitHub-related badges are ' + 'therefore more robust.

' + '

Thanks for contributing to a smoother experience for ' + 'everyone!

' + '

Back to the website

'); sendTokenToAllServers(token) .catch(function(e) { console.error('GitHub user token transmission failed:', e); }); }); }); server.route(/^\/github-auth\/add-token$/, function(data, match, end, ask) { if (!constEq(data.shieldsSecret, serverSecrets.shieldsSecret)) { // An unknown entity tries to connect. Let the connection linger for 10s. return setTimeout(function() { end('Invalid secret.'); }, 10000); } addGithubToken(data.token); end('Thanks!'); }); // Allow the admin to obtain the tokens for operational and debugging // purposes. This could be used to: // // - Ensure tokens have been propagated to all servers // - Debug GitHub badge failures // // The admin can authenticate with HTTP Basic Auth, with an empty/any // username and the shields secret in the password and an empty/any // password. // // e.g. // curl -u ':very-very-secret' 'https://example.com/$github-auth/tokens' server.ajax.on('github-auth/tokens', (json, end, ask) => { if (! constEq(ask.password, serverSecrets.shieldsSecret)) { // An unknown entity tries to connect. Let the connection linger for a minute. return setTimeout(function() { end('Invalid secret.'); }, 10000); } end(getTokenDebugInfo({ sanitize: false })); }); } function sendTokenToAllServers(token) { const ips = serverSecrets.shieldsIps; return Promise.all(ips.map(function(ip) { return new Promise(function(resolve, reject) { const options = { url: 'https://' + ip + '/github-auth/add-token', method: 'POST', form: { shieldsSecret: serverSecrets.shieldsSecret, token: token, }, // We target servers by IP, and we use HTTPS. Assuming that // 1. Internet routers aren't hacked, and // 2. We don't unknowingly lose our IP to someone else, // we're not leaking people's and our information. // (If we did, it would have no impact, as we only ask for a token, // no GitHub scope. The malicious entity would only be able to use // our rate limit pool.) // FIXME: use letsencrypt. strictSSL: false, }; request(options, function(err, res, body) { if (err != null) { return reject(err); } resolve(); }); }); })); } // Track rate limit requests remaining. // Ideally, we would want priority queues here. const reqRemaining = new Map(); // From token to requests remaining. const reqReset = new Map(); // From token to timestamp. // token: client token as a string. // reqs: number of requests remaining. // reset: timestamp when the number of remaining requests is reset. function setReqRemaining(token, reqs, reset) { reqRemaining.set(token, reqs); reqReset.set(token, reset); } function rmReqRemaining(token) { reqRemaining.delete(token); reqReset.delete(token); } function utcEpochSeconds() { return ((Date.now() / 1000) >>> 0); } const userTokenRateLimit = 12500; // Return false if the token cannot reasonably be expected to perform // a GitHub request. function isTokenUsable(token, now) { const reqs = reqRemaining.get(token); const reset = reqReset.get(token); // We don't want to empty more than 3/4 of a user's rate limit. const hasRemainingReqs = reqs > (userTokenRateLimit / 4); const isBeyondRateLimitReset = reset < now; return hasRemainingReqs || isBeyondRateLimitReset; } // Return a list of tokens (as strings) which can be used for a GitHub request, // with a reasonable chance that the request will succeed. function usableTokens() { const now = utcEpochSeconds(); return githubUserTokens.data.filter(function(token) { return isTokenUsable(token, now); }); } // Retrieve a user token if there is one for which we believe there are requests // remaining. Return undefined if we could not find one. function getReqRemainingToken() { // Go through the user tokens. // Among usable ones, use the one with the highest number of remaining // requests. const tokens = usableTokens(); let highestReq = -1; let highestToken; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; const reqs = reqRemaining.get(token); if (reqs > highestReq) { highestReq = reqs; highestToken = token; } } return highestToken; } function addGithubToken(token) { // A reset date of 0 has to be in the past. setReqRemaining(token, userTokenRateLimit, 0); // Insert it only if it is not registered yet. if (githubUserTokens.data.indexOf(token) === -1) { githubUserTokens.data.push(token); } } function rmGithubToken(token) { rmReqRemaining(token); // Remove it only if it is in there. const idx = githubUserTokens.data.indexOf(token); if (idx >= 0) { githubUserTokens.data.splice(idx, 1); } } // Convert an ES6 Map to an object. function mapToObject(map) { const result = {}; for (const [k, v] of map) { result[k] = v; } return result; } // Compute a one-way hash of the input string. function sha(str) { return crypto.createHash('sha256') .update(str, 'utf-8') .digest('hex'); } function getTokenDebugInfo(options) { // Apply defaults. const { sanitize } = Object.assign({ sanitize: true }, options); const unsanitized = { tokens: githubUserTokens.data, reqRemaining: mapToObject(reqRemaining), reqReset: mapToObject(reqReset), utcEpochSeconds: utcEpochSeconds(), sanitized: false, }; if (sanitize) { return { tokens: unsanitized.tokens.map(k => sha(k)), reqRemaining: mapKeys(unsanitized.reqRemaining, (v, k) => sha(k)), reqReset: mapKeys(unsanitized.reqReset, (v, k) => sha(k)), utcEpochSeconds: unsanitized.utcEpochSeconds, sanitized: true, }; } else { return unsanitized; } } // When a global gh_token is configured, use that in place of our shields.io // token-cycling logic. This produces more predictable behavior when a token // is provided, and more predictable failures if that token is exhausted. // // You can manage your personal GitHub token at https://github.com/settings/tokens const globalToken = (serverSecrets || {}).gh_token; // Act like request(), but tweak headers and query to avoid hitting a rate // limit. function githubRequest(request, url, query, cb) { query = query || {}; // A special User-Agent is required: // http://developer.github.com/v3/#user-agent-required const headers = { 'User-Agent': 'Shields.io', 'Accept': 'application/vnd.github.v3+json', }; const githubToken = globalToken === undefined ? getReqRemainingToken() : globalToken; if (githubToken != null) { // Typically, GitHub user tokens grants us 12500 req/hour. headers['Authorization'] = 'token ' + githubToken; } else if (serverSecrets && serverSecrets.gh_client_id) { // Using our OAuth App secret grants us 5000 req/hour // instead of the standard 60 req/hour. query.client_id = serverSecrets.gh_client_id; query.client_secret = serverSecrets.gh_client_secret; } const qs = queryString.stringify(query); if (qs) { url += '?' + qs; } request(url, {headers: headers}, function(err, res, buffer) { if (globalToken !== null && githubToken !== null && err === null) { if (res.statusCode === 401) { // Unauthorized. rmGithubToken(githubToken); } else { const remaining = +res.headers['x-ratelimit-remaining']; // reset is in UTC epoch seconds. const reset = +res.headers['x-ratelimit-reset']; setReqRemaining(githubToken, remaining, reset); if (remaining === 0) { return; } // Hope for the best in the cache. } } cb(err, res, buffer); }); } function constEq(a, b) { if (a.length !== b.length) { return false; } let zero = 0; for (let i = 0; i < a.length; i++) { zero |= a.charCodeAt(i) ^ b.charCodeAt(i); } return (zero === 0); } module.exports = { scheduleAutosaving, cancelAutosaving, request: githubRequest, setRoutes, getTokenDebugInfo, };