From 127b46aef820fe694ad9fd157ff99f2df5511411 Mon Sep 17 00:00:00 2001 From: Paul Melnikow Date: Thu, 30 Nov 2017 13:21:27 -0500 Subject: [PATCH] Github auth admin endpoint and logging (#1267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Periodically log github auth information - Tokens are hashed which reduces the security risk inherent in the logs - A consistent hash is used so tokens can be correlated across the three data structures and across the three servers - Add an admin endpoint for github auth information - Tokens are returned as-is to enable troubleshooting (e.g. comparing our reqRemaining to github’s) --- lib/github-auth.js | 60 +++++++++++++++++++++++++++++++ lib/server-config.js | 4 +++ package-lock.json | 85 +++++++++++++++++++++----------------------- package.json | 3 +- server.js | 11 ++++++ 5 files changed, 118 insertions(+), 45 deletions(-) diff --git a/lib/github-auth.js b/lib/github-auth.js index a6f6ddd252..98bd6bfd68 100644 --- a/lib/github-auth.js +++ b/lib/github-auth.js @@ -1,10 +1,12 @@ 'use strict'; +const crypto = require('crypto'); const log = require('./log'); const queryString = require('query-string'); const request = require('request'); const autosave = require('json-autosave'); const serverSecrets = require('./server-secrets'); +const mapKeys = require('lodash.mapkeys'); // This is an initial value which makes the code work while the initial data // is loaded. In the then() callback of scheduleAutosaving(), it's reassigned @@ -111,6 +113,22 @@ function setRoutes(server) { addGithubToken(data.token); end('Thanks!'); }); + + // Allow the admin to obtain the tokens for operational and debugging + // purposes. This could be used to: + // + // - Ensure tokens have been propagated to all servers + // - Debug GitHub badge failures + // + // The admin can authenticate with HTTP Basic Auth, with the shields secret + // in the username and an empty/any password. + server.ajax.on('github-auth/tokens', (json, end, ask) => { + if (! constEq(ask.username, serverSecrets.shieldsSecret)) { + // An unknown entity tries to connect. Let the connection linger for a minute. + return setTimeout(function() { end('Invalid secret.'); }, 10000); + } + end(getTokenDebugInfo({ sanitize: false })); + }); } function sendTokenToAllServers(token) { @@ -225,6 +243,47 @@ function rmGithubToken(token) { } } +// Convert an ES6 Map to an object. +function mapToObject(map) { + const result = {}; + for (const [k, v] of map) { + result[k] = v; + } + return result; +} + +// Compute a one-way hash of the input string. +function sha(str) { + return crypto.createHash('sha256') + .update(str, 'utf-8') + .digest('hex'); +} + +function getTokenDebugInfo(options) { + // Apply defaults. + const { sanitize } = Object.assign({ sanitize: true }, options); + + const unsanitized = { + tokens: githubUserTokens.data, + reqRemaining: mapToObject(reqRemaining), + reqReset: mapToObject(reqReset), + utcEpochSeconds: utcEpochSeconds(), + sanitized: false, + }; + + if (sanitize) { + return { + tokens: unsanitized.tokens.map(k => sha(k)), + reqRemaining: mapKeys(unsanitized.reqRemaining, (v, k) => sha(k)), + reqReset: mapKeys(unsanitized.reqReset, (v, k) => sha(k)), + utcEpochSeconds: unsanitized.utcEpochSeconds, + sanitized: true, + }; + } else { + return unsanitized; + } +} + // When a global gh_token is configured, use that in place of our shields.io // token-cycling logic. This produces more predictable behavior when a token // is provided, and more predictable failures if that token is exhausted. @@ -288,4 +347,5 @@ module.exports = { cancelAutosaving, request: githubRequest, setRoutes, + getTokenDebugInfo, }; diff --git a/lib/server-config.js b/lib/server-config.js index de41ef279b..f3b9000649 100644 --- a/lib/server-config.js +++ b/lib/server-config.js @@ -47,6 +47,10 @@ const config = { services: { github: { baseUri: process.env.GITHUB_URL || 'https://api.github.com', + debug: { + enabled: envFlag(process.env.GITHUB_DEBUG_ENABLED, true), + intervalSeconds: process.env.GITHUB_DEBUG_INTERVAL_SECONDS || 300, + }, }, }, }; diff --git a/package-lock.json b/package-lock.json index 793f67775a..724b54d8d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -245,6 +245,21 @@ } } }, + "JSONSelect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.4.0.tgz", + "integrity": "sha1-oI7cxn6z/L6Z7WMIVTRKDPKCu40=" + }, + "JSONStream": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.1.tgz", + "integrity": "sha1-cH92HgHa6eFvG8+TcDt4xwlmV5o=", + "dev": true, + "requires": { + "jsonparse": "1.3.1", + "through": "2.3.8" + } + }, "abab": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz", @@ -578,8 +593,7 @@ "assertion-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.2.tgz", - "integrity": "sha1-E8pRXYYgbaC6xm6DTdOX2HWBCUw=", - "dev": true + "integrity": "sha1-E8pRXYYgbaC6xm6DTdOX2HWBCUw=" }, "ast-transform": { "version": "0.0.0", @@ -2000,8 +2014,7 @@ "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, "check-node-version": { "version": "3.1.1", @@ -2727,8 +2740,8 @@ "integrity": "sha512-8od6g684Fhi5Vpp4ABRv/RBsW1AY6wSHbJHEK6FGTv+8jvAAnlABniZu/FVmX9TcirkHepaEsa1QGkRvbg0CKw==", "dev": true, "requires": { - "is-text-path": "1.0.1", "JSONStream": "1.3.1", + "is-text-path": "1.0.1", "lodash": "4.17.4", "meow": "3.7.0", "split2": "2.2.0", @@ -2982,7 +2995,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", - "dev": true, "requires": { "type-detect": "4.0.3" } @@ -5294,15 +5306,6 @@ } } }, - "string_decoder": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.1.tgz", - "integrity": "sha1-YuIA8DmVWmgQ2N8KM//A8BNmLZg=", - "dev": true, - "requires": { - "safe-buffer": "5.0.1" - } - }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -5314,6 +5317,15 @@ "strip-ansi": "3.0.1" } }, + "string_decoder": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.1.tgz", + "integrity": "sha1-YuIA8DmVWmgQ2N8KM//A8BNmLZg=", + "dev": true, + "requires": { + "safe-buffer": "5.0.1" + } + }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", @@ -5522,8 +5534,7 @@ "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", - "dev": true + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, "get-pkg-repo": { "version": "1.4.0", @@ -6605,12 +6616,12 @@ "resolved": "https://registry.npmjs.org/jison/-/jison-0.4.13.tgz", "integrity": "sha1-kEFwfWIkE2f1iDRTK58ZwsNvrHg=", "requires": { + "JSONSelect": "0.4.0", "cjson": "0.2.1", "ebnf-parser": "0.1.10", "escodegen": "0.0.21", "esprima": "1.0.4", "jison-lex": "0.2.1", - "JSONSelect": "0.4.0", "lex-parser": "0.1.4", "nomnom": "1.5.2" }, @@ -6862,21 +6873,6 @@ "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk=", "dev": true }, - "JSONSelect": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.4.0.tgz", - "integrity": "sha1-oI7cxn6z/L6Z7WMIVTRKDPKCu40=" - }, - "JSONStream": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.1.tgz", - "integrity": "sha1-cH92HgHa6eFvG8+TcDt4xwlmV5o=", - "dev": true, - "requires": { - "jsonparse": "1.3.1", - "through": "2.3.8" - } - }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -7204,6 +7200,11 @@ "integrity": "sha1-oIYCrBLk+4P5H8H7ejYKTZujUgU=", "dev": true }, + "lodash.mapkeys": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.mapkeys/-/lodash.mapkeys-4.6.0.tgz", + "integrity": "sha1-3yz6Ix18V8eorQA6va1dc9PqUZU=" + }, "lodash.restparam": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz", @@ -8007,7 +8008,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/node-env-flag/-/node-env-flag-0.1.0.tgz", "integrity": "sha1-vn1DxRHCeBqg+GiOfY9Cb3WSS8U=", - "dev": true, "requires": { "chai": "4.1.2" }, @@ -8016,7 +8016,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", - "dev": true, "requires": { "assertion-error": "1.0.2", "check-error": "1.0.2", @@ -10334,8 +10333,7 @@ "pathval": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", - "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", - "dev": true + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=" }, "pbkdf2": { "version": "3.0.14", @@ -11909,11 +11907,6 @@ "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, "string-hash": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string-hash/-/string-hash-1.1.1.tgz", @@ -11956,6 +11949,11 @@ } } }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", @@ -12628,8 +12626,7 @@ "type-detect": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.3.tgz", - "integrity": "sha1-Dj8mcLRAmbC0bChNE2p+9Jx0wuo=", - "dev": true + "integrity": "sha1-Dj8mcLRAmbC0bChNE2p+9Jx0wuo=" }, "typedarray": { "version": "0.0.6", diff --git a/package.json b/package.json index 9caf3a545c..06f099c3d7 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,9 @@ "json-autosave": "~1.1.2", "jsonpath": "~0.2.12", "lodash.countby": "^4.6.0", + "lodash.mapkeys": "^4.6.0", "moment": "^2.19.3", + "node-env-flag": "^0.1.0", "pdfkit": "~0.8.0", "pretty-bytes": "^4.0.2", "query-string": "^5.0.0", @@ -113,7 +115,6 @@ "mocha": "^4.0.1", "next": "^4.1.4", "nock": "^9.0.13", - "node-env-flag": "^0.1.0", "node-fetch": "^1.6.3", "nyc": "^11.2.1", "opn-cli": "^3.1.0", diff --git a/server.js b/server.js index b953b970a6..74153a07d5 100644 --- a/server.js +++ b/server.js @@ -113,6 +113,10 @@ function reset() { function stop(callback) { githubAuth.cancelAutosaving(); + if (githubDebugInterval) { + clearInterval(githubDebugInterval); + githubDebugInterval = null; + } analytics.cancelAutosaving(); camp.close(callback); } @@ -134,6 +138,13 @@ if (serverSecrets && serverSecrets.gh_client_id) { githubAuth.setRoutes(camp); } +let githubDebugInterval; +if (config.services.github.debug.enabled) { + githubDebugInterval = setInterval(() => { + log(githubAuth.getTokenDebugInfo()); + }, 1000 * config.services.github.debug.intervalSeconds); +} + suggest.setRoutes(config.cors.allowedOrigin, camp); camp.notfound(/\.(svg|png|gif|jpg|json)/, function(query, match, end, request) {