Rewrite and test Github auth logic, separating standard and search quota (#1205)
The end of an era.
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
'use strict'
|
||||
|
||||
const fsos = require('fsos')
|
||||
const githubAuth = require('./github-auth')
|
||||
const TokenPersistence = require('./token-persistence')
|
||||
|
||||
class FsTokenPersistence extends TokenPersistence {
|
||||
@@ -23,21 +22,28 @@ class FsTokenPersistence extends TokenPersistence {
|
||||
}
|
||||
|
||||
const tokens = JSON.parse(contents)
|
||||
tokens.forEach(tokenString => {
|
||||
githubAuth.addGithubToken(tokenString)
|
||||
})
|
||||
this._tokens = new Set(tokens)
|
||||
return tokens
|
||||
}
|
||||
|
||||
async save() {
|
||||
const tokens = githubAuth.getAllTokenIds()
|
||||
const tokens = Array.from(this._tokens)
|
||||
await fsos.set(this.path, JSON.stringify(tokens))
|
||||
}
|
||||
|
||||
async onTokenAdded(token) {
|
||||
if (!this._tokens) {
|
||||
throw Error('initialize() has not been called')
|
||||
}
|
||||
this._tokens.add(token)
|
||||
await this.save()
|
||||
}
|
||||
|
||||
async onTokenRemoved(token) {
|
||||
if (!this._tokens) {
|
||||
throw Error('initialize() has not been called')
|
||||
}
|
||||
this._tokens.delete(token)
|
||||
await this.save()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,8 @@ const tmp = require('tmp')
|
||||
const readFile = require('fs-readfile-promise')
|
||||
const { expect } = require('chai')
|
||||
const FsTokenPersistence = require('./fs-token-persistence')
|
||||
const githubAuth = require('./github-auth')
|
||||
|
||||
describe('File system token persistence', function() {
|
||||
beforeEach(githubAuth.removeAllTokens)
|
||||
afterEach(githubAuth.removeAllTokens)
|
||||
|
||||
let path, persistence
|
||||
beforeEach(function() {
|
||||
path = tmp.tmpNameSync()
|
||||
@@ -19,8 +15,8 @@ describe('File system token persistence', function() {
|
||||
|
||||
context('when the file does not exist', function() {
|
||||
it('does nothing', async function() {
|
||||
await persistence.initialize()
|
||||
expect(githubAuth.getAllTokenIds()).to.deep.equal([])
|
||||
const tokens = await persistence.initialize()
|
||||
expect(tokens).to.deep.equal([])
|
||||
})
|
||||
|
||||
it('saving creates an empty file', async function() {
|
||||
@@ -41,18 +37,17 @@ describe('File system token persistence', function() {
|
||||
})
|
||||
|
||||
it('loads the contents', async function() {
|
||||
await persistence.initialize()
|
||||
expect(githubAuth.getAllTokenIds()).to.deep.equal(initialTokens)
|
||||
const tokens = await persistence.initialize()
|
||||
expect(tokens).to.deep.equal(initialTokens)
|
||||
})
|
||||
|
||||
context('when tokens are added', function() {
|
||||
it('saves the change', async function() {
|
||||
const newToken = 'e'.repeat(40)
|
||||
const expected = initialTokens.slice()
|
||||
const expected = Array.from(initialTokens)
|
||||
expected.push(newToken)
|
||||
|
||||
await persistence.initialize()
|
||||
githubAuth.addGithubToken(newToken)
|
||||
await persistence.noteTokenAdded(newToken)
|
||||
|
||||
const savedTokens = JSON.parse(await readFile(path))
|
||||
@@ -67,7 +62,6 @@ describe('File system token persistence', function() {
|
||||
|
||||
await persistence.initialize()
|
||||
|
||||
githubAuth.rmGithubToken(toRemove)
|
||||
await persistence.noteTokenRemoved(toRemove)
|
||||
|
||||
const savedTokens = JSON.parse(await readFile(path))
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
'use strict'
|
||||
|
||||
const { EventEmitter } = require('events')
|
||||
const queryString = require('query-string')
|
||||
const mapKeys = require('lodash.mapkeys')
|
||||
const serverSecrets = require('./server-secrets')
|
||||
const { sanitizeToken } = require('./token-pool')
|
||||
|
||||
const userTokenRateLimit = 12500
|
||||
|
||||
let githubUserTokens = []
|
||||
// Ideally, we would want priority queues here.
|
||||
const reqRemaining = new Map() // From token to requests remaining.
|
||||
const reqReset = new Map() // From token to timestamp.
|
||||
|
||||
const emitter = new EventEmitter()
|
||||
|
||||
// Personal tokens allow access to GitHub private repositories.
|
||||
// You can manage your personal GitHub token at
|
||||
// <https://github.com/settings/tokens>.
|
||||
if (serverSecrets && serverSecrets.gh_token) {
|
||||
addGithubToken(serverSecrets.gh_token)
|
||||
}
|
||||
|
||||
// token: client token as a string.
|
||||
// reqs: number of requests remaining.
|
||||
// reset: timestamp when the number of remaining requests is reset.
|
||||
function setReqRemaining(token, reqs, reset) {
|
||||
reqRemaining.set(token, reqs)
|
||||
reqReset.set(token, reset)
|
||||
}
|
||||
|
||||
function rmReqRemaining(token) {
|
||||
reqRemaining.delete(token)
|
||||
reqReset.delete(token)
|
||||
}
|
||||
|
||||
function utcEpochSeconds() {
|
||||
return (Date.now() / 1000) >>> 0
|
||||
}
|
||||
|
||||
// Return false if the token cannot reasonably be expected to perform
|
||||
// a GitHub request.
|
||||
function isTokenUsable(token, now) {
|
||||
const reqs = reqRemaining.get(token)
|
||||
const reset = reqReset.get(token)
|
||||
// We don't want to empty more than 3/4 of a user's rate limit.
|
||||
const hasRemainingReqs = reqs > userTokenRateLimit / 4
|
||||
const isBeyondRateLimitReset = reset < now
|
||||
return hasRemainingReqs || isBeyondRateLimitReset
|
||||
}
|
||||
|
||||
// Return a list of tokens (as strings) which can be used for a GitHub request,
|
||||
// with a reasonable chance that the request will succeed.
|
||||
function usableTokens() {
|
||||
const now = utcEpochSeconds()
|
||||
return githubUserTokens.filter(token => isTokenUsable(token, now))
|
||||
}
|
||||
|
||||
// Retrieve a user token if there is one for which we believe there are requests
|
||||
// remaining. Return undefined if we could not find one.
|
||||
function getReqRemainingToken() {
|
||||
// Go through the user tokens.
|
||||
// Among usable ones, use the one with the highest number of remaining
|
||||
// requests.
|
||||
const tokens = usableTokens()
|
||||
let highestReq = -1
|
||||
let highestToken
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
const token = tokens[i]
|
||||
const reqs = reqRemaining.get(token)
|
||||
if (reqs > highestReq) {
|
||||
highestReq = reqs
|
||||
highestToken = token
|
||||
}
|
||||
}
|
||||
return highestToken
|
||||
}
|
||||
|
||||
function addGithubToken(token) {
|
||||
// A reset date of 0 has to be in the past.
|
||||
setReqRemaining(token, userTokenRateLimit, 0)
|
||||
// Insert it only if it is not registered yet.
|
||||
if (githubUserTokens.indexOf(token) === -1) {
|
||||
githubUserTokens.push(token)
|
||||
}
|
||||
emitter.emit('token-added', token)
|
||||
}
|
||||
|
||||
function rmGithubToken(token) {
|
||||
rmReqRemaining(token)
|
||||
// Remove it only if it is in there.
|
||||
const idx = githubUserTokens.indexOf(token)
|
||||
if (idx >= 0) {
|
||||
githubUserTokens.splice(idx, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert an ES6 Map to an object.
|
||||
function mapToObject(map) {
|
||||
const result = {}
|
||||
for (const [k, v] of map) {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function getAllTokenIds() {
|
||||
return githubUserTokens.slice()
|
||||
}
|
||||
|
||||
function removeAllTokens() {
|
||||
githubUserTokens = []
|
||||
}
|
||||
|
||||
function serializeDebugInfo(options) {
|
||||
// Apply defaults.
|
||||
const { sanitize } = Object.assign({ sanitize: true }, options)
|
||||
|
||||
const unsanitized = {
|
||||
tokens: githubUserTokens,
|
||||
reqRemaining: mapToObject(reqRemaining),
|
||||
reqReset: mapToObject(reqReset),
|
||||
utcEpochSeconds: utcEpochSeconds(),
|
||||
sanitized: false,
|
||||
}
|
||||
|
||||
if (sanitize) {
|
||||
return {
|
||||
tokens: unsanitized.tokens.map(k => sanitizeToken(k)),
|
||||
reqRemaining: mapKeys(unsanitized.reqRemaining, (v, k) =>
|
||||
sanitizeToken(k)
|
||||
),
|
||||
reqReset: mapKeys(unsanitized.reqReset, (v, k) => sanitizeToken(k)),
|
||||
utcEpochSeconds: unsanitized.utcEpochSeconds,
|
||||
sanitized: true,
|
||||
}
|
||||
} else {
|
||||
return unsanitized
|
||||
}
|
||||
}
|
||||
|
||||
// When a global gh_token is configured, use that in place of our shields.io
|
||||
// token-cycling logic. This produces more predictable behavior when a token
|
||||
// is provided, and more predictable failures if that token is exhausted.
|
||||
//
|
||||
// You can manage your personal GitHub token at https://github.com/settings/tokens
|
||||
const globalToken = (serverSecrets || {}).gh_token
|
||||
|
||||
// Act like request(), but tweak headers and query to avoid hitting a rate
|
||||
// limit.
|
||||
function githubRequest(request, url, query, cb) {
|
||||
query = query || {}
|
||||
// A special User-Agent is required:
|
||||
// http://developer.github.com/v3/#user-agent-required
|
||||
const headers = {
|
||||
'User-Agent': 'Shields.io',
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
}
|
||||
|
||||
const githubToken =
|
||||
globalToken === undefined ? getReqRemainingToken() : globalToken
|
||||
|
||||
if (githubToken != null) {
|
||||
// Typically, GitHub user tokens grants us 12500 req/hour.
|
||||
headers['Authorization'] = `token ${githubToken}`
|
||||
} else if (serverSecrets && serverSecrets.gh_client_id) {
|
||||
// Using our OAuth App secret grants us 5000 req/hour
|
||||
// instead of the standard 60 req/hour.
|
||||
query.client_id = serverSecrets.gh_client_id
|
||||
query.client_secret = serverSecrets.gh_client_secret
|
||||
}
|
||||
|
||||
const qs = queryString.stringify(query)
|
||||
if (qs) {
|
||||
url += `?${qs}`
|
||||
}
|
||||
|
||||
request(url, { headers }, (err, res, buffer) => {
|
||||
if (globalToken !== null && githubToken !== null && err === null) {
|
||||
if (res.statusCode === 401) {
|
||||
// Unauthorized.
|
||||
rmGithubToken(githubToken)
|
||||
emitter.emit('token-removed', githubToken)
|
||||
} else {
|
||||
const remaining = +res.headers['x-ratelimit-remaining']
|
||||
// reset is in UTC epoch seconds.
|
||||
const reset = +res.headers['x-ratelimit-reset']
|
||||
setReqRemaining(githubToken, remaining, reset)
|
||||
if (remaining === 0) {
|
||||
return
|
||||
} // Hope for the best in the cache.
|
||||
}
|
||||
}
|
||||
cb(err, res, buffer)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
request: githubRequest,
|
||||
serializeDebugInfo,
|
||||
addGithubToken,
|
||||
rmGithubToken,
|
||||
getAllTokenIds,
|
||||
removeAllTokens,
|
||||
emitter,
|
||||
}
|
||||
@@ -5,12 +5,8 @@ const RedisServer = require('redis-server')
|
||||
const redis = require('redis')
|
||||
const { expect } = require('chai')
|
||||
const RedisTokenPersistence = require('./redis-token-persistence')
|
||||
const githubAuth = require('./github-auth')
|
||||
|
||||
describe('Redis token persistence', function() {
|
||||
beforeEach(githubAuth.removeAllTokens)
|
||||
afterEach(githubAuth.removeAllTokens)
|
||||
|
||||
let server
|
||||
// In CI, expect redis already to be running.
|
||||
if (!process.env.CI) {
|
||||
@@ -56,8 +52,8 @@ describe('Redis token persistence', function() {
|
||||
|
||||
context('when the key does not exist', function() {
|
||||
it('does nothing', async function() {
|
||||
await persistence.initialize()
|
||||
expect(githubAuth.getAllTokenIds()).to.deep.equal([])
|
||||
const tokens = await persistence.initialize()
|
||||
expect(tokens).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -75,8 +71,8 @@ describe('Redis token persistence', function() {
|
||||
})
|
||||
|
||||
it('loads the contents', async function() {
|
||||
await persistence.initialize()
|
||||
expect(githubAuth.getAllTokenIds()).to.deep.equal(initialTokens)
|
||||
const tokens = await persistence.initialize()
|
||||
expect(tokens).to.deep.equal(initialTokens)
|
||||
})
|
||||
|
||||
context('when tokens are added', function() {
|
||||
@@ -86,7 +82,6 @@ describe('Redis token persistence', function() {
|
||||
expected.push(newToken)
|
||||
|
||||
await persistence.initialize()
|
||||
githubAuth.addGithubToken(newToken)
|
||||
await persistence.noteTokenAdded(newToken)
|
||||
|
||||
const savedTokens = await lrange(key, 0, -1)
|
||||
@@ -101,7 +96,6 @@ describe('Redis token persistence', function() {
|
||||
|
||||
await persistence.initialize()
|
||||
|
||||
githubAuth.rmGithubToken(toRemove)
|
||||
await persistence.noteTokenRemoved(toRemove)
|
||||
|
||||
const savedTokens = await lrange(key, 0, -1)
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
const redis = require('redis')
|
||||
const { promisify } = require('util')
|
||||
const log = require('./log')
|
||||
const githubAuth = require('./github-auth')
|
||||
const TokenPersistence = require('./token-persistence')
|
||||
|
||||
class RedisTokenPersistence extends TokenPersistence {
|
||||
@@ -21,16 +20,8 @@ class RedisTokenPersistence extends TokenPersistence {
|
||||
|
||||
const lrange = promisify(this.client.lrange).bind(this.client)
|
||||
|
||||
let tokens
|
||||
try {
|
||||
tokens = await lrange(this.key, 0, -1)
|
||||
} catch (e) {
|
||||
log.error(e)
|
||||
}
|
||||
|
||||
tokens.forEach(tokenString => {
|
||||
githubAuth.addGithubToken(tokenString)
|
||||
})
|
||||
const tokens = await lrange(this.key, 0, -1)
|
||||
return tokens
|
||||
}
|
||||
|
||||
async stop() {
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
const queryString = require('query-string')
|
||||
const request = require('request')
|
||||
const log = require('../../../lib/log')
|
||||
const githubAuth = require('../../../lib/github-auth')
|
||||
const serverSecrets = require('../../../lib/server-secrets')
|
||||
const secretIsValid = require('../../../lib/sys/secret-is-valid')
|
||||
|
||||
@@ -45,7 +44,7 @@ function sendTokenToAllServers(token) {
|
||||
)
|
||||
}
|
||||
|
||||
function setRoutes(server) {
|
||||
function setRoutes({ server, onTokenAccepted }) {
|
||||
const baseUrl = process.env.BASE_URL || 'https://img.shields.io'
|
||||
|
||||
server.route(/^\/github-auth$/, (data, match, end, ask) => {
|
||||
@@ -125,12 +124,11 @@ function setRoutes(server) {
|
||||
return
|
||||
}
|
||||
|
||||
githubAuth.addGithubToken(data.token)
|
||||
onTokenAccepted(data.token)
|
||||
end('Thanks!')
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sendTokenToAllServers,
|
||||
setRoutes,
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
const { expect } = require('chai')
|
||||
const Camp = require('camp')
|
||||
const got = require('got')
|
||||
const sinon = require('sinon')
|
||||
const portfinder = require('portfinder')
|
||||
const queryString = require('query-string')
|
||||
const nock = require('nock')
|
||||
@@ -10,18 +11,21 @@ const serverSecrets = require('../../../lib/server-secrets')
|
||||
const acceptor = require('./acceptor')
|
||||
|
||||
const fakeClientId = 'githubdabomb'
|
||||
const fakeShieldsSecret = 'letmeinplz'
|
||||
|
||||
describe('Github token acceptor', function() {
|
||||
// Frustratingly, potentially undefined properties can't reliably be stubbed
|
||||
// with Sinon.
|
||||
// https://github.com/sinonjs/sinon/pull/1557
|
||||
before(function() {
|
||||
serverSecrets.gh_client_id = fakeClientId
|
||||
serverSecrets.shields_ips = []
|
||||
// Make sure properties exist.
|
||||
// https://github.com/sinonjs/sinon/pull/1557
|
||||
serverSecrets.gh_client_id = undefined
|
||||
serverSecrets.shields_ips = undefined
|
||||
serverSecrets.shields_secret = undefined
|
||||
sinon.stub(serverSecrets, 'gh_client_id').value(fakeClientId)
|
||||
sinon.stub(serverSecrets, 'shields_ips').value([])
|
||||
sinon.stub(serverSecrets, 'shields_secret').value(fakeShieldsSecret)
|
||||
})
|
||||
after(function() {
|
||||
delete serverSecrets.gh_client_id
|
||||
delete serverSecrets.shields_ips
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
let port, baseUrl
|
||||
@@ -42,8 +46,13 @@ describe('Github token acceptor', function() {
|
||||
}
|
||||
})
|
||||
|
||||
let onTokenAccepted
|
||||
beforeEach(function() {
|
||||
acceptor.setRoutes(camp)
|
||||
onTokenAccepted = sinon.stub()
|
||||
acceptor.setRoutes({
|
||||
server: camp,
|
||||
onTokenAccepted,
|
||||
})
|
||||
})
|
||||
|
||||
it('should start the OAuth process', async function() {
|
||||
@@ -108,4 +117,16 @@ describe('Github token acceptor', function() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('should add a received token', async function() {
|
||||
const fakeAccessToken = 'its-my-token'
|
||||
|
||||
const { body } = await got(`${baseUrl}/github-auth/add-token`, {
|
||||
form: true,
|
||||
body: { shieldsSecret: fakeShieldsSecret, token: fakeAccessToken },
|
||||
})
|
||||
|
||||
expect(onTokenAccepted).to.have.been.calledWith(fakeAccessToken)
|
||||
expect(body).to.equal('Thanks!')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
'use strict'
|
||||
|
||||
const { serializeDebugInfo } = require('../../../lib/github-auth')
|
||||
const secretIsValid = require('../../../lib/sys/secret-is-valid')
|
||||
|
||||
function setRoutes(server) {
|
||||
function setRoutes(apiProvider, server) {
|
||||
// Allow the admin to obtain the tokens for operational and debugging
|
||||
// purposes. This could be used to:
|
||||
//
|
||||
@@ -23,7 +22,7 @@ function setRoutes(server) {
|
||||
end('Invalid secret.')
|
||||
}, 10000)
|
||||
}
|
||||
end(serializeDebugInfo({ sanitize: false }))
|
||||
end(apiProvider.serializeDebugInfo({ sanitize: false }))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ const Camp = require('camp')
|
||||
const fetch = require('node-fetch')
|
||||
const portfinder = require('portfinder')
|
||||
const serverSecrets = require('../../../lib/server-secrets')
|
||||
const GithubApiProvider = require('../github-api-provider')
|
||||
const { setRoutes } = require('./admin')
|
||||
|
||||
function createAuthHeader({ username, password }) {
|
||||
@@ -53,7 +54,8 @@ describe('GitHub admin route', function() {
|
||||
})
|
||||
|
||||
before(function() {
|
||||
setRoutes(camp)
|
||||
const apiProvider = new GithubApiProvider({ withPooling: true })
|
||||
setRoutes(apiProvider, camp)
|
||||
})
|
||||
|
||||
context('the password is correct', function() {
|
||||
|
||||
@@ -1,14 +1,27 @@
|
||||
'use strict'
|
||||
|
||||
const { expect } = require('chai')
|
||||
const serverSecrets = require('../../lib/server-secrets')
|
||||
const GithubApiProvider = require('./github-api-provider')
|
||||
|
||||
describe('Github API provider', function() {
|
||||
describe('Github API provider with token pool', function() {
|
||||
const baseUrl = process.env.GITHUB_URL || 'https://api.github.com'
|
||||
const reserveFraction = 0.333
|
||||
|
||||
let githubApiProvider
|
||||
before(function() {
|
||||
githubApiProvider = new GithubApiProvider({ baseUrl })
|
||||
githubApiProvider = new GithubApiProvider({
|
||||
baseUrl,
|
||||
withPooling: true,
|
||||
reserveFraction,
|
||||
})
|
||||
|
||||
const { gh_token: token } = serverSecrets
|
||||
if (!token) {
|
||||
throw Error('The integration tests require a gh_token to be set')
|
||||
}
|
||||
|
||||
githubApiProvider.addToken(token)
|
||||
})
|
||||
|
||||
const headers = []
|
||||
@@ -38,4 +51,23 @@ describe('Github API provider', function() {
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('should update the token with the final limit remaining and reset time', function() {
|
||||
const lastHeaders = headers.slice(-1)[0]
|
||||
const reserve = reserveFraction * +lastHeaders['x-ratelimit-limit']
|
||||
const usesRemaining = +lastHeaders['x-ratelimit-remaining'] - reserve
|
||||
const nextReset = +lastHeaders['x-ratelimit-reset']
|
||||
|
||||
const tokens = []
|
||||
githubApiProvider.standardTokens.forEach(t => {
|
||||
tokens.push(t)
|
||||
})
|
||||
|
||||
// Confidence check.
|
||||
expect(tokens).to.have.lengthOf(1)
|
||||
|
||||
const [token] = tokens
|
||||
expect(token.usesRemaining).to.equal(usesRemaining)
|
||||
expect(token.nextReset).to.equal(nextReset)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,15 +1,79 @@
|
||||
'use strict'
|
||||
|
||||
const githubAuth = require('../../lib/github-auth')
|
||||
const { TokenPool } = require('../../lib/token-pool')
|
||||
|
||||
// Provide an interface to the Github API. Manages the base URL.
|
||||
//
|
||||
// Eventually this class will be responsible for managing headers,
|
||||
// authentication, and actually making the request. Currently it's delegating
|
||||
// to legacy code.
|
||||
// Provides an interface to the Github API. Manages the base URL.
|
||||
class GithubApiProvider {
|
||||
constructor({ baseUrl }) {
|
||||
this.baseUrl = baseUrl
|
||||
// reserveFraction: The amount of much of a token's quota we avoid using, to
|
||||
// reserve it for the user.
|
||||
constructor({
|
||||
baseUrl,
|
||||
withPooling = true,
|
||||
onTokenInvalidated = tokenString => {},
|
||||
globalToken,
|
||||
reserveFraction = 0.25,
|
||||
}) {
|
||||
Object.assign(this, {
|
||||
baseUrl,
|
||||
withPooling,
|
||||
onTokenInvalidated,
|
||||
globalToken,
|
||||
reserveFraction,
|
||||
})
|
||||
|
||||
if (this.withPooling) {
|
||||
this.standardTokens = new TokenPool({ batchSize: 25 })
|
||||
this.searchTokens = new TokenPool({ batchSize: 5 })
|
||||
}
|
||||
}
|
||||
|
||||
serializeDebugInfo({ sanitize = true } = {}) {
|
||||
if (this.withPooling) {
|
||||
return {
|
||||
standardTokens: this.standardTokens.serializeDebugInfo({ sanitize }),
|
||||
searchTokens: this.searchTokens.serializeDebugInfo({ sanitize }),
|
||||
}
|
||||
} else {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
addToken(tokenString) {
|
||||
if (this.withPooling) {
|
||||
this.standardTokens.add(tokenString)
|
||||
this.searchTokens.add(tokenString)
|
||||
} else {
|
||||
throw Error('When not using a token pool, do not provide tokens')
|
||||
}
|
||||
}
|
||||
|
||||
updateToken(token, headers) {
|
||||
const rateLimit = +headers['x-ratelimit-limit']
|
||||
const reserve = this.reserveFraction * rateLimit
|
||||
const usesRemaining = +headers['x-ratelimit-remaining'] - reserve
|
||||
|
||||
const nextReset = +headers['x-ratelimit-reset']
|
||||
|
||||
token.update(usesRemaining, nextReset)
|
||||
}
|
||||
|
||||
invalidateToken(token) {
|
||||
token.invalidate()
|
||||
this.onTokenInvalidated(token.id)
|
||||
}
|
||||
|
||||
tokenForUrl(url) {
|
||||
const { globalToken } = this
|
||||
if (globalToken) {
|
||||
// When a global gh_token is configured, use that in place of our token
|
||||
// pool. This produces more predictable behavior, and more predictable
|
||||
// failures when that token is exhausted.
|
||||
return { id: globalToken }
|
||||
} else if (url.startsWith('/search')) {
|
||||
return this.searchTokens.next()
|
||||
} else {
|
||||
return this.standardTokens.next()
|
||||
}
|
||||
}
|
||||
|
||||
// Act like request(), but tweak headers and query to avoid hitting a rate
|
||||
@@ -18,14 +82,35 @@ class GithubApiProvider {
|
||||
request(request, url, query, callback) {
|
||||
const { baseUrl } = this
|
||||
|
||||
githubAuth.request(
|
||||
request,
|
||||
`${baseUrl}${url}`,
|
||||
query,
|
||||
(err, res, buffer) => {
|
||||
callback(err, res, buffer)
|
||||
let token
|
||||
try {
|
||||
token = this.tokenForUrl(url)
|
||||
} catch (e) {
|
||||
callback(e)
|
||||
return
|
||||
}
|
||||
|
||||
const options = {
|
||||
url,
|
||||
baseUrl,
|
||||
qs: query,
|
||||
headers: {
|
||||
'User-Agent': 'Shields.io',
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
Authorization: `token ${token.id}`,
|
||||
},
|
||||
}
|
||||
|
||||
request(options, (err, res, buffer) => {
|
||||
if (err === null) {
|
||||
if (res.statusCode === 401) {
|
||||
this.invalidateToken(token)
|
||||
} else if (res.statusCode < 500) {
|
||||
this.updateToken(token, res.headers)
|
||||
}
|
||||
}
|
||||
)
|
||||
callback(err, res, buffer)
|
||||
})
|
||||
}
|
||||
|
||||
requestAsPromise(request, url, query) {
|
||||
|
||||
@@ -1,14 +1,50 @@
|
||||
'use strict'
|
||||
|
||||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
const GithubApiProvider = require('./github-api-provider')
|
||||
|
||||
describe('Github API provider', function() {
|
||||
const baseUrl = 'https://github-api.example.com'
|
||||
const reserveFraction = 0.333
|
||||
|
||||
let provider
|
||||
let mockStandardToken, mockSearchToken, provider
|
||||
beforeEach(function() {
|
||||
provider = new GithubApiProvider({ baseUrl })
|
||||
provider = new GithubApiProvider({ baseUrl, reserveFraction })
|
||||
|
||||
mockStandardToken = { update: sinon.spy(), invalidate: sinon.spy() }
|
||||
sinon.stub(provider.standardTokens, 'next').returns(mockStandardToken)
|
||||
|
||||
mockSearchToken = { update: sinon.spy(), invalidate: sinon.spy() }
|
||||
sinon.stub(provider.searchTokens, 'next').returns(mockSearchToken)
|
||||
})
|
||||
|
||||
context('a search API request', function() {
|
||||
const mockRequest = (options, callback) => {
|
||||
callback()
|
||||
}
|
||||
it('should obtain an appropriate token', function(done) {
|
||||
provider.request(mockRequest, '/search', {}, (err, res, buffer) => {
|
||||
expect(err).to.be.undefined
|
||||
expect(provider.searchTokens.next).to.have.been.calledOnce
|
||||
expect(provider.standardTokens.next).not.to.have.been.called
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
context('a core API request', function() {
|
||||
const mockRequest = (options, callback) => {
|
||||
callback()
|
||||
}
|
||||
it('should obtain an appropriate token', function(done) {
|
||||
provider.request(mockRequest, '/repo', {}, (err, res, buffer) => {
|
||||
expect(err).to.be.undefined
|
||||
expect(provider.searchTokens.next).not.to.have.been.called
|
||||
expect(provider.standardTokens.next).to.have.been.calledOnce
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
context('a valid response', function() {
|
||||
@@ -37,6 +73,19 @@ describe('Github API provider', function() {
|
||||
done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should update the token with the expected values', function(done) {
|
||||
provider.request(mockRequest, '/foo', {}, (err, res, buffer) => {
|
||||
expect(err).to.equal(null)
|
||||
const expectedUsesRemaining = remaining - reserveFraction * rateLimit
|
||||
expect(
|
||||
mockStandardToken.update.withArgs(expectedUsesRemaining, nextReset)
|
||||
.calledOnce
|
||||
).to.be.true
|
||||
expect(mockStandardToken.invalidate).not.to.have.been.called
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
context('an unauthorized response', function() {
|
||||
@@ -47,10 +96,11 @@ describe('Github API provider', function() {
|
||||
callback(null, mockResponse, mockBuffer)
|
||||
}
|
||||
|
||||
it('should invoke the callback', function(done) {
|
||||
it('should invoke the callback and update the token with the expected values', function(done) {
|
||||
provider.request(mockRequest, '/foo', {}, (err, res, buffer) => {
|
||||
expect(err).to.equal(null)
|
||||
// Add more?
|
||||
expect(mockStandardToken.invalidate).to.have.been.calledOnce
|
||||
expect(mockStandardToken.update).not.to.have.been.called
|
||||
done()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
'use strict'
|
||||
|
||||
const path = require('path')
|
||||
const githubAuth = require('../../lib/github-auth')
|
||||
const serverSecrets = require('../../lib/server-secrets')
|
||||
const log = require('../../lib/log')
|
||||
const RedisTokenPersistence = require('../../lib/redis-token-persistence')
|
||||
@@ -33,55 +32,78 @@ class GithubConstellation {
|
||||
this.persistence = new FsTokenPersistence({ path: userTokensPath })
|
||||
}
|
||||
|
||||
const globalToken = serverSecrets.gh_token
|
||||
const baseUrl = process.env.GITHUB_URL || 'https://api.github.com'
|
||||
this.apiProvider = new GithubApiProvider({ baseUrl })
|
||||
this.apiProvider = new GithubApiProvider({
|
||||
baseUrl,
|
||||
globalToken,
|
||||
onTokenInvalidated: tokenString => this.onTokenInvalidated(tokenString),
|
||||
})
|
||||
}
|
||||
|
||||
scheduleDebugLogging() {
|
||||
if (this._debugEnabled) {
|
||||
this.debugInterval = setInterval(() => {
|
||||
log(githubAuth.serializeDebugInfo())
|
||||
log(this.apiProvider.getTokenDebugInfo())
|
||||
}, 1000 * this._debugIntervalSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
async initialize(server) {
|
||||
if (!this.apiProvider.withPooling) {
|
||||
return
|
||||
}
|
||||
|
||||
this.scheduleDebugLogging()
|
||||
|
||||
let tokens = []
|
||||
try {
|
||||
await this.persistence.initialize()
|
||||
tokens = await this.persistence.initialize()
|
||||
} catch (e) {
|
||||
log.error(e)
|
||||
}
|
||||
|
||||
// Register for this event after `initialize()` finishes, so we don't
|
||||
// catch `token-added` events for the initial tokens, which would be
|
||||
// inefficient, though it wouldn't break anything.
|
||||
githubAuth.emitter.on('token-added', this.persistence.noteTokenAdded)
|
||||
githubAuth.emitter.on('token-removed', this.persistence.noteTokenRemoved)
|
||||
tokens.forEach(tokenString => {
|
||||
this.apiProvider.addToken(tokenString)
|
||||
})
|
||||
|
||||
setAdminRoutes(server)
|
||||
setAdminRoutes(this.apiProvider, server)
|
||||
|
||||
if (serverSecrets.gh_client_id && serverSecrets.gh_client_secret) {
|
||||
setAcceptorRoutes(server)
|
||||
setAcceptorRoutes({
|
||||
server,
|
||||
onTokenAccepted: tokenString => this.onTokenAdded(tokenString),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
onTokenAdded(tokenString) {
|
||||
this.apiProvider.addToken(tokenString)
|
||||
process.nextTick(async () => {
|
||||
try {
|
||||
await this.persistence.noteTokenAdded(tokenString)
|
||||
} catch (e) {
|
||||
log.error(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
onTokenInvalidated(tokenString) {
|
||||
process.nextTick(async () => {
|
||||
try {
|
||||
await this.persistence.noteTokenRemoved(tokenString)
|
||||
} catch (e) {
|
||||
log.error(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (this.debugInterval) {
|
||||
clearInterval(this.debugInterval)
|
||||
this.debugInterval = undefined
|
||||
}
|
||||
|
||||
githubAuth.emitter.removeListener(
|
||||
'token-added',
|
||||
this.persistence.noteTokenAdded
|
||||
)
|
||||
githubAuth.emitter.removeListener(
|
||||
'token-removed',
|
||||
this.persistence.noteTokenRemoved
|
||||
)
|
||||
|
||||
try {
|
||||
await this.persistence.stop()
|
||||
} catch (e) {
|
||||
|
||||
Reference in New Issue
Block a user