diff --git a/core/base-service/base.js b/core/base-service/base.js index 0a22086601..a547814ea4 100644 --- a/core/base-service/base.js +++ b/core/base-service/base.js @@ -20,6 +20,7 @@ import { Deprecated, } from './errors.js' import { validateExample, transformExample } from './examples.js' +import { fetchFactory } from './got.js' import { makeFullUrl, assertValidRoute, @@ -431,6 +432,8 @@ class BaseService { ServiceClass: this, }) + const fetcher = fetchFactory(fetchLimitBytes) + camp.route( regex, handleRequest(cacheHeaderConfig, { @@ -441,7 +444,7 @@ class BaseService { const namedParams = namedParamsForMatch(captureNames, match, this) const serviceData = await this.invoke( { - sendAndCacheRequest: request.asPromise, + sendAndCacheRequest: fetcher, sendAndCacheRequestWithCallbacks: request, githubApiProvider, metricHelper, diff --git a/core/base-service/got.js b/core/base-service/got.js new file mode 100644 index 0000000000..0120e216cd --- /dev/null +++ b/core/base-service/got.js @@ -0,0 +1,96 @@ +import got from 'got' +import { Inaccessible, InvalidResponse } from './errors.js' + +const userAgent = 'Shields.io/2003a' + +function requestOptions2GotOptions(options) { + const requestOptions = Object.assign({}, options) + const gotOptions = {} + const interchangableOptions = ['body', 'form', 'headers', 'method', 'url'] + + interchangableOptions.forEach(function (opt) { + if (opt in requestOptions) { + gotOptions[opt] = requestOptions[opt] + delete requestOptions[opt] + } + }) + + if ('qs' in requestOptions) { + gotOptions.searchParams = requestOptions.qs + delete requestOptions.qs + } + + if ('gzip' in requestOptions) { + gotOptions.decompress = requestOptions.gzip + delete requestOptions.gzip + } + + if ('strictSSL' in requestOptions) { + gotOptions.https = { + rejectUnauthorized: requestOptions.strictSSL, + } + delete requestOptions.strictSSL + } + + if ('auth' in requestOptions) { + gotOptions.username = requestOptions.auth.user + gotOptions.password = requestOptions.auth.pass + delete requestOptions.auth + } + + if (Object.keys(requestOptions).length > 0) { + throw new Error(`Found unrecognised options ${Object.keys(requestOptions)}`) + } + + return gotOptions +} + +async function sendRequest(gotWrapper, url, options) { + const gotOptions = requestOptions2GotOptions(options) + gotOptions.throwHttpErrors = false + gotOptions.retry = 0 + gotOptions.headers = gotOptions.headers || {} + gotOptions.headers['User-Agent'] = userAgent + try { + const resp = await gotWrapper(url, gotOptions) + return { res: resp, buffer: resp.body } + } catch (err) { + if (err instanceof got.CancelError) { + throw new InvalidResponse({ + underlyingError: new Error('Maximum response size exceeded'), + }) + } + throw new Inaccessible({ underlyingError: err }) + } +} + +function fetchFactory(fetchLimitBytes) { + const gotWithLimit = got.extend({ + handlers: [ + (options, next) => { + const promiseOrStream = next(options) + promiseOrStream.on('downloadProgress', progress => { + if ( + progress.transferred > fetchLimitBytes && + // just accept the file if we've already finished downloading + // the entire file before we went over the limit + progress.percent !== 1 + ) { + /* + TODO: we should be able to pass cancel() a message + https://github.com/sindresorhus/got/blob/main/documentation/advanced-creation.md#examples + but by the time we catch it, err.message is just "Promise was canceled" + */ + promiseOrStream.cancel('Maximum response size exceeded') + } + }) + + return promiseOrStream + }, + ], + }) + + return sendRequest.bind(sendRequest, gotWithLimit) +} + +export { requestOptions2GotOptions, fetchFactory } diff --git a/core/base-service/got.spec.js b/core/base-service/got.spec.js new file mode 100644 index 0000000000..185052d163 --- /dev/null +++ b/core/base-service/got.spec.js @@ -0,0 +1,102 @@ +import { expect } from 'chai' +import nock from 'nock' +import { requestOptions2GotOptions, fetchFactory } from './got.js' +import { Inaccessible, InvalidResponse } from './errors.js' + +describe('requestOptions2GotOptions function', function () { + it('translates valid options', function () { + expect( + requestOptions2GotOptions({ + body: 'body', + form: 'form', + headers: 'headers', + method: 'method', + url: 'url', + qs: 'qs', + gzip: 'gzip', + strictSSL: 'strictSSL', + auth: { user: 'user', pass: 'pass' }, + }) + ).to.deep.equal({ + body: 'body', + form: 'form', + headers: 'headers', + method: 'method', + url: 'url', + searchParams: 'qs', + decompress: 'gzip', + https: { rejectUnauthorized: 'strictSSL' }, + username: 'user', + password: 'pass', + }) + }) + + it('throws if unrecognised options are found', function () { + expect(() => + requestOptions2GotOptions({ body: 'body', foobar: 'foobar' }) + ).to.throw(Error, 'Found unrecognised options foobar') + }) +}) + +describe('got wrapper', function () { + it('should not throw an error if the response <= fetchLimitBytes', async function () { + nock('https://www.google.com') + .get('/foo/bar') + .once() + .reply(200, 'x'.repeat(100)) + const sendRequest = fetchFactory(100) + const { res } = await sendRequest('https://www.google.com/foo/bar') + expect(res.statusCode).to.equal(200) + }) + + it('should throw an InvalidResponse error if the response is > fetchLimitBytes', async function () { + nock('https://www.google.com') + .get('/foo/bar') + .once() + .reply(200, 'x'.repeat(101)) + const sendRequest = fetchFactory(100) + return expect( + sendRequest('https://www.google.com/foo/bar') + ).to.be.rejectedWith(InvalidResponse, 'Maximum response size exceeded') + }) + + it('should throw an Inaccessible error if the request throws a (non-HTTP) error', async function () { + nock('https://www.google.com').get('/foo/bar').replyWithError('oh no') + const sendRequest = fetchFactory(1024) + return expect( + sendRequest('https://www.google.com/foo/bar') + ).to.be.rejectedWith(Inaccessible, 'oh no') + }) + + it('should throw an Inaccessible error if the host can not be accessed', async function () { + this.timeout(5000) + nock.disableNetConnect() + const sendRequest = fetchFactory(1024) + return expect( + sendRequest('https://www.google.com/foo/bar') + ).to.be.rejectedWith( + Inaccessible, + 'Nock: Disallowed net connect for "www.google.com:443/foo/bar"' + ) + }) + + it('should pass a custom user agent header', async function () { + nock('https://www.google.com', { + reqheaders: { + 'user-agent': function (agent) { + return agent.startsWith('Shields.io') + }, + }, + }) + .get('/foo/bar') + .once() + .reply(200) + const sendRequest = fetchFactory(1024) + await sendRequest('https://www.google.com/foo/bar') + }) + + afterEach(function () { + nock.cleanAll() + nock.enableNetConnect() + }) +}) diff --git a/core/server/server.js b/core/server/server.js index b19278b049..4dbcf4c232 100644 --- a/core/server/server.js +++ b/core/server/server.js @@ -4,6 +4,7 @@ import path from 'path' import url, { fileURLToPath } from 'url' +import { bootstrap } from 'global-agent' import cloudflareMiddleware from 'cloudflare-middleware' import bytes from 'bytes' import Camp from '@shields_io/camp' @@ -422,6 +423,25 @@ class Server { ) } + bootstrapAgent() { + /* + Bootstrap global agent. + This allows self-hosting users to configure a proxy with + HTTP_PROXY, HTTPS_PROXY, NO_PROXY variables + */ + if (!('GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE' in process.env)) { + process.env.GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE = '' + } + + const proxyPrefix = process.env.GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE + const HTTP_PROXY = process.env[`${proxyPrefix}HTTP_PROXY`] || null + const HTTPS_PROXY = process.env[`${proxyPrefix}HTTPS_PROXY`] || null + + if (HTTP_PROXY || HTTPS_PROXY) { + bootstrap() + } + } + /** * Start the HTTP server: * Bootstrap Scoutcamp, @@ -436,6 +456,8 @@ class Server { requireCloudflare, } = this.config.public + this.bootstrapAgent() + log.log(`Server is starting up: ${this.baseUrl}`) const camp = (this.camp = Camp.create({ diff --git a/package-lock.json b/package-lock.json index f2d8bd4455..9b436cd7d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "escape-string-regexp": "^4.0.0", "fast-xml-parser": "^3.19.0", "glob": "^7.1.7", + "global-agent": "^2.2.0", "got": "11.8.2", "graphql": "^15.5.0", "graphql-tag": "^2.12.5", @@ -44,6 +45,7 @@ "pretty-bytes": "^5.6.0", "priorityqueuejs": "^2.0.0", "prom-client": "^13.1.0", + "qs": "^6.10.1", "query-string": "^7.0.1", "request": "~2.88.2", "semver": "~7.3.5", @@ -2227,6 +2229,15 @@ "node": ">= 0.12" } }, + "node_modules/@cypress/request/node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, "node_modules/@cypress/xvfb": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz", @@ -6286,6 +6297,11 @@ "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", "dev": true }, + "node_modules/boolean": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.1.2.tgz", + "integrity": "sha512-YN6UmV0FfLlBVvRvNPx3pz5W/mUoYB24J4WSXOKP/OOJpi+Oq6WYqPaNTHzjI0QzwWtnvEd5CGYyQPgp1jFxnw==" + }, "node_modules/boxen": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", @@ -6685,7 +6701,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -7969,21 +7984,6 @@ "node": ">=6" } }, - "node_modules/contentful-sdk-core/node_modules/qs": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", - "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", - "dev": true, - "dependencies": { - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/convert-hrtime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/convert-hrtime/-/convert-hrtime-3.0.0.tgz", @@ -8078,7 +8078,6 @@ "version": "3.10.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.10.0.tgz", "integrity": "sha512-MQx/7TLgmmDVamSyfE+O+5BHvG1aUGj/gHhLn1wVtm2B5u1eVIPvh7vkfjwWKNCjrTJB8+He99IntSQ1qP+vYQ==", - "dev": true, "hasInstallScript": true, "funding": { "type": "opencollective", @@ -9439,7 +9438,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "dependencies": { "object-keys": "^1.0.12" }, @@ -10250,8 +10248,7 @@ "node_modules/es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", - "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" }, "node_modules/es6-iterator": { "version": "2.0.3", @@ -12962,8 +12959,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -15127,7 +15123,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -15321,6 +15316,23 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "node_modules/global-agent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-2.2.0.tgz", + "integrity": "sha512-+20KpaW6DDLqhG7JDiJpD1JvNvb8ts+TNl7BPOYcURqCrXqnN1Vf+XVOrkKJAFPqfX+oEhsdzOj1hLWkBTdNJg==", + "dependencies": { + "boolean": "^3.0.1", + "core-js": "^3.6.5", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, "node_modules/global-dirs": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.0.tgz", @@ -15401,6 +15413,20 @@ "node": ">=4" } }, + "node_modules/globalthis": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.2.tgz", + "integrity": "sha512-ZQnSFO1la8P7auIOQECnm0sSuoMeaSq0EEdXMBFF2QJO4uNcwbyhSgG3MruWNbFTqCLmxVwGOl7LZ9kASvHdeQ==", + "dependencies": { + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/globby": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.2.tgz", @@ -15804,7 +15830,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -15882,7 +15907,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -19733,6 +19757,17 @@ "node": ">= 8.16.2" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/md5": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", @@ -21807,7 +21842,6 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -21829,7 +21863,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -22776,21 +22809,6 @@ "query-string": "^6.13.8" } }, - "node_modules/parse-path/node_modules/qs": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", - "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", - "dev": true, - "dependencies": { - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/parse-path/node_modules/query-string": { "version": "6.14.1", "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", @@ -24264,11 +24282,17 @@ } }, "node_modules/qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", + "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", + "dependencies": { + "side-channel": "^1.0.4" + }, "engines": { "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/query-string": { @@ -25669,6 +25693,14 @@ "node": ">= 0.12" } }, + "node_modules/request/node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -25842,6 +25874,27 @@ "rimraf": "bin.js" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/roarr/node_modules/sprintf-js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" + }, "node_modules/rollup-plugin-typescript2": { "version": "0.25.2", "resolved": "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.25.2.tgz", @@ -26114,8 +26167,7 @@ "node_modules/semver-compare": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", - "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", - "dev": true + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=" }, "node_modules/semver-diff": { "version": "3.1.1", @@ -26193,6 +26245,31 @@ "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serialize-javascript": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", @@ -26347,7 +26424,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -33473,6 +33549,12 @@ "combined-stream": "^1.0.6", "mime-types": "^2.1.12" } + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true } } }, @@ -36946,6 +37028,11 @@ "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", "dev": true }, + "boolean": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.1.2.tgz", + "integrity": "sha512-YN6UmV0FfLlBVvRvNPx3pz5W/mUoYB24J4WSXOKP/OOJpi+Oq6WYqPaNTHzjI0QzwWtnvEd5CGYyQPgp1jFxnw==" + }, "boxen": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", @@ -37269,7 +37356,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -38298,17 +38384,6 @@ "requires": { "fast-copy": "^2.1.0", "qs": "^6.9.4" - }, - "dependencies": { - "qs": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", - "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", - "dev": true, - "requires": { - "side-channel": "^1.0.4" - } - } } }, "convert-hrtime": { @@ -38386,8 +38461,7 @@ "core-js": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.10.0.tgz", - "integrity": "sha512-MQx/7TLgmmDVamSyfE+O+5BHvG1aUGj/gHhLn1wVtm2B5u1eVIPvh7vkfjwWKNCjrTJB8+He99IntSQ1qP+vYQ==", - "dev": true + "integrity": "sha512-MQx/7TLgmmDVamSyfE+O+5BHvG1aUGj/gHhLn1wVtm2B5u1eVIPvh7vkfjwWKNCjrTJB8+He99IntSQ1qP+vYQ==" }, "core-js-compat": { "version": "3.14.0", @@ -39424,7 +39498,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "requires": { "object-keys": "^1.0.12" } @@ -40087,8 +40160,7 @@ "es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", - "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" }, "es6-iterator": { "version": "2.0.3", @@ -42192,8 +42264,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -43872,7 +43943,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -44031,6 +44101,20 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "global-agent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-2.2.0.tgz", + "integrity": "sha512-+20KpaW6DDLqhG7JDiJpD1JvNvb8ts+TNl7BPOYcURqCrXqnN1Vf+XVOrkKJAFPqfX+oEhsdzOj1hLWkBTdNJg==", + "requires": { + "boolean": "^3.0.1", + "core-js": "^3.6.5", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + } + }, "global-dirs": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.0.tgz", @@ -44091,6 +44175,14 @@ "integrity": "sha512-io6LkyPVuzCHBSQV9fmOwxZkUk6nIaGmxheLDgmuFv89j0fm2aqDbIXKAGfzCMHqz3HLF2Zf8WSG6VqMh2qFmA==", "dev": true }, + "globalthis": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.2.tgz", + "integrity": "sha512-ZQnSFO1la8P7auIOQECnm0sSuoMeaSq0EEdXMBFF2QJO4uNcwbyhSgG3MruWNbFTqCLmxVwGOl7LZ9kASvHdeQ==", + "requires": { + "define-properties": "^1.1.3" + } + }, "globby": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.2.tgz", @@ -44420,7 +44512,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -44481,8 +44572,7 @@ "has-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" }, "has-to-string-tag-x": { "version": "1.4.1", @@ -47527,6 +47617,14 @@ "integrity": "sha512-MIL0xKRDQM3DE7dJr/wa6JV0EmK9yZ3cwuTc2bu66FNm/tmEMm9cJCgJZpt9R+K1T+pB2iBNV55wvnwSd345zg==", "dev": true }, + "matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "requires": { + "escape-string-regexp": "^4.0.0" + } + }, "md5": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", @@ -49154,8 +49252,7 @@ "object-inspect": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", - "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==", - "dev": true + "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==" }, "object-is": { "version": "1.1.5", @@ -49170,8 +49267,7 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, "object-path": { "version": "0.11.5", @@ -49848,15 +49944,6 @@ "query-string": "^6.13.8" }, "dependencies": { - "qs": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", - "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", - "dev": true, - "requires": { - "side-channel": "^1.0.4" - } - }, "query-string": { "version": "6.14.1", "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", @@ -50960,9 +51047,12 @@ } }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.1.tgz", + "integrity": "sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==", + "requires": { + "side-channel": "^1.0.4" + } }, "query-string": { "version": "7.0.1", @@ -52073,6 +52163,11 @@ "combined-stream": "^1.0.6", "mime-types": "^2.1.12" } + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" } } }, @@ -52226,6 +52321,26 @@ "glob": "^7.1.3" } }, + "roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "requires": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "dependencies": { + "sprintf-js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" + } + } + }, "rollup-plugin-typescript2": { "version": "0.25.2", "resolved": "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.25.2.tgz", @@ -52464,8 +52579,7 @@ "semver-compare": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", - "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", - "dev": true + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=" }, "semver-diff": { "version": "3.1.1", @@ -52519,6 +52633,21 @@ } } }, + "serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "requires": { + "type-fest": "^0.13.1" + }, + "dependencies": { + "type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==" + } + } + }, "serialize-javascript": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", @@ -52653,7 +52782,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "requires": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", diff --git a/package.json b/package.json index 1c6dbbfe00..e8fade0bb0 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "escape-string-regexp": "^4.0.0", "fast-xml-parser": "^3.19.0", "glob": "^7.1.7", + "global-agent": "^2.2.0", "got": "11.8.2", "graphql": "^15.5.0", "graphql-tag": "^2.12.5", @@ -57,6 +58,7 @@ "pretty-bytes": "^5.6.0", "priorityqueuejs": "^2.0.0", "prom-client": "^13.1.0", + "qs": "^6.10.1", "query-string": "^7.0.1", "request": "~2.88.2", "semver": "~7.3.5", diff --git a/services/dynamic/dynamic-json.tester.js b/services/dynamic/dynamic-json.tester.js index 38bb48ee74..7d34bdc1db 100644 --- a/services/dynamic/dynamic-json.tester.js +++ b/services/dynamic/dynamic-json.tester.js @@ -27,7 +27,7 @@ t.create('Malformed url') ) .expectBadge({ label: 'Package Name', - message: 'invalid', + message: 'inaccessible', color: 'lightgrey', }) diff --git a/services/keybase/keybase-btc.service.js b/services/keybase/keybase-btc.service.js index 1ef7d547ff..f0a3196135 100644 --- a/services/keybase/keybase-btc.service.js +++ b/services/keybase/keybase-btc.service.js @@ -58,7 +58,7 @@ export default class KeybaseBTC extends KeybaseProfile { async handle({ username }) { const options = { - form: { + qs: { usernames: username, fields: 'cryptocurrency_addresses', }, diff --git a/services/keybase/keybase-pgp.service.js b/services/keybase/keybase-pgp.service.js index 117a6f2582..6e049754ec 100644 --- a/services/keybase/keybase-pgp.service.js +++ b/services/keybase/keybase-pgp.service.js @@ -51,7 +51,7 @@ export default class KeybasePGP extends KeybaseProfile { async handle({ username }) { const options = { - form: { + qs: { usernames: username, fields: 'public_keys', }, diff --git a/services/keybase/keybase-xlm.service.js b/services/keybase/keybase-xlm.service.js index 652f0a0b3a..d7005ff6f2 100644 --- a/services/keybase/keybase-xlm.service.js +++ b/services/keybase/keybase-xlm.service.js @@ -56,7 +56,7 @@ export default class KeybaseXLM extends KeybaseProfile { async handle({ username }) { const options = { - form: { + qs: { usernames: username, fields: 'stellar', }, diff --git a/services/keybase/keybase-zec.service.js b/services/keybase/keybase-zec.service.js index 0e367a9dfc..399a61e588 100644 --- a/services/keybase/keybase-zec.service.js +++ b/services/keybase/keybase-zec.service.js @@ -58,7 +58,7 @@ export default class KeybaseZEC extends KeybaseProfile { async handle({ username }) { const options = { - form: { + qs: { usernames: username, fields: 'cryptocurrency_addresses', }, diff --git a/services/opm/opm-version.service.js b/services/opm/opm-version.service.js index 41260c85a3..6d9f41c438 100644 --- a/services/opm/opm-version.service.js +++ b/services/opm/opm-version.service.js @@ -39,8 +39,8 @@ export default class OpmVersion extends BaseService { }, }) - // XXX: intercept 302 redirects and set followRedirect to false - const location = res.request.path + // TODO: set followRedirect to false and intercept 302 redirects + const location = res.request.redirects[0] if (!location) { throw new NotFound({ prettyMessage: 'module not found' }) } diff --git a/services/test-helpers.js b/services/test-helpers.js index 0896cf57d5..876467ebed 100644 --- a/services/test-helpers.js +++ b/services/test-helpers.js @@ -1,7 +1,7 @@ +import bytes from 'bytes' import nock from 'nock' -import request from 'request' import config from 'config' -import { promisify } from '../core/base-service/legacy-request-handler.js' +import { fetchFactory } from '../core/base-service/got.js' const runnerConfig = config.util.toObject() function cleanUpNockAfterEach() { @@ -31,7 +31,7 @@ function noToken(serviceClass) { } } -const sendAndCacheRequest = promisify(request) +const sendAndCacheRequest = fetchFactory(bytes(runnerConfig.public.fetchLimit)) const defaultContext = { sendAndCacheRequest } diff --git a/services/wercker/wercker.service.js b/services/wercker/wercker.service.js index 7dcb1faa9f..286c24f21a 100644 --- a/services/wercker/wercker.service.js +++ b/services/wercker/wercker.service.js @@ -91,16 +91,21 @@ export default class Wercker extends BaseJsonService { } } - async fetch({ baseUrl, branch }) { + async fetch({ projectId, applicationName, branch }) { + let url + const qs = { branch, limit: 1 } + + if (applicationName) { + url = `https://app.wercker.com/api/v3/applications/${applicationName}/builds` + } else { + url = 'https://app.wercker.com/api/v3/runs' + qs.applicationId = projectId + } + return this._requestJson({ schema: werckerSchema, - url: baseUrl, - options: { - qs: { - branch, - limit: 1, - }, - }, + url, + options: { qs }, errorMessages: { 401: 'private application not supported', 404: 'application not found', @@ -110,10 +115,8 @@ export default class Wercker extends BaseJsonService { async handle({ projectId, applicationName, branch }) { const json = await this.fetch({ - baseUrl: this.constructor.getBaseUrl({ - projectId, - applicationName, - }), + projectId, + applicationName, branch, }) if (json.length === 0) { diff --git a/services/wordpress/wordpress-base.js b/services/wordpress/wordpress-base.js index c11852b4f5..7f660e7262 100644 --- a/services/wordpress/wordpress-base.js +++ b/services/wordpress/wordpress-base.js @@ -1,4 +1,5 @@ import Joi from 'joi' +import qs from 'qs' import { nonNegativeInteger } from '../validators.js' import { BaseJsonService, NotFound } from '../index.js' @@ -49,29 +50,32 @@ export default class BaseWordpress extends BaseJsonService { } else if (extensionType === 'theme') { schemas = themeSchemas } + + const queryString = qs.stringify( + { + action: `${extensionType}_information`, + request: { + slug, + fields: { + active_installs: 1, + sections: 0, + homepage: 0, + tags: 0, + screenshot_url: 0, + downloaded: 1, + last_updated: 1, + requires_php: 1, + }, + }, + }, + { encode: false } + ) + const json = await this._requestJson({ url, schema: schemas, options: { - qs: { - action: `${extensionType}_information`, - request: { - slug, - fields: { - active_installs: 1, - sections: 0, - homepage: 0, - tags: 0, - screenshot_url: 0, - downloaded: 1, - last_updated: 1, - requires_php: 1, - }, - }, - }, - qsStringifyOptions: { - encode: false, - }, + qs: queryString, }, }) if ('error' in json) {