diff --git a/lib/all-badge-examples.js b/lib/all-badge-examples.js index 7eda013c83..d4370fa774 100644 --- a/lib/all-badge-examples.js +++ b/lib/all-badge-examples.js @@ -735,16 +735,6 @@ const allBadgeExamples = [ previewUrl: '/vscode-marketplace/d/ritwickdey.LiveServer.svg', keywords: ['vscode-marketplace'], }, - { - title: 'Eclipse Marketplace', - previewUrl: '/eclipse-marketplace/dt/notepad4e.svg', - keywords: ['eclipse', 'marketplace'], - }, - { - title: 'Eclipse Marketplace', - previewUrl: '/eclipse-marketplace/dm/notepad4e.svg', - keywords: ['eclipse', 'marketplace'], - }, { title: 'JetBrains IntelliJ plugins', previewUrl: '/jetbrains/plugin/d/1347-scala.svg', @@ -1390,11 +1380,6 @@ const allBadgeExamples = [ previewUrl: '/vscode-marketplace/v/ritwickdey.LiveServer.svg', keywords: ['vscode-marketplace'], }, - { - title: 'Eclipse Marketplace', - previewUrl: '/eclipse-marketplace/v/notepad4e.svg', - keywords: ['eclipse', 'marketplace'], - }, { title: 'iTunes App Store', previewUrl: '/itunes/v/803453959.svg', @@ -1619,16 +1604,6 @@ const allBadgeExamples = [ previewUrl: '/swagger/valid/2.0/https/raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/petstore-expanded.json.svg', }, - { - title: 'Eclipse Marketplace', - previewUrl: '/eclipse-marketplace/favorites/notepad4e.svg', - keywords: ['eclipse', 'marketplace'], - }, - { - title: 'Eclipse Marketplace', - previewUrl: '/eclipse-marketplace/last-update/notepad4e.svg', - keywords: ['eclipse', 'marketplace'], - }, { title: 'Vaadin Directory', previewUrl: '/vaadin-directory/status/vaadinvaadin-grid.svg', diff --git a/lib/error-helper.js b/lib/error-helper.js index 9e7fee8a3a..4d96d31fa1 100644 --- a/lib/error-helper.js +++ b/lib/error-helper.js @@ -50,18 +50,6 @@ checkErrorResponse.asPromise = function(errorMessages = {}) { } } -async function asJson({ buffer, res }) { - try { - return JSON.parse(buffer) - } catch (err) { - throw new InvalidResponse({ - prettyMessage: 'unparseable json response', - underlyingError: err, - }) - } -} - module.exports = { checkErrorResponse, - asJson, } diff --git a/package-lock.json b/package-lock.json index ace18865cb..75ac8aecc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -352,8 +352,7 @@ "acorn": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-2.7.0.tgz", - "integrity": "sha1-q259nYhqrKiwhbwzEreaGYQz8Oc=", - "optional": true + "integrity": "sha1-q259nYhqrKiwhbwzEreaGYQz8Oc=" }, "acorn-dynamic-import": { "version": "2.0.2", @@ -2506,8 +2505,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -2528,14 +2526,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2550,20 +2546,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -2680,8 +2673,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -2693,7 +2685,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2708,7 +2699,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2716,14 +2706,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.2.4", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -2742,7 +2730,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -2823,8 +2810,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -2836,7 +2822,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -2922,8 +2907,7 @@ "safe-buffer": { "version": "5.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -2959,7 +2943,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -2979,7 +2962,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -3023,14 +3005,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -3794,8 +3774,7 @@ "cssom": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.2.tgz", - "integrity": "sha1-uANhcMefB6kP8vFuIihAJ6JDhIs=", - "optional": true + "integrity": "sha1-uANhcMefB6kP8vFuIihAJ6JDhIs=" }, "cssstyle": { "version": "0.2.37", @@ -4850,14 +4829,6 @@ "dev": true, "requires": { "eslint-config-standard-jsx": "^6.0.1" - }, - "dependencies": { - "eslint-config-standard-jsx": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/eslint-config-standard-jsx/-/eslint-config-standard-jsx-6.0.2.tgz", - "integrity": "sha512-D+YWAoXw+2GIdbMBRAzWwr1ZtvnSf4n4yL0gKGg7ShUOGXkSOLerI17K4F6LdQMJPNMoWYqepzQD/fKY+tXNSg==", - "dev": true - } } }, "eslint-import-resolver-node": { @@ -5411,6 +5382,14 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, + "fast-xml-parser": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.12.0.tgz", + "integrity": "sha512-7IcV+5z6uZoIgGi3GxJTt94V8DGONWk/nL6ynGJdy+Mg219o2nLXjZeB1WDQRL+MOBKMtVXofioFW1POEdahKQ==", + "requires": { + "nimnjs": "^1.3.2" + } + }, "fbjs": { "version": "0.8.16", "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.16.tgz", @@ -8577,6 +8556,25 @@ "integrity": "sha512-2NpiFHqC87y/zFke0fC0spBXL3bBsoh/p5H1EFhshxjCR5+0g2d6BiXbUFz9v1sAcxsk2htp2eQnNIci2dIYcA==", "dev": true }, + "nimn-date-parser": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/nimn-date-parser/-/nimn-date-parser-1.0.0.tgz", + "integrity": "sha512-1Nf+x3EeMvHUiHsVuEhiZnwA8RMeOBVTQWfB1S2n9+i6PYCofHd2HRMD+WOHIHYshy4T4Gk8wQoCol7Hq3av8Q==" + }, + "nimn_schema_builder": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/nimn_schema_builder/-/nimn_schema_builder-1.1.0.tgz", + "integrity": "sha512-DK5/B8CM4qwzG2URy130avcwPev4uO0ev836FbQyKo1ms6I9z/i6EJyiZ+d9xtgloxUri0W+5gfR8YbPq7SheA==" + }, + "nimnjs": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/nimnjs/-/nimnjs-1.3.2.tgz", + "integrity": "sha512-TIOtI4iqkQrUM1tiM76AtTQem0c7e56SkDZ7sj1d1MfUsqRcq2ZWQvej/O+HBTZV7u/VKnwlKTDugK/75IRPPw==", + "requires": { + "nimn-date-parser": "^1.0.0", + "nimn_schema_builder": "^1.0.0" + } + }, "nise": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/nise/-/nise-1.4.2.tgz", @@ -8873,7 +8871,6 @@ "version": "0.1.4", "bundled": true, "dev": true, - "optional": true, "requires": { "kind-of": "^3.0.2", "longest": "^1.0.1", @@ -9195,8 +9192,7 @@ "is-buffer": { "version": "1.1.6", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "is-builtin-module": { "version": "1.0.0", @@ -9280,7 +9276,6 @@ "version": "3.2.2", "bundled": true, "dev": true, - "optional": true, "requires": { "is-buffer": "^1.1.5" } @@ -9327,8 +9322,7 @@ "longest": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "lru-cache": { "version": "4.1.3", @@ -9594,8 +9588,7 @@ "repeat-string": { "version": "1.6.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "require-directory": { "version": "2.1.1", diff --git a/package.json b/package.json index c1738cdc65..37fadc3a0e 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dot": "~1.1.2", "emojic": "^1.1.14", "escape-string-regexp": "^1.0.5", + "fast-xml-parser": "^3.12.0", "fsos": "^1.1.3", "glob": "^7.1.1", "gm": "^1.23.0", diff --git a/services/base-http.js b/services/base-http.js deleted file mode 100644 index c1b2888711..0000000000 --- a/services/base-http.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -// See available emoji at http://emoji.muan.co/ -const emojic = require('emojic') -const { checkErrorResponse } = require('../lib/error-helper') -const BaseService = require('./base') -const trace = require('./trace') - -class BaseHTTPService extends BaseService { - async _requestHTTP({ url, options = {}, errorMessages = {} }) { - const logTrace = (...args) => trace.logTrace('fetch', ...args) - logTrace(emojic.bowAndArrow, 'Request', url, '\n', options) - return this._sendAndCacheRequest(url, options) - .then(({ res, buffer }) => { - logTrace(emojic.dart, 'Response status code', res.statusCode) - return { res, buffer } - }) - .then(checkErrorResponse.asPromise(errorMessages)) - } -} - -module.exports = BaseHTTPService diff --git a/services/base-json.js b/services/base-json.js index bfeec0244d..dde261d0f1 100644 --- a/services/base-json.js +++ b/services/base-json.js @@ -2,59 +2,35 @@ // See available emoji at http://emoji.muan.co/ const emojic = require('emojic') -const Joi = require('joi') -const { asJson } = require('../lib/error-helper') -const BaseHTTPService = require('./base-http') -const { InvalidResponse } = require('./errors') +const BaseService = require('./base') const trace = require('./trace') +const { InvalidResponse } = require('./errors') -class BaseJsonService extends BaseHTTPService { - static _validate(json, schema) { - const { error, value } = Joi.validate(json, schema, { - allowUnknown: true, - stripUnknown: true, - }) - if (error) { - trace.logTrace( - 'validate', - emojic.womanShrugging, - 'Response did not match schema', - error.message - ) - throw new InvalidResponse({ - prettyMessage: 'invalid json response', - underlyingError: error, - }) - } else { - trace.logTrace( - 'validate', - emojic.bathtub, - 'JSON after validation', - value, - { deep: true } - ) - return value - } - } - +class BaseJsonService extends BaseService { async _requestJson({ schema, url, options = {}, errorMessages = {} }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) - if (!schema || !schema.isJoi) { - throw Error('A Joi schema is required') - } const mergedOptions = { ...{ headers: { Accept: 'application/json' } }, ...options, } - return this._requestHTTP({ url, options: mergedOptions, errorMessages }) - .then(asJson) - .then(json => { - logTrace(emojic.dart, 'Response JSON (before validation)', json, { - deep: true, - }) - return json + const { buffer } = await this._request({ + url, + options: mergedOptions, + errorMessages, + }) + let json + try { + json = JSON.parse(buffer) + } catch (err) { + throw new InvalidResponse({ + prettyMessage: 'unparseable json response', + underlyingError: err, }) - .then(json => this.constructor._validate(json, schema)) + } + logTrace(emojic.dart, 'Response JSON (before validation)', json, { + deep: true, + }) + return this.constructor._validate(json, schema) } } diff --git a/services/base-json.spec.js b/services/base-json.spec.js index bbcf309bd1..737a9e9b34 100644 --- a/services/base-json.spec.js +++ b/services/base-json.spec.js @@ -6,8 +6,6 @@ const { expect } = chai const sinon = require('sinon') const BaseJsonService = require('./base-json') -const { invalidJSON } = require('./response-fixtures') -const trace = require('./trace') chai.use(require('chai-as-promised')) @@ -27,11 +25,11 @@ class DummyJsonService extends BaseJsonService { } async handle() { - const { value } = await this._requestJson({ + const { requiredString } = await this._requestJson({ schema: dummySchema, url: 'http://example.com/foo.json', }) - return { message: value } + return { message: requiredString } } } @@ -87,83 +85,52 @@ describe('BaseJsonService', function() { }) }) - it('handles unparseable json responses', async function() { - const sendAndCacheRequest = async () => ({ - buffer: invalidJSON, - res: { statusCode: 200 }, - }) - const serviceInstance = new DummyJsonService( - { sendAndCacheRequest }, - { handleInternalErrors: false } - ) - const serviceData = await serviceInstance.invokeHandler({}, {}) - expect(serviceData).to.deep.equal({ - color: 'lightgray', - message: 'unparseable json response', - }) - }) - - context('a schema is not provided', function() { - it('throws the expected error', async function() { - const serviceInstance = new DummyJsonService( - {}, - { handleInternalErrors: false } - ) - expect( - serviceInstance._requestJson({ schema: undefined }) - ).to.be.rejectedWith('A Joi schema is required') - }) - }) - - describe('logging', function() { - let sandbox - beforeEach(function() { - sandbox = sinon.createSandbox() - }) - afterEach(function() { - sandbox.restore() - }) - beforeEach(function() { - sandbox.stub(trace, 'logTrace') - }) - - it('logs valid responses', async function() { + describe('Making badges', function() { + it('handles valid json responses', async function() { const sendAndCacheRequest = async () => ({ - buffer: JSON.stringify({ requiredString: 'bar' }), + buffer: '{"requiredString": "some-string"}', res: { statusCode: 200 }, }) const serviceInstance = new DummyJsonService( { sendAndCacheRequest }, { handleInternalErrors: false } ) - await serviceInstance.invokeHandler({}, {}) - expect(trace.logTrace).to.be.calledWithMatch( - 'validate', - sinon.match.string, - 'JSON after validation', - { requiredString: 'bar' }, - { deep: true } - ) + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + message: 'some-string', + }) }) - it('logs invalid responses', async function() { + it('handles json responses which do not match the schema', async function() { const sendAndCacheRequest = async () => ({ - buffer: JSON.stringify({ - requiredString: ['this', "shouldn't", 'work'], - }), + buffer: '{"unexpectedKey": "some-string"}', res: { statusCode: 200 }, }) const serviceInstance = new DummyJsonService( { sendAndCacheRequest }, { handleInternalErrors: false } ) - await serviceInstance.invokeHandler({}, {}) - expect(trace.logTrace).to.be.calledWithMatch( - 'validate', - sinon.match.string, - 'Response did not match schema', - 'child "requiredString" fails because ["requiredString" must be a string]' + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + color: 'lightgray', + message: 'invalid response data', + }) + }) + + it('handles unparseable json responses', async function() { + const sendAndCacheRequest = async () => ({ + buffer: 'not json', + res: { statusCode: 200 }, + }) + const serviceInstance = new DummyJsonService( + { sendAndCacheRequest }, + { handleInternalErrors: false } ) + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + color: 'lightgray', + message: 'unparseable json response', + }) }) }) }) diff --git a/services/base-xml.js b/services/base-xml.js new file mode 100644 index 0000000000..a4df36821e --- /dev/null +++ b/services/base-xml.js @@ -0,0 +1,37 @@ +'use strict' + +// See available emoji at http://emoji.muan.co/ +const emojic = require('emojic') +const fastXmlParser = require('fast-xml-parser') +const BaseService = require('./base') +const trace = require('./trace') +const { InvalidResponse } = require('./errors') + +class BaseXmlService extends BaseService { + async _requestXml({ schema, url, options = {}, errorMessages = {} }) { + const logTrace = (...args) => trace.logTrace('fetch', ...args) + const mergedOptions = { + ...{ headers: { Accept: 'application/xml, text/xml' } }, + ...options, + } + const { buffer } = await this._request({ + url, + options: mergedOptions, + errorMessages, + }) + const validateResult = fastXmlParser.validate(buffer) + if (validateResult !== true) { + throw new InvalidResponse({ + prettyMessage: 'unparseable xml response', + underlyingError: validateResult.err, + }) + } + const xml = fastXmlParser.parse(buffer) + logTrace(emojic.dart, 'Response XML (before validation)', xml, { + deep: true, + }) + return this.constructor._validate(xml, schema) + } +} + +module.exports = BaseXmlService diff --git a/services/base-xml.spec.js b/services/base-xml.spec.js new file mode 100644 index 0000000000..8e3ac1fccb --- /dev/null +++ b/services/base-xml.spec.js @@ -0,0 +1,136 @@ +'use strict' + +const Joi = require('joi') +const chai = require('chai') +const { expect } = chai +const sinon = require('sinon') + +const BaseXmlService = require('./base-xml') + +chai.use(require('chai-as-promised')) + +const dummySchema = Joi.object({ + requiredString: Joi.string().required(), +}).required() + +class DummyXmlService extends BaseXmlService { + static get category() { + return 'cat' + } + + static get url() { + return { + base: 'foo', + } + } + + async handle() { + const { requiredString } = await this._requestXml({ + schema: dummySchema, + url: 'http://example.com/foo.xml', + }) + return { message: requiredString } + } +} + +describe('BaseXmlService', function() { + describe('Making requests', function() { + let sendAndCacheRequest, serviceInstance + beforeEach(function() { + sendAndCacheRequest = sinon.stub().returns( + Promise.resolve({ + buffer: 'some-string', + res: { statusCode: 200 }, + }) + ) + serviceInstance = new DummyXmlService( + { sendAndCacheRequest }, + { handleInternalErrors: false } + ) + }) + + it('invokes _sendAndCacheRequest', async function() { + await serviceInstance.invokeHandler({}, {}) + + expect(sendAndCacheRequest).to.have.been.calledOnceWith( + 'http://example.com/foo.xml', + { + headers: { Accept: 'application/xml, text/xml' }, + } + ) + }) + + it('forwards options to _sendAndCacheRequest', async function() { + Object.assign(serviceInstance, { + async handle() { + const { value } = await this._requestXml({ + schema: dummySchema, + url: 'http://example.com/foo.xml', + options: { method: 'POST', qs: { queryParam: 123 } }, + }) + return { message: value } + }, + }) + + await serviceInstance.invokeHandler({}, {}) + + expect(sendAndCacheRequest).to.have.been.calledOnceWith( + 'http://example.com/foo.xml', + { + headers: { Accept: 'application/xml, text/xml' }, + method: 'POST', + qs: { queryParam: 123 }, + } + ) + }) + }) + + describe('Making badges', function() { + it('handles valid xml responses', async function() { + const sendAndCacheRequest = async () => ({ + buffer: 'some-string', + res: { statusCode: 200 }, + }) + const serviceInstance = new DummyXmlService( + { sendAndCacheRequest }, + { handleInternalErrors: false } + ) + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + message: 'some-string', + }) + }) + + it('handles xml responses which do not match the schema', async function() { + const sendAndCacheRequest = async () => ({ + buffer: 'some-string', + res: { statusCode: 200 }, + }) + const serviceInstance = new DummyXmlService( + { sendAndCacheRequest }, + { handleInternalErrors: false } + ) + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + color: 'lightgray', + message: 'invalid response data', + }) + }) + + it('handles unparseable xml responses', async function() { + const sendAndCacheRequest = async () => ({ + buffer: 'not xml', + res: { statusCode: 200 }, + }) + const serviceInstance = new DummyXmlService( + { sendAndCacheRequest }, + { handleInternalErrors: false } + ) + const serviceData = await serviceInstance.invokeHandler({}, {}) + expect(serviceData).to.deep.equal({ + color: 'lightgray', + message: 'unparseable xml response', + }) + }) + }) +}) diff --git a/services/base.js b/services/base.js index 1d03976aec..573831eade 100644 --- a/services/base.js +++ b/services/base.js @@ -2,6 +2,7 @@ // See available emoji at http://emoji.muan.co/ const emojic = require('emojic') +const Joi = require('joi') const { NotFound, InvalidResponse, @@ -9,6 +10,7 @@ const { InvalidParameter, Deprecated, } = require('./errors') +const { checkErrorResponse } = require('../lib/error-helper') const queryString = require('query-string') const { makeLogo, @@ -306,6 +308,45 @@ class BaseService { }) ) } + + static _validate(data, schema) { + if (!schema || !schema.isJoi) { + throw Error('A Joi schema is required') + } + const { error, value } = Joi.validate(data, schema, { + allowUnknown: true, + stripUnknown: true, + }) + if (error) { + trace.logTrace( + 'validate', + emojic.womanShrugging, + 'Response did not match schema', + error.message + ) + throw new InvalidResponse({ + prettyMessage: 'invalid response data', + underlyingError: error, + }) + } else { + trace.logTrace( + 'validate', + emojic.bathtub, + 'Data after validation', + value, + { deep: true } + ) + return value + } + } + + async _request({ url, options = {}, errorMessages = {} }) { + const logTrace = (...args) => trace.logTrace('fetch', ...args) + logTrace(emojic.bowAndArrow, 'Request', url, '\n', options) + const { res, buffer } = await this._sendAndCacheRequest(url, options) + logTrace(emojic.dart, 'Response status code', res.statusCode) + return checkErrorResponse.asPromise(errorMessages)({ buffer, res }) + } } module.exports = BaseService diff --git a/services/base.spec.js b/services/base.spec.js index 01e5e9a860..ae13e20895 100644 --- a/services/base.spec.js +++ b/services/base.spec.js @@ -1,5 +1,6 @@ 'use strict' +const Joi = require('joi') const { expect } = require('chai') const { test, given, forCases } = require('sazerac') const sinon = require('sinon') @@ -421,4 +422,127 @@ describe('BaseService', function() { expect(url).to.equal('/badge/123--123-abc--abc-blue') }) }) + + describe('validate', function() { + const dummySchema = Joi.object({ + requiredString: Joi.string().required(), + }).required() + + let sandbox + beforeEach(function() { + sandbox = sinon.createSandbox() + }) + afterEach(function() { + sandbox.restore() + }) + beforeEach(function() { + sandbox.stub(trace, 'logTrace') + }) + + it('throws the expected error if schema is not provided', async function() { + try { + DummyService._validate({ requiredString: 'bar' }, undefined) + expect.fail('Expected to throw') + } catch (e) { + expect(e).to.be.an.instanceof(Error) + expect(e.message).to.equal('A Joi schema is required') + } + }) + + it('logs valid responses', async function() { + DummyService._validate({ requiredString: 'bar' }, dummySchema) + expect(trace.logTrace).to.be.calledWithMatch( + 'validate', + sinon.match.string, + 'Data after validation', + { requiredString: 'bar' }, + { deep: true } + ) + }) + + it('logs invalid responses and throws error', async function() { + try { + DummyService._validate( + { requiredString: ['this', "shouldn't", 'work'] }, + dummySchema + ) + expect.fail('Expected to throw') + } catch (e) { + expect(e).to.be.an.instanceof(InvalidResponse) + expect(e.message).to.equal( + 'Invalid Response: child "requiredString" fails because ["requiredString" must be a string]' + ) + expect(e.prettyMessage).to.equal('invalid response data') + } + expect(trace.logTrace).to.be.calledWithMatch( + 'validate', + sinon.match.string, + 'Response did not match schema', + 'child "requiredString" fails because ["requiredString" must be a string]' + ) + }) + }) + + describe('request', function() { + let sandbox + beforeEach(function() { + sandbox = sinon.createSandbox() + }) + afterEach(function() { + sandbox.restore() + }) + beforeEach(function() { + sandbox.stub(trace, 'logTrace') + }) + + it('logs appropriate information', async function() { + const sendAndCacheRequest = async () => ({ + buffer: '', + res: { statusCode: 200 }, + }) + const serviceInstance = new DummyService( + { sendAndCacheRequest }, + defaultConfig + ) + + const url = 'some-url' + const options = { headers: { Cookie: 'some-cookie' } } + await serviceInstance._request({ url, options }) + + expect(trace.logTrace).to.be.calledWithMatch( + 'fetch', + sinon.match.string, + 'Request', + url, + '\n', + options + ) + expect(trace.logTrace).to.be.calledWithMatch( + 'fetch', + sinon.match.string, + 'Response status code', + 200 + ) + }) + + it('handles errors', async function() { + const sendAndCacheRequest = async () => ({ + buffer: '', + res: { statusCode: 404 }, + }) + const serviceInstance = new DummyService( + { sendAndCacheRequest }, + defaultConfig + ) + + try { + await serviceInstance._request({}) + expect.fail('Expected to throw') + } catch (e) { + expect(e).to.be.an.instanceof(NotFound) + expect(e.message).to.equal('Not Found') + expect(e.prettyMessage).to.equal('not found') + } + }) + }) }) diff --git a/services/circleci/circleci.tester.js b/services/circleci/circleci.tester.js index cdcd26e914..cfdf72cc18 100644 --- a/services/circleci/circleci.tester.js +++ b/services/circleci/circleci.tester.js @@ -57,6 +57,6 @@ t.create('circle ci (invalid json)') ) .expectJSON({ name: 'build', - value: 'invalid json response', + value: 'invalid response data', colorB: '#9f9f9f', }) diff --git a/services/eclipse-marketplace/eclipse-marketplace-base.js b/services/eclipse-marketplace/eclipse-marketplace-base.js new file mode 100644 index 0000000000..d2fa5cae9f --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-base.js @@ -0,0 +1,21 @@ +'use strict' + +const BaseXmlService = require('../base-xml') + +module.exports = class EclipseMarketplaceBase extends BaseXmlService { + static buildUrl(base) { + return { + base, + format: '(.+)', + capture: ['name'], + } + } + + async fetch({ name, schema }) { + return this._requestXml({ + schema, + url: `https://marketplace.eclipse.org/content/${name}/api/p`, + errorMessages: { 404: 'solution not found' }, + }) + } +} diff --git a/services/eclipse-marketplace/eclipse-marketplace-downloads.service.js b/services/eclipse-marketplace/eclipse-marketplace-downloads.service.js new file mode 100644 index 0000000000..b699a45785 --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-downloads.service.js @@ -0,0 +1,78 @@ +'use strict' + +const Joi = require('joi') +const EclipseMarketplaceBase = require('./eclipse-marketplace-base') +const { metric } = require('../../lib/text-formatters') +const { + downloadCount: downloadCountColor, +} = require('../../lib/color-formatters') +const { nonNegativeInteger } = require('../validators') + +const monthlyResponseSchema = Joi.object({ + marketplace: Joi.object({ + node: Joi.object({ + installsrecent: nonNegativeInteger, + }), + }), +}).required() + +const totalResponseSchema = Joi.object({ + marketplace: Joi.object({ + node: Joi.object({ + installstotal: nonNegativeInteger, + }), + }), +}).required() + +function DownloadsForInterval(interval) { + const { base, schema, messageSuffix = '' } = { + month: { + base: 'eclipse-marketplace/dm', + messageSuffix: '/month', + schema: monthlyResponseSchema, + }, + total: { + base: 'eclipse-marketplace/dt', + schema: totalResponseSchema, + }, + }[interval] + + return class EclipseMarketplaceDownloads extends EclipseMarketplaceBase { + static get category() { + return 'downloads' + } + + static get examples() { + return [ + { + title: 'Eclipse Marketplace', + exampleUrl: 'notepad4e', + urlPattern: ':name', + staticExample: this.render({ downloads: 30000 }), + }, + ] + } + + static get url() { + return this.buildUrl(base) + } + + static render({ downloads }) { + return { + message: `${metric(downloads)}${messageSuffix}`, + color: downloadCountColor(downloads), + } + } + + async handle({ name }) { + const { marketplace } = await this.fetch({ name, schema }) + const downloads = + interval === 'total' + ? marketplace.node.installstotal + : marketplace.node.installsrecent + return this.constructor.render({ downloads }) + } + } +} + +module.exports = ['month', 'total'].map(DownloadsForInterval) diff --git a/services/eclipse-marketplace/eclipse-marketplace-downloads.tester.js b/services/eclipse-marketplace/eclipse-marketplace-downloads.tester.js new file mode 100644 index 0000000000..6ec4e7ea9a --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-downloads.tester.js @@ -0,0 +1,38 @@ +'use strict' + +const Joi = require('joi') +const ServiceTester = require('../service-tester') + +const { isMetric, isMetricOverTimePeriod } = require('../test-validators') + +const t = new ServiceTester({ + id: 'eclipse-marketplace-downloads', + title: 'EclipseMarketplaceDownloads', + pathPrefix: '/eclipse-marketplace', +}) +module.exports = t + +t.create('total marketplace downloads') + .get('/dt/notepad4e.json') + .expectJSONTypes( + Joi.object().keys({ + name: 'downloads', + value: isMetric, + }) + ) + +t.create('monthly marketplace downloads') + .get('/dm/notepad4e.json') + .expectJSONTypes( + Joi.object().keys({ + name: 'downloads', + value: isMetricOverTimePeriod, + }) + ) + +t.create('downloads for unknown solution') + .get('/dt/this-does-not-exist.json') + .expectJSON({ + name: 'downloads', + value: 'solution not found', + }) diff --git a/services/eclipse-marketplace/eclipse-marketplace-favorites.service.js b/services/eclipse-marketplace/eclipse-marketplace-favorites.service.js new file mode 100644 index 0000000000..5a55a06e32 --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-favorites.service.js @@ -0,0 +1,54 @@ +'use strict' + +const Joi = require('joi') +const EclipseMarketplaceBase = require('./eclipse-marketplace-base') +const { nonNegativeInteger } = require('../validators') + +const favoritesResponseSchema = Joi.object({ + marketplace: Joi.object({ + node: Joi.object({ + favorited: nonNegativeInteger, + }), + }), +}).required() + +module.exports = class EclipseMarketplaceFavorites extends EclipseMarketplaceBase { + static get category() { + return 'other' + } + + static get defaultBadgeData() { + return { label: 'favorites' } + } + + static get examples() { + return [ + { + title: 'Eclipse Marketplace', + exampleUrl: 'notepad4e', + urlPattern: ':name', + staticExample: this.render({ favorited: 55 }), + }, + ] + } + + static get url() { + return this.buildUrl('eclipse-marketplace/favorites') + } + + static render({ favorited }) { + return { + message: favorited, + color: 'brightgreen', + } + } + + async handle({ name }) { + const { marketplace } = await this.fetch({ + name, + schema: favoritesResponseSchema, + }) + const favorited = marketplace.node.favorited + return this.constructor.render({ favorited }) + } +} diff --git a/services/eclipse-marketplace/eclipse-marketplace-favorites.tester.js b/services/eclipse-marketplace/eclipse-marketplace-favorites.tester.js new file mode 100644 index 0000000000..288872f2b9 --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-favorites.tester.js @@ -0,0 +1,29 @@ +'use strict' + +const Joi = require('joi') +const ServiceTester = require('../service-tester') + +const t = new ServiceTester({ + id: 'eclipse-marketplace-favorites', + title: 'EclipseMarketplaceFavorites', + pathPrefix: '/eclipse-marketplace', +}) +module.exports = t + +t.create('favorites count') + .get('/favorites/notepad4e.json') + .expectJSONTypes( + Joi.object().keys({ + name: 'favorites', + value: Joi.number() + .integer() + .positive(), + }) + ) + +t.create('favorites for unknown solution') + .get('/favorites/this-does-not-exist.json') + .expectJSON({ + name: 'favorites', + value: 'solution not found', + }) diff --git a/services/eclipse-marketplace/eclipse-marketplace-update.service.js b/services/eclipse-marketplace/eclipse-marketplace-update.service.js new file mode 100644 index 0000000000..bcf09f406f --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-update.service.js @@ -0,0 +1,56 @@ +'use strict' + +const Joi = require('joi') +const EclipseMarketplaceBase = require('./eclipse-marketplace-base') +const { formatDate } = require('../../lib/text-formatters') +const { age: ageColor } = require('../../lib/color-formatters') +const { nonNegativeInteger } = require('../validators') + +const updateResponseSchema = Joi.object({ + marketplace: Joi.object({ + node: Joi.object({ + changed: nonNegativeInteger, + }), + }), +}).required() + +module.exports = class EclipseMarketplaceUpdate extends EclipseMarketplaceBase { + static get category() { + return 'other' + } + + static get defaultBadgeData() { + return { label: 'updated' } + } + + static get examples() { + return [ + { + title: 'Eclipse Marketplace', + exampleUrl: 'notepad4e', + urlPattern: ':name', + staticExample: this.render({ date: new Date().getTime() }), + }, + ] + } + + static get url() { + return this.buildUrl('eclipse-marketplace/last-update') + } + + static render({ date }) { + return { + message: formatDate(date), + color: ageColor(date), + } + } + + async handle({ name }) { + const { marketplace } = await this.fetch({ + name, + schema: updateResponseSchema, + }) + const date = 1000 * parseInt(marketplace.node.changed) + return this.constructor.render({ date }) + } +} diff --git a/services/eclipse-marketplace/eclipse-marketplace-update.tester.js b/services/eclipse-marketplace/eclipse-marketplace-update.tester.js new file mode 100644 index 0000000000..fc6d70b1fb --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-update.tester.js @@ -0,0 +1,29 @@ +'use strict' + +const Joi = require('joi') +const ServiceTester = require('../service-tester') + +const { isFormattedDate } = require('../test-validators') + +const t = new ServiceTester({ + id: 'eclipse-marketplace-update', + title: 'EclipseMarketplaceUpdate', + pathPrefix: '/eclipse-marketplace', +}) +module.exports = t + +t.create('last update date') + .get('/last-update/notepad4e.json') + .expectJSONTypes( + Joi.object().keys({ + name: 'updated', + value: isFormattedDate, + }) + ) + +t.create('last update for unknown solution') + .get('/last-update/this-does-not-exist.json') + .expectJSON({ + name: 'updated', + value: 'solution not found', + }) diff --git a/services/eclipse-marketplace/eclipse-marketplace-version.service.js b/services/eclipse-marketplace/eclipse-marketplace-version.service.js new file mode 100644 index 0000000000..be3326858d --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-version.service.js @@ -0,0 +1,51 @@ +'use strict' + +const Joi = require('joi') +const EclipseMarketplaceBase = require('./eclipse-marketplace-base') +const { renderVersionBadge } = require('../../lib/version') + +const versionResponseSchema = Joi.object({ + marketplace: Joi.object({ + node: Joi.object({ + version: Joi.string().required(), + }), + }), +}).required() + +module.exports = class EclipseMarketplaceVersion extends EclipseMarketplaceBase { + static get category() { + return 'version' + } + + static get defaultBadgeData() { + return { label: 'eclipse marketplace' } + } + + static get examples() { + return [ + { + title: 'Eclipse Marketplace', + exampleUrl: 'notepad4e', + urlPattern: ':name', + staticExample: this.render({ version: '1.0.1' }), + }, + ] + } + + static get url() { + return this.buildUrl('eclipse-marketplace/v') + } + + static render({ version }) { + return renderVersionBadge({ version }) + } + + async handle({ name }) { + const { marketplace } = await this.fetch({ + name, + schema: versionResponseSchema, + }) + const version = marketplace.node.version + return this.constructor.render({ version }) + } +} diff --git a/services/eclipse-marketplace/eclipse-marketplace-version.tester.js b/services/eclipse-marketplace/eclipse-marketplace-version.tester.js new file mode 100644 index 0000000000..24f5d349ef --- /dev/null +++ b/services/eclipse-marketplace/eclipse-marketplace-version.tester.js @@ -0,0 +1,29 @@ +'use strict' + +const Joi = require('joi') +const ServiceTester = require('../service-tester') + +const { isVPlusDottedVersionAtLeastOne } = require('../test-validators') + +const t = new ServiceTester({ + id: 'eclipse-marketplace-version', + title: 'EclipseMarketplaceVersion', + pathPrefix: '/eclipse-marketplace', +}) +module.exports = t + +t.create('marketplace version') + .get('/v/notepad4e.json') + .expectJSONTypes( + Joi.object().keys({ + name: 'eclipse marketplace', + value: isVPlusDottedVersionAtLeastOne, + }) + ) + +t.create('last update for unknown solution') + .get('/v/this-does-not-exist.json') + .expectJSON({ + name: 'eclipse marketplace', + value: 'solution not found', + }) diff --git a/services/eclipse-marketplace/eclipse-marketplace.service.js b/services/eclipse-marketplace/eclipse-marketplace.service.js deleted file mode 100644 index 2ef89754fe..0000000000 --- a/services/eclipse-marketplace/eclipse-marketplace.service.js +++ /dev/null @@ -1,93 +0,0 @@ -'use strict' - -const xml2js = require('xml2js') -const LegacyService = require('../legacy-service') -const { - makeBadgeData: getBadgeData, - makeLabel: getLabel, -} = require('../../lib/badge-data') -const { - metric, - addv: versionText, - formatDate, -} = require('../../lib/text-formatters') -const { - age: ageColor, - downloadCount: downloadCountColor, - version: versionColor, -} = require('../../lib/color-formatters') - -module.exports = class EclipseMarketplace extends LegacyService { - static registerLegacyRouteHandler({ camp, cache }) { - camp.route( - /^\/eclipse-marketplace\/(dt|dm|v|favorites|last-update)\/(.*)\.(svg|png|gif|jpg|json)$/, - cache((data, match, sendBadge, request) => { - const type = match[1] - const project = match[2] - const format = match[3] - const apiUrl = - 'https://marketplace.eclipse.org/content/' + project + '/api/p' - const badgeData = getBadgeData('eclipse marketplace', data) - request(apiUrl, (err, res, buffer) => { - if (err != null) { - badgeData.text[1] = 'inaccessible' - sendBadge(format, badgeData) - return - } - xml2js.parseString(buffer.toString(), (parseErr, parsedData) => { - if (parseErr != null) { - badgeData.text[1] = 'invalid' - sendBadge(format, badgeData) - return - } - try { - const projectNode = parsedData.marketplace.node[0] - switch (type) { - case 'dt': { - badgeData.text[0] = getLabel('downloads', data) - const downloads = parseInt(projectNode.installstotal[0]) - badgeData.text[1] = metric(downloads) - badgeData.colorscheme = downloadCountColor(downloads) - break - } - case 'dm': { - badgeData.text[0] = getLabel('downloads', data) - const monthlydownloads = parseInt( - projectNode.installsrecent[0] - ) - badgeData.text[1] = metric(monthlydownloads) + '/month' - badgeData.colorscheme = downloadCountColor(monthlydownloads) - break - } - case 'v': { - badgeData.text[1] = versionText(projectNode.version[0]) - badgeData.colorscheme = versionColor(projectNode.version[0]) - break - } - case 'favorites': { - badgeData.text[0] = getLabel('favorites', data) - badgeData.text[1] = parseInt(projectNode.favorited[0]) - badgeData.colorscheme = 'brightgreen' - break - } - case 'last-update': { - const date = 1000 * parseInt(projectNode.changed[0]) - badgeData.text[0] = getLabel('updated', data) - badgeData.text[1] = formatDate(date) - badgeData.colorscheme = ageColor(Date.parse(date)) - break - } - default: - throw Error('Unreachable due to regex') - } - sendBadge(format, badgeData) - } catch (e) { - badgeData.text[1] = 'invalid' - sendBadge(format, badgeData) - } - }) - }) - }) - ) - } -} diff --git a/services/eclipse-marketplace/eclipse-marketplace.tester.js b/services/eclipse-marketplace/eclipse-marketplace.tester.js deleted file mode 100644 index c90a064370..0000000000 --- a/services/eclipse-marketplace/eclipse-marketplace.tester.js +++ /dev/null @@ -1,60 +0,0 @@ -'use strict' - -const Joi = require('joi') -const ServiceTester = require('../service-tester') -const { - isFormattedDate, - isMetric, - isMetricOverTimePeriod, - isVPlusDottedVersionAtLeastOne, -} = require('../test-validators') - -const t = new ServiceTester({ id: 'eclipse-marketplace', title: 'Eclipse' }) -module.exports = t - -t.create('total marketplace downloads') - .get('/dt/notepad4e.json') - .expectJSONTypes( - Joi.object().keys({ - name: 'downloads', - value: isMetric, - }) - ) - -t.create('monthly marketplace downloads') - .get('/dm/notepad4e.json') - .expectJSONTypes( - Joi.object().keys({ - name: 'downloads', - value: isMetricOverTimePeriod, - }) - ) - -t.create('marketplace version') - .get('/v/notepad4e.json') - .expectJSONTypes( - Joi.object().keys({ - name: 'eclipse marketplace', - value: isVPlusDottedVersionAtLeastOne, - }) - ) - -t.create('favorites count') - .get('/favorites/notepad4e.json') - .expectJSONTypes( - Joi.object().keys({ - name: 'favorites', - value: Joi.number() - .integer() - .positive(), - }) - ) - -t.create('last update date') - .get('/last-update/notepad4e.json') - .expectJSONTypes( - Joi.object().keys({ - name: 'updated', - value: isFormattedDate, - }) - ) diff --git a/services/f-droid/f-droid.service.js b/services/f-droid/f-droid.service.js index ca5e885be3..2d5be6e561 100644 --- a/services/f-droid/f-droid.service.js +++ b/services/f-droid/f-droid.service.js @@ -1,15 +1,15 @@ 'use strict' -const BaseHTTPService = require('../base-http') +const BaseService = require('../base') const { addv: versionText } = require('../../lib/text-formatters') const { version: versionColor } = require('../../lib/color-formatters') const { InvalidResponse } = require('../errors') -module.exports = class FDroid extends BaseHTTPService { +module.exports = class FDroid extends BaseService { async fetch({ appId }) { // currently, we only use the txt format. There are few apps using the yml format. const url = `https://gitlab.com/fdroid/fdroiddata/raw/master/metadata/${appId}.txt` - return this._requestHTTP({ + return this._request({ url, options: {}, errorMessages: { diff --git a/services/uptimerobot/uptimerobot-ratio.tester.js b/services/uptimerobot/uptimerobot-ratio.tester.js index db7bb8b778..b23858a5e3 100644 --- a/services/uptimerobot/uptimerobot-ratio.tester.js +++ b/services/uptimerobot/uptimerobot-ratio.tester.js @@ -65,7 +65,7 @@ t.create('Uptime Robot: Percentage (unexpected response, valid json)') .post('/v2/getMonitors') .reply(200, '[]') ) - .expectJSON({ name: 'uptime', value: 'invalid json response' }) + .expectJSON({ name: 'uptime', value: 'invalid response data' }) t.create('Uptime Robot: Percentage (unexpected response, invalid json)') .get('/m778918918-3e92c097147760ee39d02d36.json') diff --git a/services/uptimerobot/uptimerobot-status.tester.js b/services/uptimerobot/uptimerobot-status.tester.js index d2592fbbfe..f65205fb9b 100644 --- a/services/uptimerobot/uptimerobot-status.tester.js +++ b/services/uptimerobot/uptimerobot-status.tester.js @@ -62,7 +62,7 @@ t.create('Uptime Robot: Status (unexpected response, valid json)') .post('/v2/getMonitors') .reply(200, '[]') ) - .expectJSON({ name: 'status', value: 'invalid json response' }) + .expectJSON({ name: 'status', value: 'invalid response data' }) t.create('Uptime Robot: Status (unexpected response, invalid json)') .get('/m778918918-3e92c097147760ee39d02d36.json')