ComputerSurge/shields
3
0
Fork 0
Code Pull Requests 1 Activity

log 429s to sentry (attempt 2); affects [dynamic endpoint uptimerobot weblate opencollective discord github] (#9546)

Browse Source 
* log to sentry if upstream service responds with 429

* allow services to decide which error(s) to log, default to 429

* don't log 429s from endpoint or dynamic badges

* supress 429s from uptime robot badges

* supress 429s from weblate if not calling default server

* cache opencollective badges for longer

* cache discord badges for longer

* cache github workflow badges for longer
This commit is contained in:
chris48s
2023-12-04 13:37:58 +00:00
committed by GitHub
parent 791e635408
commit 45bb786147
25 changed files with 70 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/base-service/base-graphql.js
View File

@@ -50,6 +50,8 @@ class BaseGraphqlService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @param {Function} [attrs.transformJson=data => data] Function which takes the raw json and transforms it before
* further processing. In case of multiple query in a single graphql call and few of them
* throw error, partial data might be used ignoring the error.
@@ -69,6 +71,7 @@ class BaseGraphqlService extends BaseService {
options = {},
httpErrorMessages = {},
systemErrors = {},
logErrors = [429],
transformJson = data => data,
transformErrors = defaultTransformErrors,
}) {
@@ -83,6 +86,7 @@ class BaseGraphqlService extends BaseService {
options: mergedOptions,
httpErrors: httpErrorMessages,
systemErrors,
logErrors,
})
const json = transformJson(this._parseJson(buffer))
if (json.errors) {

4
core/base-service/base-json.js
View File

@@ -40,6 +40,8 @@ class BaseJsonService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @returns {object} Parsed response
* @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md
*/
@@ -49,6 +51,7 @@ class BaseJsonService extends BaseService {
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
}) {
const mergedOptions = {
...{ headers: { Accept: 'application/json' } },
@@ -59,6 +62,7 @@ class BaseJsonService extends BaseService {
options: mergedOptions,
httpErrors,
systemErrors,
logErrors,
})
const json = this._parseJson(buffer)
return this.constructor._validate(json, schema)

4
core/base-service/base-svg-scraping.js
View File

@@ -63,6 +63,8 @@ class BaseSvgScrapingService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @returns {object} Parsed response
* @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md
*/
@@ -73,6 +75,7 @@ class BaseSvgScrapingService extends BaseService {
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
}) {
const logTrace = (...args) => trace.logTrace('fetch', ...args)
const mergedOptions = {
@@ -84,6 +87,7 @@ class BaseSvgScrapingService extends BaseService {
options: mergedOptions,
httpErrors,
systemErrors,
logErrors,
})
logTrace(emojic.dart, 'Response SVG', buffer)
const data = {

4
core/base-service/base-toml.js
View File

@@ -33,6 +33,8 @@ class BaseTomlService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @returns {object} Parsed response
* @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md
*/
@@ -42,6 +44,7 @@ class BaseTomlService extends BaseService {
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
}) {
const logTrace = (...args) => trace.logTrace('fetch', ...args)
const mergedOptions = {
@@ -61,6 +64,7 @@ class BaseTomlService extends BaseService {
options: mergedOptions,
httpErrors,
systemErrors,
logErrors,
})
let parsed
try {

4
core/base-service/base-xml.js
View File

@@ -34,6 +34,8 @@ class BaseXmlService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @param {object} [attrs.parserOptions={}] Options to pass to fast-xml-parser. See
* [documentation](https://github.com/NaturalIntelligence/fast-xml-parser#xml-to-json)
* @returns {object} Parsed response
@@ -46,6 +48,7 @@ class BaseXmlService extends BaseService {
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
parserOptions = {},
}) {
const logTrace = (...args) => trace.logTrace('fetch', ...args)
@@ -58,6 +61,7 @@ class BaseXmlService extends BaseService {
options: mergedOptions,
httpErrors,
systemErrors,
logErrors,
})
const validateResult = XMLValidator.validate(buffer)
if (validateResult !== true) {

4
core/base-service/base-yaml.js
View File

@@ -33,6 +33,8 @@ class BaseYamlService extends BaseService {
* See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
* for allowed keys
* and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
* @param {number[]} [attrs.logErrors=[429]] An array of http error codes
* that will be logged (to sentry, if configured).
* @param {object} [attrs.encoding='utf8'] Character encoding
* @returns {object} Parsed response
* @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md
@@ -43,6 +45,7 @@ class BaseYamlService extends BaseService {
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
encoding = 'utf8',
}) {
const logTrace = (...args) => trace.logTrace('fetch', ...args)
@@ -60,6 +63,7 @@ class BaseYamlService extends BaseService {
options: mergedOptions,
httpErrors,
systemErrors,
logErrors,
})
let parsed
try {

10
core/base-service/base.js
View File

@@ -263,7 +263,13 @@ class BaseService {
this._metricHelper = metricHelper
}
async _request({ url, options = {}, httpErrors = {}, systemErrors = {} }) {
async _request({
url,
options = {},
httpErrors = {},
systemErrors = {},
logErrors = [429],
}) {
const logTrace = (...args) => trace.logTrace('fetch', ...args)
let logUrl = url
const logOptions = Object.assign({}, options)
@@ -290,7 +296,7 @@ class BaseService {
)
await this._meterResponse(res, buffer)
logTrace(emojic.dart, 'Response status code', res.statusCode)
return checkErrorResponse(httpErrors)({ buffer, res })
return checkErrorResponse(httpErrors, logErrors)({ buffer, res })
}
static enabledMetrics = []

8
core/base-service/check-error-response.js
View File

@@ -1,3 +1,4 @@
import log from '../server/log.js'
import { NotFound, InvalidResponse, Inaccessible } from './errors.js'
const defaultErrorMessages = {
@@ -5,7 +6,7 @@ const defaultErrorMessages = {
429: 'rate limited by upstream service',
}
export default function checkErrorResponse(httpErrors = {}) {
export default function checkErrorResponse(httpErrors = {}, logErrors = [429]) {
return async function ({ buffer, res }) {
let error
httpErrors = { ...defaultErrorMessages, ...httpErrors }
@@ -25,6 +26,11 @@ export default function checkErrorResponse(httpErrors = {}) {
error = new InvalidResponse(props)
}
}
if (logErrors.includes(res.statusCode)) {
log.error(new Error(`${res.statusCode} calling ${res.requestUrl.origin}`))
}
if (error) {
error.response = res
error.buffer = buffer

2
core/base-service/check-error-response.spec.js
View File

@@ -47,7 +47,7 @@ describe('async error handler', function () {
context('when status is 429', function () {
const buffer = Buffer.from('some stuff')
const res = { statusCode: 429 }
const res = { statusCode: 429, requestUrl: new URL('https://example.com/') }
it('throws InvalidResponse', async function () {
try {

2
services/discord/discord.service.js
View File

@@ -49,7 +49,7 @@ export default class Discord extends BaseJsonService {
},
}
static _cacheLength = 30
static _cacheLength = 60
static defaultBadgeData = { label: 'chat' }

1
services/dynamic/dynamic-json.service.js
View File

@@ -49,6 +49,7 @@ export default class DynamicJson extends jsonPath(BaseJsonService) {
schema,
url,
httpErrors,
logErrors: [],
})
}
}

1
services/dynamic/dynamic-toml.service.js
View File

@@ -49,6 +49,7 @@ export default class DynamicToml extends jsonPath(BaseTomlService) {
schema,
url,
httpErrors,
logErrors: [],
})
}
}

1
services/dynamic/dynamic-xml.service.js
View File

@@ -109,6 +109,7 @@ export default class DynamicXml extends BaseService {
url,
options: { headers: { Accept: 'application/xml, text/xml' } },
httpErrors,
logErrors: [],
})
const { values: value } = this.transform({

1
services/dynamic/dynamic-yaml.service.js
View File

@@ -49,6 +49,7 @@ export default class DynamicYaml extends jsonPath(BaseYamlService) {
schema,
url,
httpErrors,
logErrors: [],
})
}
}

1
services/endpoint-common.js
View File

@@ -95,6 +95,7 @@ async function fetchEndpointData(
schema: anySchema,
url,
httpErrors,
logErrors: [],
options: { decompress: true },
})
return validateEndpointData(json, {

2
services/github/github-actions-workflow-status.service.js
View File

@@ -73,6 +73,8 @@ export default class GithubActionsWorkflowStatus extends BaseSvgScrapingService
},
]
static _cacheLength = 60
static defaultBadgeData = {
label: 'build',
}

2
services/opencollective/opencollective-all.service.js
View File

@@ -16,7 +16,7 @@ export default class OpencollectiveAll extends OpencollectiveBase {
},
}
static _cacheLength = 900
static _cacheLength = 1800
static defaultBadgeData = {
label: 'backers and sponsors',

2
services/opencollective/opencollective-backers.service.js
View File

@@ -16,7 +16,7 @@ export default class OpencollectiveBackers extends OpencollectiveBase {
},
}
static _cacheLength = 900
static _cacheLength = 1800
static defaultBadgeData = {
label: 'backers',

2
services/opencollective/opencollective-sponsors.service.js
View File

@@ -16,7 +16,7 @@ export default class OpencollectiveSponsors extends OpencollectiveBase {
},
}
static _cacheLength = 900
static _cacheLength = 1800
static defaultBadgeData = {
label: 'sponsors',

1
services/uptimerobot/uptimerobot-base.js
View File

@@ -74,6 +74,7 @@ export default class UptimeRobotBase extends BaseJsonService {
...opts,
},
},
logErrors: [],
})
if (stat === 'fail') {

2
services/weblate/weblate-base.js
View File

@@ -2,6 +2,8 @@ import Joi from 'joi'
import { BaseJsonService } from '../index.js'
import { optionalUrl } from '../validators.js'
export const defaultServer = 'https://hosted.weblate.org'
export default class WeblateBase extends BaseJsonService {
static queryParamSchema = Joi.object({
server: optionalUrl,

7
services/weblate/weblate-component-license.service.js
View File

@@ -1,5 +1,5 @@
import Joi from 'joi'
import WeblateBase from './weblate-base.js'
import WeblateBase, { defaultServer } from './weblate-base.js'
const schema = Joi.object({
license: Joi.string().required(),
@@ -21,7 +21,7 @@ export default class WeblateComponentLicense extends WeblateBase {
{
title: 'Weblate component license',
namedParams: { project: 'godot-engine', component: 'godot' },
queryParams: { server: 'https://hosted.weblate.org' },
queryParams: { server: defaultServer },
staticPreview: this.render({ license: 'MIT' }),
keywords: ['i18n', 'translation', 'internationalization'],
},
@@ -33,7 +33,7 @@ export default class WeblateComponentLicense extends WeblateBase {
return { message: `${license}` }
}
async fetch({ project, component, server = 'https://hosted.weblate.org' }) {
async fetch({ project, component, server = defaultServer }) {
return super.fetch({
schema,
url: `${server}/api/components/${project}/${component}/`,
@@ -41,6 +41,7 @@ export default class WeblateComponentLicense extends WeblateBase {
403: 'access denied by remote server',
404: 'component not found',
},
logErrors: server === defaultServer ? [429] : [],
})
}

7
services/weblate/weblate-entities.service.js
View File

@@ -1,7 +1,7 @@
import Joi from 'joi'
import { nonNegativeInteger } from '../validators.js'
import { metric } from '../text-formatters.js'
import WeblateBase from './weblate-base.js'
import WeblateBase, { defaultServer } from './weblate-base.js'
const schema = Joi.object({
count: nonNegativeInteger,
@@ -20,7 +20,7 @@ export default class WeblateEntities extends WeblateBase {
{
title: 'Weblate entities',
namedParams: { type: 'projects' },
queryParams: { server: 'https://hosted.weblate.org' },
queryParams: { server: defaultServer },
staticPreview: this.render({ type: 'projects', count: 533 }),
keywords: ['i18n', 'internationalization'],
},
@@ -32,13 +32,14 @@ export default class WeblateEntities extends WeblateBase {
return { label: type, message: metric(count) }
}
async fetch({ type, server = 'https://hosted.weblate.org' }) {
async fetch({ type, server = defaultServer }) {
return super.fetch({
schema,
url: `${server}/api/${type}/`,
httpErrors: {
403: 'access denied by remote server',
},
logErrors: server === defaultServer ? [429] : [],
})
}

7
services/weblate/weblate-project-translated-percentage.service.js
View File

@@ -1,6 +1,6 @@
import Joi from 'joi'
import { colorScale } from '../color-formatters.js'
import WeblateBase from './weblate-base.js'
import WeblateBase, { defaultServer } from './weblate-base.js'
const schema = Joi.object({
translated_percent: Joi.number().required(),
@@ -23,7 +23,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase {
{
title: 'Weblate project translated',
namedParams: { project: 'godot-engine' },
queryParams: { server: 'https://hosted.weblate.org' },
queryParams: { server: defaultServer },
staticPreview: this.render({ translatedPercent: 20.5 }),
keywords: ['i18n', 'translation', 'internationalization'],
},
@@ -45,7 +45,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase {
return { message: `${translatedPercent.toFixed(0)}%`, color }
}
async fetch({ project, server = 'https://hosted.weblate.org' }) {
async fetch({ project, server = defaultServer }) {
return super.fetch({
schema,
url: `${server}/api/projects/${project}/statistics/`,
@@ -53,6 +53,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase {
403: 'access denied by remote server',
404: 'project not found',
},
logErrors: server === defaultServer ? [429] : [],
})
}

7
services/weblate/weblate-user-statistic.service.js
View File

@@ -1,7 +1,7 @@
import Joi from 'joi'
import { nonNegativeInteger } from '../validators.js'
import { metric } from '../text-formatters.js'
import WeblateBase from './weblate-base.js'
import WeblateBase, { defaultServer } from './weblate-base.js'
const schema = Joi.object({
translated: nonNegativeInteger,
@@ -33,7 +33,7 @@ export default class WeblateUserStatistic extends WeblateBase {
{
title: 'Weblate user statistic',
namedParams: { statistic: 'translations', user: 'nijel' },
queryParams: { server: 'https://hosted.weblate.org' },
queryParams: { server: defaultServer },
staticPreview: this.render({ statistic: 'translations', count: 30585 }),
keywords: ['i18n', 'internationalization'],
},
@@ -45,7 +45,7 @@ export default class WeblateUserStatistic extends WeblateBase {
return { label: statistic, message: metric(count) }
}
async fetch({ user, server = 'https://hosted.weblate.org' }) {
async fetch({ user, server = defaultServer }) {
return super.fetch({
schema,
url: `${server}/api/users/${user}/statistics/`,
@@ -53,6 +53,7 @@ export default class WeblateUserStatistic extends WeblateBase {
403: 'access denied by remote server',
404: 'user not found',
},
logErrors: server === defaultServer ? [429] : [],
})
}