From e8671be7f22d5477fef006b354d7a2b963eea6cf Mon Sep 17 00:00:00 2001 From: Leo Q Date: Mon, 22 Apr 2024 00:33:42 +0800 Subject: [PATCH] support setting pypiBaseUrl by environment variables and queryParameters; affects [pypi] (#10044) * support setting pypiBaseUrl by environment variables * Add support for pypiBaseUrl configuration * Update Pypi services to include pypiBaseUrl parameter * change package name example to a more well-known package * Update custom-environment-variables.yml * Update Pypi services to include pypiBaseUrl parameter * fix openapi mismatch * Update doc/server-secrets.md --------- Co-authored-by: chris48s --- config/custom-environment-variables.yml | 2 ++ config/default.yml | 2 ++ core/server/server.js | 3 ++ doc/server-secrets.md | 7 +++++ services/pypi/pypi-base.js | 28 +++++++++++++++++-- services/pypi/pypi-downloads.service.js | 13 +++++---- services/pypi/pypi-format.service.js | 12 +++----- .../pypi/pypi-framework-versions.service.js | 8 +++--- services/pypi/pypi-implementation.service.js | 12 +++----- services/pypi/pypi-license.service.js | 12 +++----- services/pypi/pypi-python-versions.service.js | 12 +++----- services/pypi/pypi-status.service.js | 12 +++----- services/pypi/pypi-version.service.js | 12 +++----- services/pypi/pypi-wheel.service.js | 12 +++----- 14 files changed, 78 insertions(+), 69 deletions(-) diff --git a/config/custom-environment-variables.yml b/config/custom-environment-variables.yml index 56919b467f..4e47cb9d24 100644 --- a/config/custom-environment-variables.yml +++ b/config/custom-environment-variables.yml @@ -52,6 +52,8 @@ public: authorizedOrigins: 'NPM_ORIGINS' obs: authorizedOrigins: 'OBS_ORIGINS' + pypi: + baseUri: 'PYPI_URL' sonar: authorizedOrigins: 'SONAR_ORIGINS' teamcity: diff --git a/config/default.yml b/config/default.yml index 5a6fb10393..cea2edad7b 100644 --- a/config/default.yml +++ b/config/default.yml @@ -22,6 +22,8 @@ public: restApiVersion: '2022-11-28' obs: authorizedOrigins: 'https://api.opensuse.org' + pypi: + baseUri: 'https://pypi.org' weblate: authorizedOrigins: 'https://hosted.weblate.org' trace: false diff --git a/core/server/server.js b/core/server/server.js index d6c6256a11..65c8a52f75 100644 --- a/core/server/server.js +++ b/core/server/server.js @@ -139,6 +139,9 @@ const publicConfigSchema = Joi.object({ nexus: defaultService, npm: defaultService, obs: defaultService, + pypi: { + baseUri: requiredUrl, + }, sonar: defaultService, teamcity: defaultService, weblate: defaultService, diff --git a/doc/server-secrets.md b/doc/server-secrets.md index f7406b6d8b..1518ec0a60 100644 --- a/doc/server-secrets.md +++ b/doc/server-secrets.md @@ -283,6 +283,13 @@ The Pepy API requires authentication. To obtain a key, Create an account, sign in and obtain generate a key on your [account page](https://www.pepy.tech/user). +### PyPI + +- `PYPI_URL` (yml: `public.pypi.baseUri`) + +`PYPI_URL` can be used to optionally send all the PyPI requests to a Self-hosted Pypi registry, +users can also override this by query parameter `pypiBaseUrl`. + ### SymfonyInsight (formerly Sensiolabs) - `SL_INSIGHT_USER_UUID` (yml: `private.sl_insight_userUuid`) diff --git a/services/pypi/pypi-base.js b/services/pypi/pypi-base.js index df4a38eb0f..344d249d4e 100644 --- a/services/pypi/pypi-base.js +++ b/services/pypi/pypi-base.js @@ -1,5 +1,7 @@ import Joi from 'joi' -import { BaseJsonService } from '../index.js' +import config from 'config' +import { optionalUrl } from '../validators.js' +import { BaseJsonService, queryParam, pathParam } from '../index.js' const schema = Joi.object({ info: Joi.object({ @@ -18,18 +20,38 @@ const schema = Joi.object({ .required(), }).required() +export const queryParamSchema = Joi.object({ + pypiBaseUrl: optionalUrl, +}).required() + +export const pypiPackageParam = pathParam({ + name: 'packageName', + example: 'Django', +}) + +export const pypiBaseUrlParam = queryParam({ + name: 'pypiBaseUrl', + example: 'https://pypi.org', +}) + +export const pypiGeneralParams = [pypiPackageParam, pypiBaseUrlParam] + export default class PypiBase extends BaseJsonService { static buildRoute(base) { return { base, pattern: ':egg+', + queryParamSchema, } } - async fetch({ egg }) { + async fetch({ egg, pypiBaseUrl = null }) { + const defaultpypiBaseUrl = + config.util.toObject().public.services.pypi.baseUri + pypiBaseUrl = pypiBaseUrl || defaultpypiBaseUrl return this._requestJson({ schema, - url: `https://pypi.org/pypi/${egg}/json`, + url: `${pypiBaseUrl}/pypi/${egg}/json`, httpErrors: { 404: 'package or version not found' }, }) } diff --git a/services/pypi/pypi-downloads.service.js b/services/pypi/pypi-downloads.service.js index 45106f0e65..14c988b0e3 100644 --- a/services/pypi/pypi-downloads.service.js +++ b/services/pypi/pypi-downloads.service.js @@ -1,7 +1,8 @@ import Joi from 'joi' import { nonNegativeInteger } from '../validators.js' -import { BaseJsonService, pathParams } from '../index.js' +import { BaseJsonService, pathParam } from '../index.js' import { renderDownloadsBadge } from '../downloads.js' +import { pypiPackageParam } from './pypi-base.js' const schema = Joi.object({ data: Joi.object({ @@ -42,15 +43,15 @@ export default class PypiDownloads extends BaseJsonService { summary: 'PyPI - Downloads', description: 'Python package downloads from [pypistats](https://pypistats.org/)', - parameters: pathParams( - { + parameters: [ + pathParam({ name: 'period', example: 'dd', schema: { type: 'string', enum: this.getEnum('period') }, description: 'Daily, Weekly, or Monthly downloads', - }, - { name: 'packageName', example: 'Django' }, - ), + }), + pypiPackageParam, + ], }, }, } diff --git a/services/pypi/pypi-format.service.js b/services/pypi/pypi-format.service.js index e1d02bc9fe..4c99562019 100644 --- a/services/pypi/pypi-format.service.js +++ b/services/pypi/pypi-format.service.js @@ -1,5 +1,4 @@ -import { pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { getPackageFormats } from './pypi-helpers.js' export default class PypiFormat extends PypiBase { @@ -11,10 +10,7 @@ export default class PypiFormat extends PypiBase { '/pypi/format/{packageName}': { get: { summary: 'PyPI - Format', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -40,8 +36,8 @@ export default class PypiFormat extends PypiBase { } } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) const { hasWheel, hasEgg } = getPackageFormats(packageData) return this.constructor.render({ hasWheel, hasEgg }) } diff --git a/services/pypi/pypi-framework-versions.service.js b/services/pypi/pypi-framework-versions.service.js index 6c42f8969b..5777f8b35a 100644 --- a/services/pypi/pypi-framework-versions.service.js +++ b/services/pypi/pypi-framework-versions.service.js @@ -1,5 +1,5 @@ import { InvalidResponse, pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiBaseUrlParam } from './pypi-base.js' import { sortPypiVersions, parseClassifiers } from './pypi-helpers.js' const frameworkNameMap = { @@ -63,7 +63,7 @@ export default class PypiFrameworkVersion extends PypiBase { schema: { type: 'string', enum: Object.keys(frameworkNameMap) }, }, { name: 'packageName', example: 'plone.volto' }, - ), + ).concat(pypiBaseUrlParam), }, }, } @@ -80,7 +80,7 @@ export default class PypiFrameworkVersion extends PypiBase { } } - async handle({ frameworkName, packageName }) { + async handle({ frameworkName, packageName }, { pypiBaseUrl }) { const classifier = frameworkNameMap[frameworkName] ? frameworkNameMap[frameworkName].classifier : frameworkName @@ -88,7 +88,7 @@ export default class PypiFrameworkVersion extends PypiBase { ? frameworkNameMap[frameworkName].name : frameworkName const regex = new RegExp(`^Framework :: ${classifier} :: ([\\d.]+)$`) - const packageData = await this.fetch({ egg: packageName }) + const packageData = await this.fetch({ egg: packageName, pypiBaseUrl }) const versions = parseClassifiers(packageData, regex) if (versions.length === 0) { diff --git a/services/pypi/pypi-implementation.service.js b/services/pypi/pypi-implementation.service.js index 8da1079c15..8a3c6a7a0c 100644 --- a/services/pypi/pypi-implementation.service.js +++ b/services/pypi/pypi-implementation.service.js @@ -1,5 +1,4 @@ -import { pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { parseClassifiers } from './pypi-helpers.js' export default class PypiImplementation extends PypiBase { @@ -11,10 +10,7 @@ export default class PypiImplementation extends PypiBase { '/pypi/implementation/{packageName}': { get: { summary: 'PyPI - Implementation', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -28,8 +24,8 @@ export default class PypiImplementation extends PypiBase { } } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) let implementations = parseClassifiers( packageData, diff --git a/services/pypi/pypi-license.service.js b/services/pypi/pypi-license.service.js index 4bb89dc498..11fe595feb 100644 --- a/services/pypi/pypi-license.service.js +++ b/services/pypi/pypi-license.service.js @@ -1,6 +1,5 @@ -import { pathParams } from '../index.js' import { renderLicenseBadge } from '../licenses.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { getLicenses } from './pypi-helpers.js' export default class PypiLicense extends PypiBase { @@ -12,10 +11,7 @@ export default class PypiLicense extends PypiBase { '/pypi/l/{packageName}': { get: { summary: 'PyPI - License', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -24,8 +20,8 @@ export default class PypiLicense extends PypiBase { return renderLicenseBadge({ licenses }) } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) const licenses = getLicenses(packageData) return this.constructor.render({ licenses }) } diff --git a/services/pypi/pypi-python-versions.service.js b/services/pypi/pypi-python-versions.service.js index 49db5cb050..3527fee063 100644 --- a/services/pypi/pypi-python-versions.service.js +++ b/services/pypi/pypi-python-versions.service.js @@ -1,6 +1,5 @@ import semver from 'semver' -import { pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { parseClassifiers } from './pypi-helpers.js' export default class PypiPythonVersions extends PypiBase { @@ -12,10 +11,7 @@ export default class PypiPythonVersions extends PypiBase { '/pypi/pyversions/{packageName}': { get: { summary: 'PyPI - Python Version', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -48,8 +44,8 @@ export default class PypiPythonVersions extends PypiBase { } } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) const versions = parseClassifiers( packageData, diff --git a/services/pypi/pypi-status.service.js b/services/pypi/pypi-status.service.js index 12d85e39cc..769c376c57 100644 --- a/services/pypi/pypi-status.service.js +++ b/services/pypi/pypi-status.service.js @@ -1,5 +1,4 @@ -import { pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { parseClassifiers } from './pypi-helpers.js' export default class PypiStatus extends PypiBase { @@ -11,10 +10,7 @@ export default class PypiStatus extends PypiBase { '/pypi/status/{packageName}': { get: { summary: 'PyPI - Status', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -40,8 +36,8 @@ export default class PypiStatus extends PypiBase { } } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) // Possible statuses: // - Development Status :: 1 - Planning diff --git a/services/pypi/pypi-version.service.js b/services/pypi/pypi-version.service.js index 04e018be97..98fbb034fd 100644 --- a/services/pypi/pypi-version.service.js +++ b/services/pypi/pypi-version.service.js @@ -1,7 +1,6 @@ -import { pathParams } from '../index.js' import { pep440VersionColor } from '../color-formatters.js' import { renderVersionBadge } from '../version.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' export default class PypiVersion extends PypiBase { static category = 'version' @@ -12,10 +11,7 @@ export default class PypiVersion extends PypiBase { '/pypi/v/{packageName}': { get: { summary: 'PyPI - Version', - parameters: pathParams({ - name: 'packageName', - example: 'nine', - }), + parameters: pypiGeneralParams, }, }, } @@ -26,10 +22,10 @@ export default class PypiVersion extends PypiBase { return renderVersionBadge({ version, versionFormatter: pep440VersionColor }) } - async handle({ egg }) { + async handle({ egg }, { pypiBaseUrl }) { const { info: { version }, - } = await this.fetch({ egg }) + } = await this.fetch({ egg, pypiBaseUrl }) return this.constructor.render({ version }) } } diff --git a/services/pypi/pypi-wheel.service.js b/services/pypi/pypi-wheel.service.js index ce151cb055..915ead1b6d 100644 --- a/services/pypi/pypi-wheel.service.js +++ b/services/pypi/pypi-wheel.service.js @@ -1,5 +1,4 @@ -import { pathParams } from '../index.js' -import PypiBase from './pypi-base.js' +import PypiBase, { pypiGeneralParams } from './pypi-base.js' import { getPackageFormats } from './pypi-helpers.js' export default class PypiWheel extends PypiBase { @@ -11,10 +10,7 @@ export default class PypiWheel extends PypiBase { '/pypi/wheel/{packageName}': { get: { summary: 'PyPI - Wheel', - parameters: pathParams({ - name: 'packageName', - example: 'Django', - }), + parameters: pypiGeneralParams, }, }, } @@ -35,8 +31,8 @@ export default class PypiWheel extends PypiBase { } } - async handle({ egg }) { - const packageData = await this.fetch({ egg }) + async handle({ egg }, { pypiBaseUrl }) { + const packageData = await this.fetch({ egg, pypiBaseUrl }) const { hasWheel } = getPackageFormats(packageData) return this.constructor.render({ hasWheel }) }