From b841db68ba4c032a4896a3477905951d5b6d8613 Mon Sep 17 00:00:00 2001 From: Vinta Chen Date: Sun, 16 Aug 2026 19:08:55 +0800 Subject: [PATCH] refactor: make pypi_name_overrides.json entries self-documenting Every entry is now {"package": str|null, "reason": str|null} instead of a bare string/null. Reasons are required for null packages, explaining why the name must never be queried (squatted name, stdlib module, monorepo umbrella, GitHub-only project, and so on). Reasons are optional for remaps and kept only on the six non-obvious ones: pytorch (squatter), jinja (jinja is Jinja1), strawberry (unrelated bookmarking service), django-rules (abandoned fork), django-rest-framework (dead alias), and devpi (deprecated metapackage); plain publishes-as-X remaps get a null reason. load_overrides() in the clickpy fetcher now extracts the package field from each entry; resolve() and the pepy/bigquery cross-check scripts are unchanged since they consume load_overrides()'s output. Co-Authored-By: Claude --- website/data/pypi_name_overrides.json | 80 ++++++++++----------- website/fetch_pypi_downloads_via_clickpy.py | 15 ++-- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/website/data/pypi_name_overrides.json b/website/data/pypi_name_overrides.json index 5ec7958a..8c8a5f95 100644 --- a/website/data/pypi_name_overrides.json +++ b/website/data/pypi_name_overrides.json @@ -1,42 +1,42 @@ { - "autobahn-python": "autobahn", - "azure-sdk-for-python": null, - "cloud-init": null, - "concurrent-futures": null, - "cpython": null, - "devpi": "devpi-server", - "difflib": null, - "django-ai-plugins": null, - "django-db-models": null, - "django-rest-framework": "djangorestframework", - "django-rules": "rules", - "fasthtml": "python-fasthtml", - "geodjango": null, - "google-cloud-python": null, - "graphify": null, - "httpx-url": null, - "jinja": "jinja2", - "mem0": "mem0ai", - "micropython": null, - "mimetypes": null, - "odoo": null, - "pangu-py": "pangu", - "playwright-python": "playwright", - "pyenv": null, - "pypy": null, - "python-patterns": null, - "pytorch": "torch", - "renpy": null, - "sentry-skills": null, - "social-engineer-toolkit": null, - "sqlite3": null, - "strawberry": "strawberry-graphql", - "strawberry-django": "strawberry-graphql-django", - "thealgorithms": null, - "tkinter": null, - "tomllib": null, - "trailofbits-skills": null, - "warehouse": null, - "winpython": null, - "zoneinfo": null + "autobahn-python": { "package": "autobahn", "reason": null }, + "azure-sdk-for-python": { "package": null, "reason": "monorepo umbrella; ships as many azure-* packages, no single package represents it" }, + "cloud-init": { "package": null, "reason": "installed via distro images, not pip" }, + "concurrent-futures": { "package": null, "reason": "stdlib module" }, + "cpython": { "package": null, "reason": "not pip-installable; the cpython PyPI name was squatted" }, + "devpi": { "package": "devpi-server", "reason": "devpi is a deprecated metapackage; the server package carries real usage" }, + "difflib": { "package": null, "reason": "stdlib module" }, + "django-ai-plugins": { "package": null, "reason": "GitHub-only so far; young project, remap if it publishes" }, + "django-db-models": { "package": null, "reason": "part of Django, not a package" }, + "django-rest-framework": { "package": "djangorestframework", "reason": "django-rest-framework is a dead alias package" }, + "django-rules": { "package": "rules", "reason": "PyPI django-rules is an abandoned unrelated fork; dfunckt's project publishes as rules" }, + "fasthtml": { "package": "python-fasthtml", "reason": null }, + "geodjango": { "package": null, "reason": "part of Django (django.contrib.gis), not a package" }, + "google-cloud-python": { "package": null, "reason": "monorepo umbrella; ships as many google-cloud-* packages" }, + "graphify": { "package": null, "reason": "GitHub-only so far; young project, remap if it publishes" }, + "httpx-url": { "package": null, "reason": "class in the httpx package, not a package" }, + "jinja": { "package": "jinja2", "reason": "PyPI jinja is Jinja1; current releases publish as jinja2" }, + "mem0": { "package": "mem0ai", "reason": null }, + "micropython": { "package": null, "reason": "interpreter distributed as firmware builds, not pip" }, + "mimetypes": { "package": null, "reason": "stdlib module" }, + "odoo": { "package": null, "reason": "self-hosted suite installed from odoo.com, not pip" }, + "pangu-py": { "package": "pangu", "reason": null }, + "playwright-python": { "package": "playwright", "reason": null }, + "pyenv": { "package": null, "reason": "installed via git/homebrew; PyPI pyenv is a defunct shim" }, + "pypy": { "package": null, "reason": "interpreter distributed via pypy.org, not pip" }, + "python-patterns": { "package": null, "reason": "reference repo, not a package; PyPI name held by unrelated 2012 project" }, + "pytorch": { "package": "torch", "reason": "PyPI pytorch is a squatter; real package is torch" }, + "renpy": { "package": null, "reason": "distributed via renpy.org; PyPI name was squatted, now 404" }, + "sentry-skills": { "package": null, "reason": "agent skills repo, not a package" }, + "social-engineer-toolkit": { "package": null, "reason": "GitHub-only, installed from source" }, + "sqlite3": { "package": null, "reason": "stdlib module" }, + "strawberry": { "package": "strawberry-graphql", "reason": "PyPI strawberry is an unrelated bookmarking service" }, + "strawberry-django": { "package": "strawberry-graphql-django", "reason": null }, + "thealgorithms": { "package": null, "reason": "educational monorepo, not a package" }, + "tkinter": { "package": null, "reason": "stdlib module" }, + "tomllib": { "package": null, "reason": "stdlib module" }, + "trailofbits-skills": { "package": null, "reason": "agent skills repo, not a package" }, + "warehouse": { "package": null, "reason": "the PyPI server itself; deployed, not pip-installed" }, + "winpython": { "package": null, "reason": "distribution with its own installer; PyPI name is a 2012 relic" }, + "zoneinfo": { "package": null, "reason": "stdlib module" } } diff --git a/website/fetch_pypi_downloads_via_clickpy.py b/website/fetch_pypi_downloads_via_clickpy.py index 6934dc01..c7c979f2 100644 --- a/website/fetch_pypi_downloads_via_clickpy.py +++ b/website/fetch_pypi_downloads_via_clickpy.py @@ -16,9 +16,10 @@ naturally; names with no PyPI rows are written as NOT_FOUND. Counts are looked up by README display name; when the display name differs from the canonical PyPI package (a squatter or a dead predecessor would be measured otherwise), add the mapping to the curated -data/pypi_name_overrides.json — normalized README name to the real -package, or null for projects that are not pip-installable so their -row is never queried. The file starts with a header row (name, package, +data/pypi_name_overrides.json — normalized README name to +{"package": ..., "reason": ...}, where package is the real PyPI name +or null for projects that are not pip-installable so their row is +never queried, and reason records why. The file starts with a header row (name, package, downloads, fetched_at) — package is the PyPI package the row actually measured ("-" for null overrides) — and every row carries the sweep date: a cache fetched within the last 7 days is current enough for @@ -53,7 +54,13 @@ def normalize(name: str) -> str: def load_overrides() -> dict[str, str | None]: - return json.loads(OVERRIDES_FILE.read_text()) + """Load curated overrides: normalized README name -> PyPI package to measure (None = never query). + + Each file entry is {"package": str | null, "reason": str | null}; the reason is + documentation only — required for null packages, optional for remaps. + """ + raw = json.loads(OVERRIDES_FILE.read_text()) + return {name: entry["package"] for name, entry in raw.items()} def resolve(name: str) -> str | None: