From 1abfb1af42139e2d1fc003130a36f54bd86d5c0f Mon Sep 17 00:00:00 2001 From: Vinta Chen Date: Sun, 16 Aug 2026 01:57:16 +0800 Subject: [PATCH] refactor: split download fetchers into writer and cross-check scripts Authored by the parallel fetch-scripts session (its intended 0cffb5f never landed; the content rode into an audit commit by accident and is extracted here): fetch_pypi_downloads_via_clickpy.py becomes the flagless full-README sweep and sole writer of data/pypi_downloads.tsv; new fetch_pypi_downloads_via_bigquery.py is a print-only cross-check taking explicit names behind a 400 GB billing cap; audit-the-list SKILL.md step 2 updated to match. Co-Authored-By: Claude --- .claude/skills/audit-the-list/SKILL.md | 2 +- website/fetch_pypi_downloads_via_bigquery.py | 72 ++++++++++++++++ website/fetch_pypi_downloads_via_clickpy.py | 91 ++++---------------- 3 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 website/fetch_pypi_downloads_via_bigquery.py diff --git a/.claude/skills/audit-the-list/SKILL.md b/.claude/skills/audit-the-list/SKILL.md index a29da0a6..be48f239 100644 --- a/.claude/skills/audit-the-list/SKILL.md +++ b/.claude/skills/audit-the-list/SKILL.md @@ -16,7 +16,7 @@ Resolve the scope from the arguments. Named sections mean exactly those, whether Fetch live evidence for every entry in scope before judging anything (CLAUDE.md verification rule): -- **Downloads/month**: `cd website && UV_PYTHON=3.13 uv run python fetch_pypi_downloads_via_clickpy.py --names-file ` — default source is ClickPy (free, keyless, one batched query covers the full README). `--bigquery` switches to the canonical cross-check on the maintainer's own GCP account; there, run `--dry-run` first and read the docstring's cost constraints before batching. Spot-checks: `uv run python fetch_pypi_downloads_via_pepy.py ...` (needs `PEPY_TECH_API_KEY` in repo-root `.env`, throttled to 5 requests/minute) or `https://pypistats.org/api/packages/{name}/recent` paced 8s or slower. pypistats excludes mirror/CI traffic; ClickPy, BigQuery, and pepy include it — never mix sources within one comparison. +- **Downloads/month**: `cd website && UV_PYTHON=3.13 uv run python fetch_pypi_downloads_via_clickpy.py` — free keyless ClickPy sweep of the full README, sole writer of `data/pypi_downloads.tsv` (rewritten from scratch each run). Cross-checks print to stdout, take explicit names, and never touch the cache: `fetch_pypi_downloads_via_bigquery.py ...` (canonical source, maintainer's own GCP account, `--dry-run` first — the docstring carries the cost constraints; full-README sweeps exceed the free tier, keep name lists small), `fetch_pypi_downloads_via_pepy.py ...` (needs `PEPY_TECH_API_KEY` in repo-root `.env`, throttled to 5 requests/minute), or `https://pypistats.org/api/packages/{name}/recent` paced 8s or slower. pypistats excludes mirror/CI traffic; ClickPy, BigQuery, and pepy include it — never mix sources within one comparison. - **Repo state**: archived flag, last push, created date, stars, description — `gh api repos/{owner}/{repo}`, GitLab API for GitLab-hosted projects. - **PyPI metadata** (`https://pypi.org/pypi/{name}/json`) wherever a name might not be the canonical package — ownership collisions and wrong display names surface here. diff --git a/website/fetch_pypi_downloads_via_bigquery.py b/website/fetch_pypi_downloads_via_bigquery.py new file mode 100644 index 00000000..21f69479 --- /dev/null +++ b/website/fetch_pypi_downloads_via_bigquery.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Cross-check last-30-day PyPI download counts for given names via BigQuery. + +Queries the canonical source ClickPy mirrors — +`bigquery-public-data.pypi.file_downloads` via the `bq` CLI — and prints +namecount TSV to stdout. Maintainer-local (needs a personal GCP +account) and print-only: data/pypi_downloads.tsv is written solely by +fetch_pypi_downloads_via_clickpy.py. + +Targeted cross-checks only — full-README sweeps belong to ClickPy. The +dry-run scan estimate scales with the IN list (measured 2026-08-16: +~38 GB for 2 names, ~275 GB for 53, ~1.2 TB for the full README — past +both MAX_BYTES_BILLED and the 1 TiB/month free tier), and BigQuery +enforces MAX_BYTES_BILLED against that pre-run estimate, so oversized +name lists fail before billing. Actual billed bytes come in far lower +via cluster pruning (33.7 GB for a single name). The table is +partitioned on timestamp and clustered on the top-level `project` +column — filter on `project`, never `file.project`: both hold identical +values (verified 2026-08-16, 408M rows on one day, zero mismatches), but +only a `project` filter gets cluster pruning, and the `file` record +costs ~4x more to scan. Always --dry-run first to check the estimate. + +Usage: python fetch_pypi_downloads_via_bigquery.py [--dry-run] NAME [NAME ...] +""" + +import subprocess +import sys +from json import loads + +from fetch_pypi_downloads_via_clickpy import normalize + +MAX_BYTES_BILLED = 400_000_000_000 + + +def fetch_bigquery(names: list[str], dry_run: bool) -> dict[str, int]: + in_list = ", ".join(f"'{name}'" for name in names) + query = ( + "SELECT project, COUNT(*) AS downloads " + "FROM `bigquery-public-data.pypi.file_downloads` " + "WHERE DATE(timestamp) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) " + "AND CURRENT_DATE() " + f"AND project IN ({in_list}) " + "GROUP BY project" + ) + cmd = ["bq", "query", "--use_legacy_sql=false", "--format=json", f"--maximum_bytes_billed={MAX_BYTES_BILLED}"] + if dry_run: + cmd.append("--dry_run") + cmd.append(query) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(result.stderr, file=sys.stderr) + sys.exit(1) + if dry_run: + print(result.stdout.strip() or result.stderr.strip()) + sys.exit(0) + rows = loads(result.stdout) + return {row["project"]: int(row["downloads"]) for row in rows} + + +def main() -> None: + dry_run = "--dry-run" in sys.argv + names = sorted({normalize(arg) for arg in sys.argv[1:] if arg != "--dry-run"}) + if not names: + print("Usage: python fetch_pypi_downloads_via_bigquery.py [--dry-run] NAME [NAME ...]", file=sys.stderr) + sys.exit(1) + counts = fetch_bigquery(names, dry_run) + for name in names: + print(f"{name}\t{counts.get(name, 'NOT_FOUND')}") + + +if __name__ == "__main__": + main() diff --git a/website/fetch_pypi_downloads_via_clickpy.py b/website/fetch_pypi_downloads_via_clickpy.py index 1cae54a7..e88815ac 100644 --- a/website/fetch_pypi_downloads_via_clickpy.py +++ b/website/fetch_pypi_downloads_via_clickpy.py @@ -1,42 +1,26 @@ #!/usr/bin/env python3 -"""Fetch last-30-day PyPI download counts for README entries. +"""Fetch last-30-day PyPI download counts for all README entries via ClickPy. -Default source is ClickPy, ClickHouse's public mirror of the PyPI download -dataset (sql-clickhouse.clickhouse.com, user `demo`, no key, no personal +ClickPy is ClickHouse's public mirror of the PyPI download dataset +(sql-clickhouse.clickhouse.com, user `demo`, no key, no personal account): one batched query covers the full README in under a second. Playground limits — 60 queries/hour/IP, 1000 result rows — sit far above a full-README sweep. The per-day table is a SummingMergeTree, so the query must sum(count) GROUP BY, never count raw rows. Counts include -mirror/CI traffic, same as BigQuery's; never mix these figures with -pypistats.org, which excludes mirrors by default. +mirror/CI traffic; never mix these figures with pypistats.org, which +excludes mirrors by default. ---bigquery switches to the canonical source ClickPy mirrors: -`bigquery-public-data.pypi.file_downloads` via the `bq` CLI (needs a -personal GCP account, so it is a maintainer-local cross-check, not the -default). The table is partitioned on timestamp and clustered on the -top-level `project` column — filter on `project`, never `file.project`: -both hold identical values (verified 2026-08-16, 408M rows on one day, -zero mismatches), but only a `project` filter gets cluster pruning, and -the `file` record costs ~4x more to scan. A 30-day query dry-runs at up -to ~275 GB (the pre-pruning upper bound for a full-README IN list; small -lists estimate lower); actual billed bytes shrink further with cluster -pruning (33.7 GB measured for a single name), so even a full-README sweep -fits the 1 TiB/month free tier. MAX_BYTES_BILLED is a safety net against -accidentally unpartitioned queries; BigQuery enforces it against the -pre-run upper bound, so it must stay above the dry-run estimate or every -query fails. --dry-run (BigQuery only) prints the scan estimate without -running. +This script is the sole writer of data/pypi_downloads.tsv and rewrites it +from scratch each run, so entries removed from README.md drop out +naturally; names with no PyPI rows are written as NOT_FOUND. Cross-checks +against other sources (fetch_pypi_downloads_via_bigquery.py, +fetch_pypi_downloads_via_pepy.py) print to stdout and never touch the +cache. -Results are cached to data/pypi_downloads.tsv (merged with any existing -rows); names with no PyPI rows are written as NOT_FOUND. - -Usage: python fetch_pypi_downloads_via_clickpy.py [--bigquery] [--dry-run] [--names-file FILE] +Usage: python fetch_pypi_downloads_via_clickpy.py """ import re -import subprocess -import sys -from json import loads from pathlib import Path import httpx @@ -45,7 +29,6 @@ from readme_parser import parse_readme DATA_DIR = Path(__file__).parent / "data" OUT_FILE = DATA_DIR / "pypi_downloads.tsv" README_PATH = Path(__file__).parent.parent / "README.md" -MAX_BYTES_BILLED = 400_000_000_000 CLICKPY_URL = "https://sql-clickhouse.clickhouse.com/?user=demo" # PyPI normalizes names to lowercase with runs of -, _, . collapsed to -. @@ -80,54 +63,12 @@ def fetch_clickpy(names: list[str]) -> dict[str, int]: return {row["project"]: int(row["downloads"]) for row in resp.json()["data"]} -def fetch_bigquery(names: list[str], dry_run: bool) -> dict[str, int]: - in_list = ", ".join(f"'{name}'" for name in names) - query = ( - "SELECT project, COUNT(*) AS downloads " - "FROM `bigquery-public-data.pypi.file_downloads` " - "WHERE DATE(timestamp) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) " - "AND CURRENT_DATE() " - f"AND project IN ({in_list}) " - "GROUP BY project" - ) - cmd = ["bq", "query", "--use_legacy_sql=false", "--format=json", f"--maximum_bytes_billed={MAX_BYTES_BILLED}"] - if dry_run: - cmd.append("--dry_run") - cmd.append(query) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(result.stderr, file=sys.stderr) - sys.exit(1) - if dry_run: - print(result.stdout.strip() or result.stderr.strip()) - sys.exit(0) - rows = loads(result.stdout) - return {row["project"]: int(row["downloads"]) for row in rows} - - def main() -> None: - dry_run = "--dry-run" in sys.argv - use_bigquery = "--bigquery" in sys.argv - if dry_run and not use_bigquery: - print("Error: --dry-run only applies to --bigquery.", file=sys.stderr) - sys.exit(1) - if "--names-file" in sys.argv: - names_path = Path(sys.argv[sys.argv.index("--names-file") + 1]) - names = sorted({normalize(line) for line in names_path.read_text().split() if line}) - else: - names = collect_names(README_PATH.read_text()) + names = collect_names(README_PATH.read_text()) print(f"Querying {len(names)} package names...") - counts = fetch_bigquery(names, dry_run) if use_bigquery else fetch_clickpy(names) - merged: dict[str, str] = {} - if OUT_FILE.exists(): - for line in OUT_FILE.read_text().splitlines(): - name, _, value = line.partition("\t") - merged[name] = value - for name in names: - merged[name] = str(counts.get(name, "NOT_FOUND")) - OUT_FILE.write_text("\n".join(f"{name}\t{value}" for name, value in sorted(merged.items())) + "\n") - found = sum(1 for name in names if name in counts) - print(f"Done. {found}/{len(names)} names found on PyPI. Cached to {OUT_FILE}") + counts = fetch_clickpy(names) + OUT_FILE.write_text("\n".join(f"{name}\t{counts.get(name, 'NOT_FOUND')}" for name in names) + "\n") + print(f"Done. {len(counts)}/{len(names)} names found on PyPI. Cached to {OUT_FILE}") if __name__ == "__main__":