feat: switch default PyPI download source to ClickPy

BigQuery required a personal Google Cloud account, so only the maintainer could run it and overruns cost real money. ClickPy is ClickHouse's free, keyless public mirror of the same PyPI download dataset, so it becomes the default source with a single batched query covering the full README in under a second.

BigQuery stays available behind a new --bigquery flag as the canonical-source cross-check; --dry-run now only applies to that path.

Live-tested: ClickPy full path (2/2 names), the --dry-run guard, and the BigQuery dry-run.

Co-Authored-By: Claude <noreply@anthropic.com>

website/fetch_pypi_downloads.py

@@ -1,24 +1,36 @@
 #!/usr/bin/env python3
-"""Fetch last-30-day PyPI download counts for README entries via BigQuery.
+"""Fetch last-30-day PyPI download counts for README entries.
 
-Shells out to the `bq` CLI (Google Cloud SDK) against the public
-`bigquery-public-data.pypi.file_downloads` table. The table is partitioned
-on timestamp and clustered on the top-level `project` column — filter on
-`project`, never `file.project`: both hold identical values (verified
-2026-08-16, 408M rows on one day, zero mismatches), but only a `project`
-filter gets cluster pruning, and the `file` record costs ~4x more to scan.
-A 30-day query dry-runs at up to ~275 GB (the pre-pruning upper bound for
-a full-README IN list; small lists estimate lower); actual billed bytes
-shrink further with cluster pruning (33.7 GB measured for a single name),
-so even a full-README sweep fits the 1 TiB/month free tier.
-MAX_BYTES_BILLED is a safety net against accidentally unpartitioned
-queries; BigQuery enforces it against the pre-run upper bound, so it must
-stay above the dry-run estimate or every query fails. Results are cached
-to data/pypi_downloads.tsv (merged with any existing rows); names with no
-PyPI rows are written as NOT_FOUND. Always --dry-run first to check the
-scan estimate.
+Default source is ClickPy, ClickHouse's public mirror of the PyPI download
+dataset (sql-clickhouse.clickhouse.com, user `demo`, no key, no personal
+account): one batched query covers the full README in under a second.
+Playground limits — 60 queries/hour/IP, 1000 result rows — sit far above
+a full-README sweep. The per-day table is a SummingMergeTree, so the
+query must sum(count) GROUP BY, never count raw rows. Counts include
+mirror/CI traffic, same as BigQuery's; never mix these figures with
+pypistats.org, which excludes mirrors by default.
 
-Usage: python fetch_pypi_downloads.py [--dry-run] [--names-file FILE]
+--bigquery switches to the canonical source ClickPy mirrors:
+`bigquery-public-data.pypi.file_downloads` via the `bq` CLI (needs a
+personal GCP account, so it is a maintainer-local cross-check, not the
+default). The table is partitioned on timestamp and clustered on the
+top-level `project` column — filter on `project`, never `file.project`:
+both hold identical values (verified 2026-08-16, 408M rows on one day,
+zero mismatches), but only a `project` filter gets cluster pruning, and
+the `file` record costs ~4x more to scan. A 30-day query dry-runs at up
+to ~275 GB (the pre-pruning upper bound for a full-README IN list; small
+lists estimate lower); actual billed bytes shrink further with cluster
+pruning (33.7 GB measured for a single name), so even a full-README sweep
+fits the 1 TiB/month free tier. MAX_BYTES_BILLED is a safety net against
+accidentally unpartitioned queries; BigQuery enforces it against the
+pre-run upper bound, so it must stay above the dry-run estimate or every
+query fails. --dry-run (BigQuery only) prints the scan estimate without
+running.
+
+Results are cached to data/pypi_downloads.tsv (merged with any existing
+rows); names with no PyPI rows are written as NOT_FOUND.
+
+Usage: python fetch_pypi_downloads.py [--bigquery] [--dry-run] [--names-file FILE]
 """
 
 import re
@@ -27,12 +39,14 @@ import sys
 from json import loads
 from pathlib import Path
 
+import httpx
 from readme_parser import parse_readme
 
 DATA_DIR = Path(__file__).parent / "data"
 OUT_FILE = DATA_DIR / "pypi_downloads.tsv"
 README_PATH = Path(__file__).parent.parent / "README.md"
 MAX_BYTES_BILLED = 400_000_000_000
+CLICKPY_URL = "https://sql-clickhouse.clickhouse.com/?user=demo"
 
 # PyPI normalizes names to lowercase with runs of -, _, . collapsed to -.
 PYPI_NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$")
@@ -53,7 +67,20 @@ def collect_names(readme_text: str) -> list[str]:
     return sorted(names)
 
 
-def fetch(names: list[str], dry_run: bool) -> dict[str, int]:
+def fetch_clickpy(names: list[str]) -> dict[str, int]:
+    in_list = ", ".join(f"'{name}'" for name in names)
+    query = (
+        "SELECT project, sum(count) AS downloads "
+        "FROM pypi.pypi_downloads_per_day "
+        f"WHERE project IN ({in_list}) AND date >= today() - 30 "
+        "GROUP BY project FORMAT JSON"
+    )
+    resp = httpx.post(CLICKPY_URL, content=query, timeout=60)
+    resp.raise_for_status()
+    return {row["project"]: int(row["downloads"]) for row in resp.json()["data"]}
+
+
+def fetch_bigquery(names: list[str], dry_run: bool) -> dict[str, int]:
     in_list = ", ".join(f"'{name}'" for name in names)
     query = (
         "SELECT project, COUNT(*) AS downloads "
@@ -80,13 +107,17 @@ def fetch(names: list[str], dry_run: bool) -> dict[str, int]:
 
 def main() -> None:
     dry_run = "--dry-run" in sys.argv
+    use_bigquery = "--bigquery" in sys.argv
+    if dry_run and not use_bigquery:
+        print("Error: --dry-run only applies to --bigquery.", file=sys.stderr)
+        sys.exit(1)
     if "--names-file" in sys.argv:
         names_path = Path(sys.argv[sys.argv.index("--names-file") + 1])
         names = sorted({normalize(line) for line in names_path.read_text().split() if line})
     else:
         names = collect_names(README_PATH.read_text())
     print(f"Querying {len(names)} package names...")
-    counts = fetch(names, dry_run)
+    counts = fetch_bigquery(names, dry_run) if use_bigquery else fetch_clickpy(names)
     merged: dict[str, str] = {}
     if OUT_FILE.exists():
         for line in OUT_FILE.read_text().splitlines():