mirror of
https://github.com/vinta/awesome-python.git
synced 2026-11-08 02:15:12 -06:00
refactor: split download fetchers into writer and cross-check scripts
Authored by the parallel fetch-scripts session (its intended 0cffb5f never landed; the content rode into an audit commit by accident and is extracted here): fetch_pypi_downloads_via_clickpy.py becomes the flagless full-README sweep and sole writer of data/pypi_downloads.tsv; new fetch_pypi_downloads_via_bigquery.py is a print-only cross-check taking explicit names behind a 400 GB billing cap; audit-the-list SKILL.md step 2 updated to match. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Vinta Chen
Claude
@@ -16,7 +16,7 @@ Resolve the scope from the arguments. Named sections mean exactly those, whether
|
||||
|
||||
Fetch live evidence for every entry in scope before judging anything (CLAUDE.md verification rule):
|
||||
|
||||
- **Downloads/month**: `cd website && UV_PYTHON=3.13 uv run python fetch_pypi_downloads_via_clickpy.py --names-file <file>` — default source is ClickPy (free, keyless, one batched query covers the full README). `--bigquery` switches to the canonical cross-check on the maintainer's own GCP account; there, run `--dry-run` first and read the docstring's cost constraints before batching. Spot-checks: `uv run python fetch_pypi_downloads_via_pepy.py <name> ...` (needs `PEPY_TECH_API_KEY` in repo-root `.env`, throttled to 5 requests/minute) or `https://pypistats.org/api/packages/{name}/recent` paced 8s or slower. pypistats excludes mirror/CI traffic; ClickPy, BigQuery, and pepy include it — never mix sources within one comparison.
|
||||
- **Downloads/month**: `cd website && UV_PYTHON=3.13 uv run python fetch_pypi_downloads_via_clickpy.py` — free keyless ClickPy sweep of the full README, sole writer of `data/pypi_downloads.tsv` (rewritten from scratch each run). Cross-checks print to stdout, take explicit names, and never touch the cache: `fetch_pypi_downloads_via_bigquery.py <name> ...` (canonical source, maintainer's own GCP account, `--dry-run` first — the docstring carries the cost constraints; full-README sweeps exceed the free tier, keep name lists small), `fetch_pypi_downloads_via_pepy.py <name> ...` (needs `PEPY_TECH_API_KEY` in repo-root `.env`, throttled to 5 requests/minute), or `https://pypistats.org/api/packages/{name}/recent` paced 8s or slower. pypistats excludes mirror/CI traffic; ClickPy, BigQuery, and pepy include it — never mix sources within one comparison.
|
||||
- **Repo state**: archived flag, last push, created date, stars, description — `gh api repos/{owner}/{repo}`, GitLab API for GitLab-hosted projects.
|
||||
- **PyPI metadata** (`https://pypi.org/pypi/{name}/json`) wherever a name might not be the canonical package — ownership collisions and wrong display names surface here.
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross-check last-30-day PyPI download counts for given names via BigQuery.
|
||||
|
||||
Queries the canonical source ClickPy mirrors —
|
||||
`bigquery-public-data.pypi.file_downloads` via the `bq` CLI — and prints
|
||||
name<TAB>count TSV to stdout. Maintainer-local (needs a personal GCP
|
||||
account) and print-only: data/pypi_downloads.tsv is written solely by
|
||||
fetch_pypi_downloads_via_clickpy.py.
|
||||
|
||||
Targeted cross-checks only — full-README sweeps belong to ClickPy. The
|
||||
dry-run scan estimate scales with the IN list (measured 2026-08-16:
|
||||
~38 GB for 2 names, ~275 GB for 53, ~1.2 TB for the full README — past
|
||||
both MAX_BYTES_BILLED and the 1 TiB/month free tier), and BigQuery
|
||||
enforces MAX_BYTES_BILLED against that pre-run estimate, so oversized
|
||||
name lists fail before billing. Actual billed bytes come in far lower
|
||||
via cluster pruning (33.7 GB for a single name). The table is
|
||||
partitioned on timestamp and clustered on the top-level `project`
|
||||
column — filter on `project`, never `file.project`: both hold identical
|
||||
values (verified 2026-08-16, 408M rows on one day, zero mismatches), but
|
||||
only a `project` filter gets cluster pruning, and the `file` record
|
||||
costs ~4x more to scan. Always --dry-run first to check the estimate.
|
||||
|
||||
Usage: python fetch_pypi_downloads_via_bigquery.py [--dry-run] NAME [NAME ...]
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from json import loads
|
||||
|
||||
from fetch_pypi_downloads_via_clickpy import normalize
|
||||
|
||||
MAX_BYTES_BILLED = 400_000_000_000
|
||||
|
||||
|
||||
def fetch_bigquery(names: list[str], dry_run: bool) -> dict[str, int]:
|
||||
in_list = ", ".join(f"'{name}'" for name in names)
|
||||
query = (
|
||||
"SELECT project, COUNT(*) AS downloads "
|
||||
"FROM `bigquery-public-data.pypi.file_downloads` "
|
||||
"WHERE DATE(timestamp) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) "
|
||||
"AND CURRENT_DATE() "
|
||||
f"AND project IN ({in_list}) "
|
||||
"GROUP BY project"
|
||||
)
|
||||
cmd = ["bq", "query", "--use_legacy_sql=false", "--format=json", f"--maximum_bytes_billed={MAX_BYTES_BILLED}"]
|
||||
if dry_run:
|
||||
cmd.append("--dry_run")
|
||||
cmd.append(query)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if dry_run:
|
||||
print(result.stdout.strip() or result.stderr.strip())
|
||||
sys.exit(0)
|
||||
rows = loads(result.stdout)
|
||||
return {row["project"]: int(row["downloads"]) for row in rows}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
dry_run = "--dry-run" in sys.argv
|
||||
names = sorted({normalize(arg) for arg in sys.argv[1:] if arg != "--dry-run"})
|
||||
if not names:
|
||||
print("Usage: python fetch_pypi_downloads_via_bigquery.py [--dry-run] NAME [NAME ...]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
counts = fetch_bigquery(names, dry_run)
|
||||
for name in names:
|
||||
print(f"{name}\t{counts.get(name, 'NOT_FOUND')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,42 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch last-30-day PyPI download counts for README entries.
|
||||
"""Fetch last-30-day PyPI download counts for all README entries via ClickPy.
|
||||
|
||||
Default source is ClickPy, ClickHouse's public mirror of the PyPI download
|
||||
dataset (sql-clickhouse.clickhouse.com, user `demo`, no key, no personal
|
||||
ClickPy is ClickHouse's public mirror of the PyPI download dataset
|
||||
(sql-clickhouse.clickhouse.com, user `demo`, no key, no personal
|
||||
account): one batched query covers the full README in under a second.
|
||||
Playground limits — 60 queries/hour/IP, 1000 result rows — sit far above
|
||||
a full-README sweep. The per-day table is a SummingMergeTree, so the
|
||||
query must sum(count) GROUP BY, never count raw rows. Counts include
|
||||
mirror/CI traffic, same as BigQuery's; never mix these figures with
|
||||
pypistats.org, which excludes mirrors by default.
|
||||
mirror/CI traffic; never mix these figures with pypistats.org, which
|
||||
excludes mirrors by default.
|
||||
|
||||
--bigquery switches to the canonical source ClickPy mirrors:
|
||||
`bigquery-public-data.pypi.file_downloads` via the `bq` CLI (needs a
|
||||
personal GCP account, so it is a maintainer-local cross-check, not the
|
||||
default). The table is partitioned on timestamp and clustered on the
|
||||
top-level `project` column — filter on `project`, never `file.project`:
|
||||
both hold identical values (verified 2026-08-16, 408M rows on one day,
|
||||
zero mismatches), but only a `project` filter gets cluster pruning, and
|
||||
the `file` record costs ~4x more to scan. A 30-day query dry-runs at up
|
||||
to ~275 GB (the pre-pruning upper bound for a full-README IN list; small
|
||||
lists estimate lower); actual billed bytes shrink further with cluster
|
||||
pruning (33.7 GB measured for a single name), so even a full-README sweep
|
||||
fits the 1 TiB/month free tier. MAX_BYTES_BILLED is a safety net against
|
||||
accidentally unpartitioned queries; BigQuery enforces it against the
|
||||
pre-run upper bound, so it must stay above the dry-run estimate or every
|
||||
query fails. --dry-run (BigQuery only) prints the scan estimate without
|
||||
running.
|
||||
This script is the sole writer of data/pypi_downloads.tsv and rewrites it
|
||||
from scratch each run, so entries removed from README.md drop out
|
||||
naturally; names with no PyPI rows are written as NOT_FOUND. Cross-checks
|
||||
against other sources (fetch_pypi_downloads_via_bigquery.py,
|
||||
fetch_pypi_downloads_via_pepy.py) print to stdout and never touch the
|
||||
cache.
|
||||
|
||||
Results are cached to data/pypi_downloads.tsv (merged with any existing
|
||||
rows); names with no PyPI rows are written as NOT_FOUND.
|
||||
|
||||
Usage: python fetch_pypi_downloads_via_clickpy.py [--bigquery] [--dry-run] [--names-file FILE]
|
||||
Usage: python fetch_pypi_downloads_via_clickpy.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from json import loads
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
@@ -45,7 +29,6 @@ from readme_parser import parse_readme
|
||||
DATA_DIR = Path(__file__).parent / "data"
|
||||
OUT_FILE = DATA_DIR / "pypi_downloads.tsv"
|
||||
README_PATH = Path(__file__).parent.parent / "README.md"
|
||||
MAX_BYTES_BILLED = 400_000_000_000
|
||||
CLICKPY_URL = "https://sql-clickhouse.clickhouse.com/?user=demo"
|
||||
|
||||
# PyPI normalizes names to lowercase with runs of -, _, . collapsed to -.
|
||||
@@ -80,54 +63,12 @@ def fetch_clickpy(names: list[str]) -> dict[str, int]:
|
||||
return {row["project"]: int(row["downloads"]) for row in resp.json()["data"]}
|
||||
|
||||
|
||||
def fetch_bigquery(names: list[str], dry_run: bool) -> dict[str, int]:
|
||||
in_list = ", ".join(f"'{name}'" for name in names)
|
||||
query = (
|
||||
"SELECT project, COUNT(*) AS downloads "
|
||||
"FROM `bigquery-public-data.pypi.file_downloads` "
|
||||
"WHERE DATE(timestamp) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) "
|
||||
"AND CURRENT_DATE() "
|
||||
f"AND project IN ({in_list}) "
|
||||
"GROUP BY project"
|
||||
)
|
||||
cmd = ["bq", "query", "--use_legacy_sql=false", "--format=json", f"--maximum_bytes_billed={MAX_BYTES_BILLED}"]
|
||||
if dry_run:
|
||||
cmd.append("--dry_run")
|
||||
cmd.append(query)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if dry_run:
|
||||
print(result.stdout.strip() or result.stderr.strip())
|
||||
sys.exit(0)
|
||||
rows = loads(result.stdout)
|
||||
return {row["project"]: int(row["downloads"]) for row in rows}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
dry_run = "--dry-run" in sys.argv
|
||||
use_bigquery = "--bigquery" in sys.argv
|
||||
if dry_run and not use_bigquery:
|
||||
print("Error: --dry-run only applies to --bigquery.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if "--names-file" in sys.argv:
|
||||
names_path = Path(sys.argv[sys.argv.index("--names-file") + 1])
|
||||
names = sorted({normalize(line) for line in names_path.read_text().split() if line})
|
||||
else:
|
||||
names = collect_names(README_PATH.read_text())
|
||||
names = collect_names(README_PATH.read_text())
|
||||
print(f"Querying {len(names)} package names...")
|
||||
counts = fetch_bigquery(names, dry_run) if use_bigquery else fetch_clickpy(names)
|
||||
merged: dict[str, str] = {}
|
||||
if OUT_FILE.exists():
|
||||
for line in OUT_FILE.read_text().splitlines():
|
||||
name, _, value = line.partition("\t")
|
||||
merged[name] = value
|
||||
for name in names:
|
||||
merged[name] = str(counts.get(name, "NOT_FOUND"))
|
||||
OUT_FILE.write_text("\n".join(f"{name}\t{value}" for name, value in sorted(merged.items())) + "\n")
|
||||
found = sum(1 for name in names if name in counts)
|
||||
print(f"Done. {found}/{len(names)} names found on PyPI. Cached to {OUT_FILE}")
|
||||
counts = fetch_clickpy(names)
|
||||
OUT_FILE.write_text("\n".join(f"{name}\t{counts.get(name, 'NOT_FOUND')}" for name in names) + "\n")
|
||||
print(f"Done. {len(counts)}/{len(names)} names found on PyPI. Cached to {OUT_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user