mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-11-09 10:39:41 -06:00
Vijay Janapa Reddi
27d4f8114b
Bug fixes
- Build lock is keyed on books/ so builds started from different
directories block each other (fixes two tests broken by 89b828efa3).
- ConfigManager resolves the repo root when binder runs from books/ or
binder/, so bib, newsletter, doctor and render paths are right.
- `build <word> --all` treats intro/scaling/agentic/physical as chapter
names (it built every volume), and Ctrl-C stops a --all loop.
- doctor checked chapter names that no longer exist and failed on the
generated _quarto.yml; info defaulted to a deleted books/contents.
- `headings apply --path <typo>` no longer rewrites the whole book;
`bib clean` stops on unreadable chapters instead of dropping entries.
- Discovery falls back to a line scan on invalid YAML instead of crashing.
- Volume III/IV were labeled Volume II in layout and release output.
Output
- Unexpected errors print the command, exception type, and raising
file:line (new cli/core/errors.py), with the traceback on stderr.
- Chapter-not-found suggests the closest names and the list command.
- Silent except blocks in bib lookups, info, headings, layout,
maintenance, preview and debug now say what went wrong.
Cleanup
- Shared qmd_files_under/display_path, file-rewrite loop, doctor result
helper, preview server, clean helpers, volume display names.
- Removed legacy import fallbacks, dead config attributes, legacy backup
restore logic, and unused imports.
773 lines
32 KiB
Python
773 lines
32 KiB
Python
"""
|
|
``binder bib`` — Bibliography management.
|
|
|
|
Subcommands:
|
|
list — Show all .bib files with entry counts
|
|
mechanical — Apply safe §5 field-level fixes (pre-commit first step)
|
|
clean — Remove unused entries from .bib files
|
|
update — Run betterbib sync-preserve on .bib files (fetch metadata, keep citekeys stable)
|
|
sync — Clean then update (full pipeline)
|
|
normalize — Full-tree: same 3 steps as pre-commit (mechanical → tidy → binder bib check)
|
|
add — Authenticate, format, and seal a new entry (the reference gateway)
|
|
ingest — Verify and promote candidate entries from bibs/inbox.bib
|
|
reseal — Re-verify a house-style-edited entry against its source and re-record it
|
|
"""
|
|
|
|
import argparse
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Set
|
|
|
|
from rich.console import Console
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
from cli.core.bib_mechanical import run_mechanical_fixes
|
|
from cli.core.discovery import display_path, qmd_files_under
|
|
|
|
console = Console()
|
|
|
|
|
|
class BibCommand:
|
|
"""Native ``binder bib`` command group."""
|
|
|
|
def __init__(self, config_manager, chapter_discovery):
|
|
"""Store the shared config manager and chapter discovery helpers."""
|
|
self.config_manager = config_manager
|
|
self.chapter_discovery = chapter_discovery
|
|
|
|
# ------------------------------------------------------------------
|
|
# Entry point
|
|
# ------------------------------------------------------------------
|
|
|
|
def run(self, args: List[str]) -> bool:
|
|
"""Parse ``binder bib`` arguments and dispatch to the chosen subcommand.
|
|
|
|
``mechanical`` works on the positional .bib files. ``normalize`` with no
|
|
``--path``/``--vol1``/``--vol2`` covers the whole repo; every other
|
|
subcommand needs the resolved path to exist. With no subcommand (or
|
|
``help``) prints help and returns True. Returns False on an argparse
|
|
error (True for ``-h``/``--help``), a missing path, or subcommand failure.
|
|
"""
|
|
if args == ["help"]:
|
|
self._print_help()
|
|
return True
|
|
|
|
parser = argparse.ArgumentParser(
|
|
prog="binder bib",
|
|
description="Bibliography management",
|
|
)
|
|
parser.add_argument(
|
|
"subcommand",
|
|
nargs="?",
|
|
choices=["list", "mechanical", "clean", "update", "sync", "normalize", "add", "ingest", "reseal"],
|
|
help="Subcommand to run",
|
|
)
|
|
parser.add_argument("files", nargs="*", help=".bib file(s) for mechanical fixes")
|
|
parser.add_argument("--path", default=None, help="File or directory")
|
|
parser.add_argument("--vol1", action="store_true", help="Legacy alias for the shared book bibliography")
|
|
parser.add_argument("--vol2", action="store_true", help="Legacy alias for the shared book bibliography")
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="Show what would change without modifying files",
|
|
)
|
|
parser.add_argument(
|
|
"--pre-commit",
|
|
action="store_true",
|
|
help="Mechanical fixes: exit 1 if a file was modified",
|
|
)
|
|
parser.add_argument("--doi", default=None, help="Add reference by DOI (Crossref/DataCite)")
|
|
parser.add_argument("--arxiv", default=None, help="Add reference by arXiv ID")
|
|
parser.add_argument("--isbn", default=None, help="Add reference by ISBN (OpenLibrary)")
|
|
parser.add_argument("--url", default=None, help="Add reference by official URL")
|
|
parser.add_argument("--key", default=None, help="Custom citation key for add; the entry to re-seal for reseal")
|
|
parser.add_argument("--note", default=None, help="reseal: what was edited after the gateway seal (required)")
|
|
parser.add_argument("--title", default=None, help="Custom title override for --url")
|
|
parser.add_argument("--author", default=None, help="Custom author override for --url")
|
|
parser.add_argument("--year", default=None, help="Custom year override for --url")
|
|
parser.add_argument("--venue", default=None, help="Custom venue override for --url")
|
|
parser.add_argument("--inbox", default="bibs/inbox.bib", help="Inbox path for 'ingest' subcommand")
|
|
|
|
try:
|
|
# Pre-commit appends filenames after the hook entry, so the public
|
|
# shape is `bib mechanical --pre-commit refs.bib`. Python 3.11's
|
|
# regular argparse parser rejects that optional-between-positionals
|
|
# form when a `nargs="*"` positional follows the subcommand.
|
|
parse = getattr(parser, "parse_intermixed_args", parser.parse_args)
|
|
ns = parse(args)
|
|
except SystemExit:
|
|
return ("-h" in args) or ("--help" in args)
|
|
|
|
if not ns.subcommand:
|
|
self._print_help()
|
|
return True
|
|
|
|
if ns.subcommand == "mechanical":
|
|
return self._run_mechanical(
|
|
ns.files,
|
|
dry_run=ns.dry_run,
|
|
pre_commit=ns.pre_commit,
|
|
)
|
|
|
|
if ns.subcommand == "add":
|
|
return self._run_add(
|
|
doi=ns.doi,
|
|
arxiv=ns.arxiv,
|
|
isbn=ns.isbn,
|
|
url=ns.url,
|
|
custom_key=ns.key,
|
|
title=ns.title,
|
|
author=ns.author,
|
|
year=ns.year,
|
|
venue=ns.venue,
|
|
dry_run=ns.dry_run,
|
|
)
|
|
|
|
if ns.subcommand == "reseal":
|
|
return self._run_reseal(
|
|
key=ns.key,
|
|
note=ns.note,
|
|
doi=ns.doi,
|
|
arxiv=ns.arxiv,
|
|
url=ns.url,
|
|
dry_run=ns.dry_run,
|
|
)
|
|
|
|
if ns.subcommand == "ingest":
|
|
return self._run_ingest(
|
|
inbox=ns.inbox,
|
|
dry_run=ns.dry_run,
|
|
)
|
|
|
|
root = self._resolve_path(ns.path, ns.vol1, ns.vol2)
|
|
if ns.subcommand == "normalize":
|
|
# Full-repo normalize uses git ls-files inside the script; no path needed.
|
|
if not ns.path and not ns.vol1 and not ns.vol2:
|
|
return self._run_normalize(scoped=False, root=root)
|
|
if not root.exists():
|
|
console.print(f"[red]Path not found: {root}[/red]")
|
|
return False
|
|
return self._run_normalize(scoped=True, root=root)
|
|
|
|
if not root.exists():
|
|
console.print(f"[red]Path not found: {root}[/red]")
|
|
return False
|
|
|
|
if ns.subcommand == "list":
|
|
return self._run_list(root)
|
|
elif ns.subcommand == "clean":
|
|
return self._run_clean(root, ns.dry_run)
|
|
elif ns.subcommand == "update":
|
|
return self._run_update(root, ns.dry_run)
|
|
elif ns.subcommand == "sync":
|
|
return self._run_sync(root, ns.dry_run)
|
|
return False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Help
|
|
# ------------------------------------------------------------------
|
|
|
|
def _print_help(self) -> None:
|
|
"""Print the subcommand table and usage examples."""
|
|
table = Table(show_header=True, header_style="bold cyan", box=None)
|
|
table.add_column("Subcommand", style="cyan", width=14)
|
|
table.add_column("Description", style="white", width=50)
|
|
table.add_row("add", "Authenticate, format, and seal new citation into ledger")
|
|
table.add_row("ingest", "Verify and promote candidate citations from inbox")
|
|
table.add_row("reseal", "Re-verify an edited entry against its source; re-record its manifest hash")
|
|
table.add_row("list", "Show all .bib files with entry counts")
|
|
table.add_row("mechanical", "Apply safe §5 field-level fixes")
|
|
table.add_row("clean", "Remove unused entries from .bib files")
|
|
table.add_row(
|
|
"update",
|
|
"Run betterbib sync-preserve (fetch metadata, keep citekeys stable)",
|
|
)
|
|
table.add_row("sync", "Clean then update (full pipeline)")
|
|
table.add_row(
|
|
"normalize",
|
|
"§5 mechanical fixes + bibtex-tidy + binder check bib",
|
|
)
|
|
console.print(Panel(table, title="binder bib <subcommand>", border_style="cyan"))
|
|
console.print("[dim]Examples:[/dim]")
|
|
console.print(" [cyan]./binder/binder bib add --doi 10.1145/3592433[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib add --arxiv 1706.03762[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib add --url https://github.com/huggingface/lerobot --key cadene2024lerobot[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib add --isbn 9780136566953[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib ingest --inbox bibs/inbox.bib[/cyan]")
|
|
console.print(' [cyan]./binder/binder bib reseal --key vaswani2017attention --note "title case" --dry-run[/cyan]')
|
|
console.print(" [cyan]./binder/binder bib list --path bibs/references.bib[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib mechanical refs.bib[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib clean --dry-run[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib update --path bibs/references.bib[/cyan]")
|
|
console.print(" [cyan]./binder/binder bib sync --dry-run[/cyan]")
|
|
console.print(
|
|
" [cyan]./binder/binder bib normalize[/cyan] "
|
|
"# all git-tracked .bib (run from repo root)"
|
|
)
|
|
console.print(" [cyan]./binder/binder bib normalize --path bibs/references.bib[/cyan]")
|
|
console.print()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Path resolution
|
|
# ------------------------------------------------------------------
|
|
|
|
def _resolve_path(self, path_arg: Optional[str], vol1: bool, vol2: bool) -> Path:
|
|
"""Return ``--path`` (relative paths resolve against the cwd) or the book directory.
|
|
|
|
``--vol1``/``--vol2`` are legacy aliases that also resolve to the book
|
|
directory, which holds the shared bibliography.
|
|
"""
|
|
if path_arg:
|
|
p = Path(path_arg)
|
|
return p if p.is_absolute() else Path.cwd() / p
|
|
return self.config_manager.book_dir
|
|
|
|
# ------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _find_bib_files(self, root: Path) -> List[Path]:
|
|
"""Find all .bib files under root."""
|
|
if root.is_file() and root.suffix == ".bib":
|
|
return [root]
|
|
return sorted(root.rglob("*.bib"))
|
|
|
|
def _count_entries(self, bib_path: Path) -> int:
|
|
"""Count @type{key, entries in a .bib file."""
|
|
content = bib_path.read_text(encoding="utf-8")
|
|
return len(re.findall(r"^@\w+\{", content, re.MULTILINE))
|
|
|
|
def _extract_cited_keys(self, qmd_path: Path) -> Set[str]:
|
|
"""Extract all citation keys from a .qmd file.
|
|
|
|
Read errors propagate: treating an unreadable chapter as citing nothing
|
|
would mark its entries unused and let ``bib clean`` delete them.
|
|
"""
|
|
content = qmd_path.read_text(encoding="utf-8")
|
|
# Standard @key citations
|
|
keys = set(re.findall(r"@([\w\d:\-_.]+)", content))
|
|
# Filter out cross-ref prefixes
|
|
keys = {
|
|
k.rstrip(".,;:")
|
|
for k in keys
|
|
if not k.startswith(("fig-", "tbl-", "sec-", "eq-", "lst-", "alg-", "algo-", "Algo-"))
|
|
}
|
|
return keys
|
|
|
|
def _extract_bib_keys(self, bib_path: Path) -> Set[str]:
|
|
"""Extract all entry keys from a .bib file."""
|
|
content = bib_path.read_text(encoding="utf-8")
|
|
return set(re.findall(r"^@\w+\{([^,\s]+)", content, re.MULTILINE))
|
|
|
|
def _bib_for_qmd(self, qmd_path: Path) -> Optional[Path]:
|
|
"""Find the .bib file referenced in a .qmd file's YAML front matter."""
|
|
content = qmd_path.read_text(encoding="utf-8")
|
|
m = re.search(r"^bibliography:\s*([\w\-.]+\.bib)\s*$", content, re.MULTILINE)
|
|
if m:
|
|
bib = qmd_path.parent / m.group(1)
|
|
return bib if bib.exists() else None
|
|
return None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Mechanical fixes
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_mechanical(
|
|
self,
|
|
files: List[str],
|
|
*,
|
|
dry_run: bool = False,
|
|
pre_commit: bool = False,
|
|
) -> bool:
|
|
"""Apply safe section 5 mechanical fixes through Binder-native code."""
|
|
repo = self.config_manager.root_dir
|
|
return (
|
|
run_mechanical_fixes(
|
|
repo,
|
|
files,
|
|
dry_run=dry_run,
|
|
pre_commit=pre_commit,
|
|
)
|
|
== 0
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# List
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_list(self, root: Path) -> bool:
|
|
"""Show all .bib files with entry counts."""
|
|
bib_files = self._find_bib_files(root)
|
|
if not bib_files:
|
|
console.print("[yellow]No .bib files found.[/yellow]")
|
|
return True
|
|
|
|
table = Table(show_header=True, header_style="bold cyan", box=None)
|
|
table.add_column("File", style="cyan", width=50)
|
|
table.add_column("Entries", style="white", width=8, justify="right")
|
|
|
|
total = 0
|
|
for bib in bib_files:
|
|
count = self._count_entries(bib)
|
|
total += count
|
|
table.add_row(display_path(bib, self.config_manager.book_dir), str(count))
|
|
|
|
console.print(Panel(table, title="Bibliography Files", border_style="cyan"))
|
|
console.print(
|
|
f"[bold]Total:[/bold] {len(bib_files)} files, {total} entries"
|
|
)
|
|
return True
|
|
|
|
# ------------------------------------------------------------------
|
|
# Clean
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_clean(self, root: Path, dry_run: bool = False) -> bool:
|
|
"""Remove unused entries from .bib files."""
|
|
qmd_files = qmd_files_under(root)
|
|
if not qmd_files:
|
|
console.print("[yellow]No QMD files found.[/yellow]")
|
|
return False
|
|
|
|
# Collect ALL cited keys from all QMD files in scope
|
|
all_cited: Set[str] = set()
|
|
for qmd in qmd_files:
|
|
all_cited.update(self._extract_cited_keys(qmd))
|
|
|
|
# Find all .bib files in scope — bibliography is declared at project
|
|
# level in Quarto YAML configs, not in individual QMD front matter.
|
|
# So we match all bib files under root against all citations under root.
|
|
bib_files = self._find_bib_files(root)
|
|
|
|
# Also check per-QMD bibliography declarations (some chapters have their own)
|
|
for qmd in qmd_files:
|
|
bib = self._bib_for_qmd(qmd)
|
|
if bib and bib not in bib_files:
|
|
bib_files.append(bib)
|
|
|
|
if not bib_files:
|
|
console.print("[yellow]No .bib files found.[/yellow]")
|
|
return True
|
|
|
|
# Build mapping: each bib file checked against all cited keys
|
|
bib_cited: Dict[Path, Set[str]] = {bib: all_cited for bib in bib_files}
|
|
|
|
# Warn if scoped to a narrow path — citations from other chapters
|
|
# won't be visible, which could cause false positives.
|
|
contents_dir = self.config_manager.book_dir
|
|
is_narrow = root != contents_dir and not any(
|
|
root == contents_dir / v for v in ("vol1", "vol2")
|
|
)
|
|
if is_narrow:
|
|
console.print(
|
|
"[yellow]⚠ Narrow scope:[/yellow] Only citations from QMD files "
|
|
f"under [cyan]{display_path(root, self.config_manager.book_dir)}[/cyan] are visible.\n"
|
|
" Entries cited by other chapters may appear unused. "
|
|
"Use the full [cyan]books[/cyan] tree for safe cleaning.\n"
|
|
)
|
|
|
|
console.print(
|
|
f"[dim]Collected {len(all_cited)} citation keys from "
|
|
f"{len(qmd_files)} QMD files[/dim]\n"
|
|
)
|
|
|
|
total_removed = 0
|
|
for bib_path, cited_keys in sorted(bib_cited.items()):
|
|
bib_keys = self._extract_bib_keys(bib_path)
|
|
unused = bib_keys - cited_keys
|
|
rel = display_path(bib_path, self.config_manager.book_dir)
|
|
|
|
if not unused:
|
|
console.print(f" [green]✓[/green] {rel}: all {len(bib_keys)} entries used")
|
|
continue
|
|
|
|
total_removed += len(unused)
|
|
if dry_run:
|
|
console.print(
|
|
f" [yellow]⚠[/yellow] {rel}: {len(unused)} unused of {len(bib_keys)} "
|
|
f"[dim](dry run — no changes)[/dim]"
|
|
)
|
|
for key in sorted(unused)[:10]:
|
|
console.print(f" [dim]- {key}[/dim]")
|
|
if len(unused) > 10:
|
|
console.print(f" [dim]... and {len(unused) - 10} more[/dim]")
|
|
else:
|
|
# Remove unused entries by rewriting the file
|
|
self._remove_entries(bib_path, unused)
|
|
console.print(
|
|
f" [green]✓[/green] {rel}: removed {len(unused)} unused entries "
|
|
f"({len(bib_keys) - len(unused)} remaining)"
|
|
)
|
|
|
|
action = "would remove" if dry_run else "removed"
|
|
console.print(f"\n[bold]Total:[/bold] {action} {total_removed} unused entries from {len(bib_cited)} files")
|
|
return True
|
|
|
|
def _remove_entries(self, bib_path: Path, keys_to_remove: Set[str]) -> None:
|
|
"""Remove specific entries from a .bib file, preserving formatting."""
|
|
content = bib_path.read_text(encoding="utf-8")
|
|
lines = content.split("\n")
|
|
output: List[str] = []
|
|
skip = False
|
|
brace_depth = 0
|
|
|
|
for line in lines:
|
|
# Detect entry start
|
|
m = re.match(r"^@\w+\{([^,\s]+)", line)
|
|
if m:
|
|
key = m.group(1)
|
|
if key in keys_to_remove:
|
|
skip = True
|
|
brace_depth = line.count("{") - line.count("}")
|
|
continue
|
|
else:
|
|
skip = False
|
|
|
|
if skip:
|
|
brace_depth += line.count("{") - line.count("}")
|
|
if brace_depth <= 0:
|
|
skip = False
|
|
continue
|
|
|
|
output.append(line)
|
|
|
|
# Clean up multiple blank lines
|
|
cleaned = re.sub(r"\n{3,}", "\n\n", "\n".join(output))
|
|
bib_path.write_text(cleaned.strip() + "\n", encoding="utf-8")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Normalize (mechanical fixes + pre-commit tidy + binder check bib)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_normalize(self, scoped: bool, root: Path) -> bool:
|
|
"""Run the same pipeline needed for consistent .bib with pre-commit."""
|
|
repo = self.config_manager.root_dir
|
|
|
|
if scoped:
|
|
bibs = self._find_bib_files(root)
|
|
if not bibs:
|
|
console.print("[yellow]No .bib files in scope.[/yellow]")
|
|
return True
|
|
mechanical_files = [str(path) for path in bibs]
|
|
else:
|
|
mechanical_files = None
|
|
|
|
console.print(
|
|
"[bold]Step 1/3:[/bold] "
|
|
"Mechanical section 5 fixes ([cyan]binder bib mechanical[/cyan])...\n"
|
|
)
|
|
if run_mechanical_fixes(repo, mechanical_files) != 0:
|
|
console.print("[red]Step 1/3 failed: fix the mechanical issues reported above, then rerun "
|
|
"./binder/binder bib normalize.[/red]")
|
|
return False
|
|
|
|
if not shutil.which("pre-commit"):
|
|
console.print(
|
|
"\n[yellow]pre-commit not on PATH — skipped Step 2. "
|
|
"Install pre-commit and run: "
|
|
"[cyan]pre-commit run bibtex-tidy --all-files[/cyan][/yellow]\n"
|
|
)
|
|
else:
|
|
console.print(
|
|
"\n[bold]Step 2/3:[/bold] [cyan]pre-commit run bibtex-tidy --all-files[/cyan] "
|
|
"(loop until Pass)…\n"
|
|
)
|
|
settled = False
|
|
for _i in range(12):
|
|
p = subprocess.run(
|
|
["pre-commit", "run", "bibtex-tidy", "--all-files"],
|
|
cwd=str(repo),
|
|
)
|
|
if p.returncode == 0:
|
|
settled = True
|
|
break
|
|
if not settled:
|
|
console.print(
|
|
"\n[red]bibtex-tidy did not reach a stable pass in 12 attempts; "
|
|
"resolve diff manually.[/red]"
|
|
)
|
|
return False
|
|
|
|
binder = repo / "binder" / "binder"
|
|
check_cmd = [sys.executable, str(binder), "check", "bib"]
|
|
if scoped:
|
|
check_cmd.extend(["--path", str(root)])
|
|
console.print(
|
|
"\n[bold]Step 3/3:[/bold] "
|
|
"[cyan]./binder/binder check bib[/cyan] "
|
|
"(same curated gate as the pre-commit [cyan]book-check-bib[/cyan] hook)…\n"
|
|
)
|
|
p2 = subprocess.run(
|
|
check_cmd,
|
|
cwd=str(repo),
|
|
)
|
|
if p2.returncode != 0:
|
|
console.print(
|
|
"\n[red]binder check bib reported new issues. Fix them or update "
|
|
"the appropriate reviewed baseline after accepting the debt.[/red]"
|
|
)
|
|
else:
|
|
console.print(
|
|
"\n[green]Bib files match the tidy + curated bib checks "
|
|
"used in pre-commit.[/green]"
|
|
)
|
|
return p2.returncode == 0
|
|
|
|
# ------------------------------------------------------------------
|
|
# Update
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_update(self, root: Path, dry_run: bool = False) -> bool:
|
|
"""Run betterbib sync-preserve on .bib files."""
|
|
repo = self.config_manager.root_dir
|
|
script = repo / "binder" / "tools" / "bib_sync_preserve.py"
|
|
if not script.is_file():
|
|
console.print(f"[red]Missing {script}[/red]")
|
|
return False
|
|
|
|
# Check if betterbib is available
|
|
try:
|
|
subprocess.run(
|
|
["betterbib", "--version"],
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
except (FileNotFoundError, subprocess.CalledProcessError):
|
|
console.print(
|
|
"[red]betterbib not found.[/red] Install with: "
|
|
"[cyan]pip install betterbib[/cyan]"
|
|
)
|
|
return False
|
|
|
|
bib_files = self._find_bib_files(root)
|
|
if not bib_files:
|
|
console.print("[yellow]No .bib files found.[/yellow]")
|
|
return True
|
|
|
|
console.print(
|
|
f"[bold]Updating {len(bib_files)} .bib file(s) with betterbib sync-preserve...[/bold]\n"
|
|
)
|
|
|
|
cmd = [sys.executable, str(script)]
|
|
if dry_run:
|
|
cmd.append("--dry-run")
|
|
if root.is_file() and root.suffix == ".bib":
|
|
cmd.append(str(root))
|
|
else:
|
|
cmd.extend(str(b) for b in bib_files)
|
|
|
|
result = subprocess.run(
|
|
cmd,
|
|
cwd=str(repo),
|
|
text=True,
|
|
)
|
|
if result.returncode == 0:
|
|
console.print("\n[green]Done.[/green]")
|
|
return True
|
|
console.print(f"\n[red]{script.name} exited with code {result.returncode}; "
|
|
"see the failures it printed above.[/red]")
|
|
return False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Sync (clean + update)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_sync(self, root: Path, dry_run: bool = False) -> bool:
|
|
"""Full pipeline: clean unused entries, then update with betterbib."""
|
|
console.print("[bold cyan]Step 1/2: Cleaning unused entries...[/bold cyan]\n")
|
|
clean_ok = self._run_clean(root, dry_run)
|
|
|
|
if dry_run:
|
|
console.print(
|
|
"\n[yellow]Dry run — skipping betterbib update. "
|
|
"Run without --dry-run to apply changes and update.[/yellow]"
|
|
)
|
|
return clean_ok
|
|
|
|
console.print("\n[bold cyan]Step 2/2: Updating with betterbib...[/bold cyan]\n")
|
|
update_ok = self._run_update(root)
|
|
|
|
return clean_ok and update_ok
|
|
|
|
# ------------------------------------------------------------------
|
|
# Add (authoritative gateway)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_add(
|
|
self,
|
|
doi: Optional[str] = None,
|
|
arxiv: Optional[str] = None,
|
|
isbn: Optional[str] = None,
|
|
url: Optional[str] = None,
|
|
custom_key: Optional[str] = None,
|
|
title: Optional[str] = None,
|
|
author: Optional[str] = None,
|
|
year: Optional[str] = None,
|
|
venue: Optional[str] = None,
|
|
dry_run: bool = False,
|
|
) -> bool:
|
|
"""Authenticate, format, and seal a new reference into the master bibliography."""
|
|
from cli.core import bib_gateway
|
|
|
|
repo = self.config_manager.root_dir
|
|
console.print("[bold cyan]Resolving reference via authoritative gateway...[/bold cyan]")
|
|
|
|
resolved = None
|
|
if doi:
|
|
console.print(f"[dim]Authenticating DOI '{doi}' via Crossref / DataCite...[/dim]")
|
|
resolved = bib_gateway.fetch_doi(doi, custom_key=custom_key)
|
|
elif arxiv:
|
|
console.print(f"[dim]Authenticating arXiv ID '{arxiv}' via arXiv.org...[/dim]")
|
|
resolved = bib_gateway.fetch_arxiv(arxiv, custom_key=custom_key)
|
|
elif isbn:
|
|
console.print(f"[dim]Authenticating ISBN '{isbn}' via OpenLibrary...[/dim]")
|
|
resolved = bib_gateway.fetch_isbn(isbn, custom_key=custom_key)
|
|
elif url:
|
|
console.print(f"[dim]Authenticating URL '{url}' via HTTP live check...[/dim]")
|
|
resolved = bib_gateway.fetch_url(
|
|
url,
|
|
custom_key=custom_key,
|
|
title_override=title or "",
|
|
author_override=author or "",
|
|
year_override=year or "",
|
|
venue_override=venue or "",
|
|
)
|
|
else:
|
|
console.print("[red]Error:[/red] Must provide one of --doi, --arxiv, --isbn, or --url.")
|
|
return False
|
|
|
|
if not resolved:
|
|
console.print(
|
|
"[red]Authentication failed:[/red] Could not verify or retrieve canonical "
|
|
"metadata from primary external registries. Lookup errors, if any, are "
|
|
"printed above; check the identifier or pass a different source "
|
|
"(--doi, --arxiv, --isbn, --url)."
|
|
)
|
|
return False
|
|
|
|
console.print(
|
|
f"[green]Authenticated:[/green] {resolved['evidence']['source']} -> "
|
|
f"'{resolved['fields'].get('title')}'"
|
|
)
|
|
console.print(f"[dim]Candidate citekey: @{resolved['entry_type']}{{{resolved['key']}}}[/dim]")
|
|
|
|
if dry_run:
|
|
console.print("\n[yellow]Dry run — entry would be formatted and sealed as follows:[/yellow]")
|
|
console.print(bib_gateway.format_bibtex_entry(resolved["entry_type"], resolved["key"], resolved["fields"]))
|
|
return True
|
|
|
|
ok, msg, record = bib_gateway.add_entry(repo, resolved, check_duplicates=True)
|
|
if not ok:
|
|
console.print(f"[red]Error:[/red] {msg}")
|
|
return False
|
|
|
|
console.print(f"\n[bold green]Success![/bold green] {msg}")
|
|
if record:
|
|
console.print(f"[dim]Cryptographic SHA-256: {record['entry_hash'][:16]}...[/dim]")
|
|
return True
|
|
|
|
# ------------------------------------------------------------------
|
|
# Reseal (house-style edit to a sealed entry)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_reseal(
|
|
self,
|
|
key: Optional[str],
|
|
note: Optional[str],
|
|
doi: Optional[str] = None,
|
|
arxiv: Optional[str] = None,
|
|
url: Optional[str] = None,
|
|
dry_run: bool = False,
|
|
) -> bool:
|
|
"""Re-resolve an edited entry's source, check its identity fields, and re-record it.
|
|
|
|
The source is the identifier passed on the command line, else the one the
|
|
manifest record was sealed from (DOI, arXiv ID, or verified URL), else the
|
|
entry's own ``doi``/``url`` field. Refuses when any checked field disagrees.
|
|
"""
|
|
import json as _json
|
|
|
|
from cli.core import bib_gateway
|
|
|
|
if not key or not note:
|
|
console.print("[red]Error:[/red] reseal needs --key and --note (what was edited after the seal).")
|
|
return False
|
|
|
|
repo = self.config_manager.root_dir
|
|
record = {}
|
|
manifest_path = repo / "bibs" / "manifest.json"
|
|
if manifest_path.is_file():
|
|
record = _json.loads(manifest_path.read_text(encoding="utf-8")).get(key) or {}
|
|
details = record.get("details") or {}
|
|
|
|
entry_fields: Dict[str, str] = {}
|
|
bib_text = (repo / "bibs" / "references.bib").read_text(encoding="utf-8")
|
|
for entry in bib_gateway.bib_lint.parse_bib(bib_text)[0]:
|
|
if entry.key == key:
|
|
entry_fields = {n: (entry.get(n).value if entry.get(n) else "") for n in ("doi", "url")}
|
|
break
|
|
if not entry_fields:
|
|
console.print(f"[red]Error:[/red] @{{{key}}} not found in bibs/references.bib.")
|
|
return False
|
|
|
|
if not (doi or arxiv or url):
|
|
doi = details.get("doi") if details.get("source") in ("crossref", "datacite") else None
|
|
arxiv = details.get("arxiv_id")
|
|
if not (doi or arxiv) and details.get("source") == "verified_web":
|
|
url = details.get("url")
|
|
if not (doi or arxiv or url):
|
|
m_arx = bib_gateway.ARXIV_REGEX.search(entry_fields.get("url", "")) if "arxiv" in entry_fields.get("url", "") else None
|
|
if entry_fields.get("doi"):
|
|
doi = entry_fields["doi"]
|
|
elif m_arx:
|
|
arxiv = m_arx.group(1)
|
|
elif entry_fields.get("url"):
|
|
url = entry_fields["url"]
|
|
if not (doi or arxiv or url):
|
|
console.print("[red]Error:[/red] no source identifier found; pass --doi, --arxiv, or --url.")
|
|
return False
|
|
|
|
if doi:
|
|
source_desc, resolved = f"DOI {doi}", bib_gateway.fetch_doi(doi)
|
|
elif arxiv:
|
|
source_desc, resolved = f"arXiv {arxiv}", bib_gateway.fetch_arxiv(arxiv)
|
|
else:
|
|
source_desc, resolved = f"URL {url}", bib_gateway.fetch_url(url)
|
|
if not resolved:
|
|
console.print(f"[red]Could not resolve {source_desc}; lookup errors, if any, are printed above.[/red]")
|
|
return False
|
|
|
|
ok, msg, _ = bib_gateway.reseal_entry(repo, key, resolved, note, dry_run=dry_run)
|
|
style = "green" if ok else "red"
|
|
console.print(f"[{style}]{source_desc}: {msg}[/{style}]")
|
|
return ok
|
|
|
|
# ------------------------------------------------------------------
|
|
# Ingest (staging inbox)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_ingest(self, inbox: str = "bibs/inbox.bib", dry_run: bool = False) -> bool:
|
|
"""Scan staging inbox file, authenticate each entry online, promote verified entries."""
|
|
from cli.core import bib_gateway
|
|
|
|
repo = self.config_manager.root_dir
|
|
inbox_path = repo / inbox
|
|
if not inbox_path.is_file():
|
|
console.print(f"[yellow]Inbox file not found: {inbox_path}[/yellow]")
|
|
return True
|
|
|
|
console.print(f"[bold cyan]Scanning staging inbox {inbox_path}...[/bold cyan]")
|
|
added, rejected, messages = bib_gateway.ingest_inbox(repo, inbox_path, dry_run=dry_run)
|
|
|
|
for msg in messages:
|
|
console.print(f" {msg}")
|
|
|
|
console.print("\n" + "=" * 50)
|
|
console.print(f"[bold]Ingestion Summary:[/bold] {added} verified & added, {rejected} rejected/pending.")
|
|
return rejected == 0
|
|
|