mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-05-03 08:08:51 -05:00
bb5dd7a9f6
The nbdev_export CLI was not reading settings.ini correctly in CI, causing exports to silently fail. Using nbdev.export.nb_export() Python API directly with explicit lib_path ensures exports work reliably regardless of environment.
308 lines
12 KiB
Python
308 lines
12 KiB
Python
"""
|
|
Developer export command: rebuilds curriculum from source files.
|
|
|
|
This is a DEVELOPER command for maintainers, NOT for students.
|
|
Workflow: src/*.py → modules/*.ipynb → tinytorch/core/*.py
|
|
|
|
Students should use `tito module complete` which only exports their
|
|
notebook work to the package (without overwriting their notebooks).
|
|
"""
|
|
|
|
import subprocess
|
|
import logging
|
|
from argparse import ArgumentParser, Namespace
|
|
from pathlib import Path
|
|
from typing import Optional, Dict
|
|
|
|
from rich.panel import Panel
|
|
from rich.text import Text
|
|
|
|
from ..base import BaseCommand
|
|
from ..export_utils import (
|
|
add_autogenerated_warnings,
|
|
convert_all_modules,
|
|
convert_py_to_notebook,
|
|
discover_modules,
|
|
ensure_writable_target,
|
|
get_export_target,
|
|
validate_notebook_integrity,
|
|
)
|
|
from ...core.modules import get_module_name
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DevExportCommand(BaseCommand):
|
|
"""Developer export command: rebuild curriculum from src/ files."""
|
|
|
|
@property
|
|
def name(self) -> str:
|
|
return "export"
|
|
|
|
@property
|
|
def description(self) -> str:
|
|
return "Rebuild curriculum: src/*.py → modules/*.ipynb → tinytorch/core/*.py"
|
|
|
|
def add_arguments(self, parser: ArgumentParser) -> None:
|
|
"""Add export arguments."""
|
|
parser.add_argument(
|
|
'modules',
|
|
nargs='*',
|
|
help='Module names to export (e.g., 01 or 01_tensor)'
|
|
)
|
|
parser.add_argument(
|
|
'--all',
|
|
action='store_true',
|
|
help='Export all modules'
|
|
)
|
|
parser.add_argument(
|
|
'--test-checkpoint',
|
|
action='store_true',
|
|
help='Run checkpoint test after successful export'
|
|
)
|
|
|
|
def _get_export_target(self, module_path: Path) -> str:
|
|
return get_export_target(module_path)
|
|
|
|
def _discover_modules(self) -> list:
|
|
return discover_modules()
|
|
|
|
def _add_autogenerated_warnings(self, console):
|
|
add_autogenerated_warnings(console)
|
|
|
|
def _convert_py_to_notebook(self, module_path: Path) -> bool:
|
|
return convert_py_to_notebook(module_path, self.venv_path, self.console)
|
|
|
|
def _convert_all_modules(self) -> list:
|
|
return convert_all_modules(self.venv_path, self.console)
|
|
|
|
def run(self, args: Namespace) -> int:
|
|
"""Handle the export command."""
|
|
console = self.console
|
|
logger.info("Starting dev export command")
|
|
|
|
# Show developer warning
|
|
console.print(Panel(
|
|
"[bold yellow]⚠️ Developer Command[/bold yellow]\n\n"
|
|
"This rebuilds notebooks from src/*.py files.\n"
|
|
"[bold red]Student notebooks in modules/ will be OVERWRITTEN![/bold red]\n\n"
|
|
"[dim]Students: Use 'tito module complete' instead.[/dim]",
|
|
title="🛠️ Developer Export",
|
|
border_style="yellow"
|
|
))
|
|
console.print()
|
|
|
|
# Guard: Ensure we're in the correct directory (tinytorch project root)
|
|
# Check for key files that indicate we're in the right place
|
|
cwd = Path.cwd()
|
|
is_tinytorch_root = (
|
|
(cwd / "tinytorch" / "__init__.py").exists() or # Running from repo root
|
|
(cwd / "src").exists() and (cwd / "settings.ini").exists() # Already in tinytorch/
|
|
)
|
|
if not is_tinytorch_root:
|
|
console.print(Panel(
|
|
"[red]❌ Must run from TinyTorch project directory[/red]\n\n"
|
|
"[dim]Expected structure:[/dim]\n"
|
|
"[dim] tinytorch/ ← run from here[/dim]\n"
|
|
"[dim] ├── src/[/dim]\n"
|
|
"[dim] ├── tinytorch/ ← package exports here[/dim]\n"
|
|
"[dim] │ └── __init__.py[/dim]\n"
|
|
"[dim] └── settings.ini[/dim]",
|
|
title="Wrong Directory", border_style="red"
|
|
))
|
|
return 1
|
|
|
|
# Determine what to export
|
|
if hasattr(args, 'modules') and args.modules:
|
|
return self._export_specific_modules(args.modules, console)
|
|
elif hasattr(args, 'all') and args.all:
|
|
return self._export_all_modules(console)
|
|
else:
|
|
console.print(Panel(
|
|
"[red]❌ Must specify either module names or --all[/red]\n\n"
|
|
"[dim]Examples:[/dim]\n"
|
|
"[dim] tito dev export 01[/dim] (shorthand)\n"
|
|
"[dim] tito dev export 01_tensor[/dim] (full name)\n"
|
|
"[dim] tito dev export 01 02 03[/dim] (multiple)\n"
|
|
"[dim] tito dev export --all[/dim] (all modules)",
|
|
title="Missing Arguments", border_style="red"
|
|
))
|
|
return 1
|
|
|
|
def _export_specific_modules(self, modules_to_export: list, console) -> int:
|
|
"""Export specific modules."""
|
|
# Normalize module names (e.g., "01" -> "01_tensor", "1" -> "01_tensor")
|
|
normalized_modules = []
|
|
for module_input in modules_to_export:
|
|
module_name = get_module_name(module_input)
|
|
if module_name:
|
|
normalized_modules.append(module_name)
|
|
else:
|
|
# If normalization fails, use as-is (will fail later with helpful error)
|
|
normalized_modules.append(module_input)
|
|
|
|
logger.info(f"Exporting specific modules: {normalized_modules}")
|
|
|
|
console.print(Panel(
|
|
f"🔄 Exporting Modules: {', '.join(normalized_modules)}",
|
|
title="Developer Export Workflow",
|
|
border_style="bright_cyan"
|
|
))
|
|
|
|
exported_notebooks = []
|
|
|
|
# Step 1: Convert each py file to notebook
|
|
for module_name in normalized_modules:
|
|
logger.debug(f"Processing module: {module_name}")
|
|
module_path = Path(f"src/{module_name}")
|
|
|
|
if not module_path.exists():
|
|
console.print(Panel(
|
|
f"[red]❌ Module '{module_name}' not found in src/[/red]",
|
|
title="Module Not Found", border_style="red"
|
|
))
|
|
self._show_available_modules(console)
|
|
return 1
|
|
|
|
# Convert Python file to notebook (OVERWRITES existing notebook!)
|
|
short_name = module_name.split("_", 1)[1] if "_" in module_name else module_name
|
|
notebook_file = Path("modules") / module_name / f"{short_name}.ipynb"
|
|
|
|
console.print(f"📝 Converting {module_name}: src/*.py → modules/*.ipynb")
|
|
if not self._convert_py_to_notebook(module_path):
|
|
logger.error(f"Failed to convert .py to notebook for {module_name}")
|
|
return 1
|
|
exported_notebooks.append(str(notebook_file))
|
|
|
|
# Step 2: Export notebooks to package
|
|
logger.info(f"Exporting {len(exported_notebooks)} notebooks to tinytorch package")
|
|
return self._run_nbdev_export(exported_notebooks, console)
|
|
|
|
def _export_all_modules(self, console) -> int:
|
|
"""Export all modules."""
|
|
logger.info("Exporting all modules")
|
|
|
|
console.print(Panel(
|
|
"🔄 Exporting All Modules",
|
|
title="Developer Export Workflow",
|
|
border_style="bright_cyan"
|
|
))
|
|
|
|
# Step 1: Convert all .py files to .ipynb
|
|
console.print("📝 Converting all Python files to notebooks...")
|
|
converted = self._convert_all_modules()
|
|
|
|
if not converted:
|
|
logger.error("No modules converted")
|
|
console.print(Panel(
|
|
"[red]❌ No modules converted.[/red]\n\n"
|
|
"[dim]Check that:[/dim]\n"
|
|
"[dim] • jupytext is installed[/dim]\n"
|
|
"[dim] • src/*.py files exist[/dim]",
|
|
title="Conversion Error", border_style="red"
|
|
))
|
|
return 1
|
|
|
|
console.print(f"✅ Converted {len(converted)} modules: {', '.join(converted)}")
|
|
|
|
# Step 2: Export all using nbdev Python API
|
|
console.print("🔄 Exporting all notebooks to tinytorch package...")
|
|
try:
|
|
from nbdev.export import nb_export
|
|
from pathlib import Path as P
|
|
|
|
lib_path = Path.cwd() / "tinytorch"
|
|
nbs_path = Path.cwd() / "modules"
|
|
|
|
# Export all notebooks in the modules directory
|
|
exported_count = 0
|
|
for nb_file in nbs_path.rglob("*.ipynb"):
|
|
if ".ipynb_checkpoints" not in str(nb_file):
|
|
try:
|
|
nb_export(nb_file, lib_path=lib_path)
|
|
exported_count += 1
|
|
except Exception as e:
|
|
console.print(f"[yellow]⚠️ Skipped {nb_file.name}: {e}[/yellow]")
|
|
|
|
if exported_count > 0:
|
|
self._add_autogenerated_warnings(console)
|
|
console.print(Panel(
|
|
f"[green]✅ Successfully exported {exported_count} notebooks![/green]",
|
|
title="Export Success", border_style="green"
|
|
))
|
|
return 0
|
|
else:
|
|
console.print(Panel(
|
|
"[red]❌ No notebooks exported[/red]",
|
|
title="Export Error", border_style="red"
|
|
))
|
|
return 1
|
|
|
|
except ImportError:
|
|
console.print(Panel(
|
|
"[red]❌ nbdev not found[/red]\n\n"
|
|
"[dim]Install with: pip install nbdev[/dim]",
|
|
title="Missing Dependency", border_style="red"
|
|
))
|
|
return 1
|
|
except Exception as e:
|
|
console.print(Panel(
|
|
f"[red]❌ Export failed: {e}[/red]",
|
|
title="Export Error", border_style="red"
|
|
))
|
|
return 1
|
|
|
|
def _run_nbdev_export(self, notebook_paths: list, console) -> int:
|
|
"""Run nbdev_export on the given notebooks using Python API directly.
|
|
|
|
Uses nbdev.export.nb_export() instead of subprocess to ensure
|
|
settings.ini is read correctly regardless of environment.
|
|
"""
|
|
from nbdev.export import nb_export
|
|
success_count = 0
|
|
lib_path = Path.cwd() / "tinytorch"
|
|
|
|
for notebook_path_str in notebook_paths:
|
|
try:
|
|
notebook_path = Path(notebook_path_str)
|
|
notebook_name = notebook_path.name
|
|
console.print(f"[dim]🔄 Exporting {notebook_name} → tinytorch/core/...[/dim]")
|
|
|
|
# Ensure target file is writable
|
|
module_path = notebook_path.parent
|
|
export_target = self._get_export_target(module_path)
|
|
if export_target != "unknown":
|
|
ensure_writable_target(export_target)
|
|
|
|
# Use nbdev Python API directly (more reliable than subprocess)
|
|
nb_export(notebook_path, lib_path=lib_path)
|
|
success_count += 1
|
|
console.print(f"✅ Exported: {notebook_name}")
|
|
|
|
except Exception as e:
|
|
console.print(f"❌ Error exporting {Path(notebook_path_str).name}: {e}")
|
|
|
|
if success_count == len(notebook_paths):
|
|
self._add_autogenerated_warnings(console)
|
|
console.print(Panel(
|
|
f"[green]✅ Successfully exported {success_count}/{len(notebook_paths)} modules![/green]",
|
|
title="Export Success", border_style="green"
|
|
))
|
|
return 0
|
|
else:
|
|
console.print(Panel(
|
|
f"[yellow]⚠️ Exported {success_count}/{len(notebook_paths)} modules. Some failed.[/yellow]",
|
|
title="Partial Success", border_style="yellow"
|
|
))
|
|
return 1
|
|
|
|
def _show_available_modules(self, console) -> None:
|
|
"""Show available modules."""
|
|
available_modules = self._discover_modules()
|
|
if available_modules:
|
|
help_text = Text()
|
|
help_text.append("Available modules:\n", style="bold yellow")
|
|
for module in available_modules:
|
|
help_text.append(f" • {module}\n", style="white")
|
|
console.print(Panel(help_text, title="Available Modules", border_style="yellow"))
|