mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-03-12 10:34:34 -05:00
Simplify module.yaml and enhance export command with real export targets
- Remove redundant fields from module.yaml files: exports_to, files, components - Keep only essential system metadata: name, title, description, dependencies - Export command now reads actual export targets from dev files (#| default_exp directive) - Status command updated to use dev files as source of truth for export targets - Export command shows detailed source → target mapping for better clarity - Dependencies field retained as it's useful for CLI module ordering and prerequisites - Eliminates duplication between YAML and dev files - dev files are the real truth
This commit is contained in:
@@ -2,25 +2,10 @@
|
||||
# Essential system information for CLI tools and build systems
|
||||
|
||||
name: "activations"
|
||||
title: "Activations"
|
||||
description: "Activation functions (ReLU, Sigmoid, Tanh, etc.)"
|
||||
title: "Activation Functions"
|
||||
description: "Neural network activation functions (ReLU, Sigmoid, Tanh)"
|
||||
|
||||
# Dependencies
|
||||
# Dependencies - Used by CLI for module ordering and prerequisites
|
||||
dependencies:
|
||||
prerequisites: ["setup", "tensor"]
|
||||
enables: ["layers", "networks"]
|
||||
|
||||
# Package Export
|
||||
exports_to: "tinytorch.core.activations"
|
||||
|
||||
# File Structure
|
||||
files:
|
||||
dev_file: "activations_dev.py"
|
||||
test_file: "tests/test_activations.py"
|
||||
readme: "README.md"
|
||||
|
||||
# Components
|
||||
components:
|
||||
- "ReLU"
|
||||
- "Sigmoid"
|
||||
- "Tanh"
|
||||
prerequisites: ["tensor"]
|
||||
enables: ["layers", "networks"]
|
||||
@@ -2,25 +2,10 @@
|
||||
# Essential system information for CLI tools and build systems
|
||||
|
||||
name: "setup"
|
||||
title: "Setup"
|
||||
description: "Development environment setup and project configuration"
|
||||
title: "Setup & Environment"
|
||||
description: "Development environment setup and basic TinyTorch functionality"
|
||||
|
||||
# Dependencies
|
||||
# Dependencies - Used by CLI for module ordering and prerequisites
|
||||
dependencies:
|
||||
prerequisites: []
|
||||
enables: ["tensor", "activations", "layers"]
|
||||
|
||||
# Package Export
|
||||
exports_to: "tinytorch.core.utils"
|
||||
|
||||
# File Structure
|
||||
files:
|
||||
dev_file: "setup_dev.py"
|
||||
test_file: "tests/test_setup.py"
|
||||
readme: "README.md"
|
||||
|
||||
# Components
|
||||
components:
|
||||
- "hello_tinytorch"
|
||||
- "environment_check"
|
||||
- "project_structure"
|
||||
enables: ["tensor", "activations", "layers"]
|
||||
@@ -5,22 +5,7 @@ name: "tensor"
|
||||
title: "Tensor"
|
||||
description: "Core tensor data structure and operations"
|
||||
|
||||
# Dependencies
|
||||
# Dependencies - Used by CLI for module ordering and prerequisites
|
||||
dependencies:
|
||||
prerequisites: ["setup"]
|
||||
enables: ["activations", "layers", "autograd"]
|
||||
|
||||
# Package Export
|
||||
exports_to: "tinytorch.core.tensor"
|
||||
|
||||
# File Structure
|
||||
files:
|
||||
dev_file: "tensor_dev.py"
|
||||
test_file: "tests/test_tensor.py"
|
||||
readme: "README.md"
|
||||
|
||||
# Components
|
||||
components:
|
||||
- "Tensor"
|
||||
- "tensor_operations"
|
||||
- "shape_manipulation"
|
||||
enables: ["activations", "layers", "autograd"]
|
||||
@@ -3,9 +3,6 @@
|
||||
# %% auto 0
|
||||
__all__ = ['ReLU', 'Sigmoid', 'Tanh']
|
||||
|
||||
from tinytorch.core.tensor import Tensor
|
||||
import numpy as np
|
||||
|
||||
# %% ../../modules/activations/activations_dev.ipynb 5
|
||||
class ReLU:
|
||||
"""
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""
|
||||
Sync command for TinyTorch CLI: exports notebook code to Python package using nbdev.
|
||||
Export command for TinyTorch CLI: exports notebook code to Python package using nbdev.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import re
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from pathlib import Path
|
||||
from rich.panel import Panel
|
||||
@@ -23,6 +24,71 @@ class ExportCommand(BaseCommand):
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument("--module", help="Export specific module (e.g., setup, tensor)")
|
||||
|
||||
def _get_export_target(self, module_path: Path) -> str:
|
||||
"""
|
||||
Read the actual export target from the dev file's #| default_exp directive.
|
||||
This is the source of truth, not the YAML file.
|
||||
"""
|
||||
dev_file = module_path / f"{module_path.name}_dev.py"
|
||||
if not dev_file.exists():
|
||||
return "unknown"
|
||||
|
||||
try:
|
||||
with open(dev_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
# Look for #| default_exp directive with more flexible regex
|
||||
match = re.search(r'#\|\s*default_exp\s+([^\n\r]+)', content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
except Exception as e:
|
||||
# Debug: print the error for troubleshooting
|
||||
print(f"Debug: Error reading {dev_file}: {e}")
|
||||
|
||||
return "unknown"
|
||||
|
||||
def _show_export_details(self, console, module_name: str = None):
|
||||
"""Show detailed export information including where each module exports to."""
|
||||
exports_text = Text()
|
||||
exports_text.append("📦 Export Details:\n", style="bold cyan")
|
||||
|
||||
if module_name:
|
||||
# Single module export
|
||||
module_path = Path(f"modules/{module_name}")
|
||||
export_target = self._get_export_target(module_path)
|
||||
if export_target != "unknown":
|
||||
target_file = export_target.replace('.', '/') + '.py'
|
||||
exports_text.append(f" 🔄 {module_name} → tinytorch/{target_file}\n", style="green")
|
||||
exports_text.append(f" Source: modules/{module_name}/{module_name}_dev.py\n", style="dim")
|
||||
exports_text.append(f" Target: tinytorch/{target_file}\n", style="dim")
|
||||
else:
|
||||
exports_text.append(f" ❓ {module_name} → export target not found\n", style="yellow")
|
||||
else:
|
||||
# All modules export
|
||||
modules_dir = Path("modules")
|
||||
if modules_dir.exists():
|
||||
exclude_dirs = {'.quarto', '__pycache__', '.git', '.pytest_cache'}
|
||||
for module_dir in modules_dir.iterdir():
|
||||
if module_dir.is_dir() and module_dir.name not in exclude_dirs:
|
||||
export_target = self._get_export_target(module_dir)
|
||||
if export_target != "unknown":
|
||||
target_file = export_target.replace('.', '/') + '.py'
|
||||
exports_text.append(f" 🔄 {module_dir.name} → tinytorch/{target_file}\n", style="green")
|
||||
|
||||
# Show what was actually created
|
||||
exports_text.append("\n📁 Generated Files:\n", style="bold cyan")
|
||||
tinytorch_path = Path("tinytorch")
|
||||
if tinytorch_path.exists():
|
||||
for py_file in tinytorch_path.rglob("*.py"):
|
||||
if py_file.name != "__init__.py" and py_file.stat().st_size > 100: # Non-empty files
|
||||
rel_path = py_file.relative_to(tinytorch_path)
|
||||
exports_text.append(f" ✅ tinytorch/{rel_path}\n", style="green")
|
||||
|
||||
exports_text.append("\n💡 Next steps:\n", style="bold yellow")
|
||||
exports_text.append(" • Run: tito module test --all\n", style="white")
|
||||
exports_text.append(" • Or: tito module test --module <name>\n", style="white")
|
||||
|
||||
console.print(Panel(exports_text, title="Export Summary", border_style="bright_green"))
|
||||
|
||||
def run(self, args: Namespace) -> int:
|
||||
console = self.console
|
||||
|
||||
@@ -55,23 +121,8 @@ class ExportCommand(BaseCommand):
|
||||
console.print(Panel("[green]✅ Successfully exported notebook code to tinytorch package![/green]",
|
||||
title="Export Success", border_style="green"))
|
||||
|
||||
# Show what was exported
|
||||
exports_text = Text()
|
||||
exports_text.append("📦 Exported modules:\n", style="bold cyan")
|
||||
|
||||
# Check for exported files
|
||||
tinytorch_path = Path("tinytorch")
|
||||
if tinytorch_path.exists():
|
||||
for py_file in tinytorch_path.rglob("*.py"):
|
||||
if py_file.name != "__init__.py" and py_file.stat().st_size > 100: # Non-empty files
|
||||
rel_path = py_file.relative_to(tinytorch_path)
|
||||
exports_text.append(f" ✅ tinytorch/{rel_path}\n", style="green")
|
||||
|
||||
exports_text.append("\n💡 Next steps:\n", style="bold yellow")
|
||||
exports_text.append(" • Run: tito test --module setup\n", style="white")
|
||||
exports_text.append(" • Or: tito test --all\n", style="white")
|
||||
|
||||
console.print(Panel(exports_text, title="Export Summary", border_style="bright_green"))
|
||||
# Show detailed export information
|
||||
self._show_export_details(console, args.module if hasattr(args, 'module') else None)
|
||||
|
||||
else:
|
||||
error_msg = result.stderr.strip() if result.stderr else "Unknown error"
|
||||
@@ -84,7 +135,7 @@ class ExportCommand(BaseCommand):
|
||||
help_text.append(" • Missing #| default_exp directive in notebook\n", style="white")
|
||||
help_text.append(" • Syntax errors in exported code\n", style="white")
|
||||
help_text.append(" • Missing settings.ini configuration\n", style="white")
|
||||
help_text.append("\n🔧 Run 'tito doctor' for detailed diagnosis", style="cyan")
|
||||
help_text.append("\n🔧 Run 'tito system doctor' for detailed diagnosis", style="cyan")
|
||||
|
||||
console.print(Panel(help_text, title="Troubleshooting", border_style="yellow"))
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ Status command for TinyTorch CLI: checks status of all modules in modules/ direc
|
||||
import subprocess
|
||||
import sys
|
||||
import yaml
|
||||
import re
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from pathlib import Path
|
||||
from rich.panel import Panel
|
||||
@@ -26,6 +27,27 @@ class StatusCommand(BaseCommand):
|
||||
parser.add_argument("--details", action="store_true", help="Show detailed file structure")
|
||||
parser.add_argument("--metadata", action="store_true", help="Show module metadata information")
|
||||
|
||||
def _get_export_target(self, module_path: Path) -> str:
|
||||
"""
|
||||
Read the actual export target from the dev file's #| default_exp directive.
|
||||
Same logic as the export command.
|
||||
"""
|
||||
dev_file = module_path / f"{module_path.name}_dev.py"
|
||||
if not dev_file.exists():
|
||||
return "unknown"
|
||||
|
||||
try:
|
||||
with open(dev_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
# Look for #| default_exp directive
|
||||
match = re.search(r'#\|\s*default_exp\s+([^\n\r]+)', content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return "unknown"
|
||||
|
||||
def run(self, args: Namespace) -> int:
|
||||
console = self.console
|
||||
|
||||
@@ -59,7 +81,7 @@ class StatusCommand(BaseCommand):
|
||||
|
||||
if args.metadata:
|
||||
status_table.add_column("Export", width=15, justify="center")
|
||||
status_table.add_column("Components", width=15, justify="center")
|
||||
status_table.add_column("Prerequisites", width=15, justify="center")
|
||||
|
||||
# Check each module
|
||||
modules_status = []
|
||||
@@ -80,9 +102,14 @@ class StatusCommand(BaseCommand):
|
||||
# Add metadata columns if requested
|
||||
if args.metadata:
|
||||
metadata = status.get('metadata', {})
|
||||
row.append(metadata.get('exports_to', 'unknown'))
|
||||
components = metadata.get('components', [])
|
||||
row.append(f"{len(components)} items" if components else 'none')
|
||||
# Get export target from dev file (source of truth)
|
||||
export_target = self._get_export_target(module_dir)
|
||||
row.append(export_target if export_target != "unknown" else 'unknown')
|
||||
|
||||
# Show prerequisites from dependencies
|
||||
deps = metadata.get('dependencies', {})
|
||||
prereqs = deps.get('prerequisites', [])
|
||||
row.append(', '.join(prereqs) if prereqs else 'none')
|
||||
|
||||
status_table.add_row(*row)
|
||||
|
||||
@@ -222,9 +249,11 @@ class StatusCommand(BaseCommand):
|
||||
if metadata.get('description'):
|
||||
console.print(f"📝 {metadata['description']}")
|
||||
|
||||
# Export info (no longer showing static status)
|
||||
if metadata.get('exports_to'):
|
||||
console.print(f"📦 Exports to: {metadata['exports_to']}")
|
||||
# Export info (read from dev file - source of truth)
|
||||
module_path = Path(f"modules/{module_name}")
|
||||
export_target = self._get_export_target(module_path)
|
||||
if export_target != "unknown":
|
||||
console.print(f"📦 Exports to: {export_target}")
|
||||
|
||||
# Dependencies
|
||||
if metadata.get('dependencies'):
|
||||
|
||||
Reference in New Issue
Block a user