Simplify module.yaml and enhance export command with real export targets

- Remove redundant fields from module.yaml files: exports_to, files, components
- Keep only essential system metadata: name, title, description, dependencies
- Export command now reads actual export targets from dev files (#| default_exp directive)
- Status command updated to use dev files as source of truth for export targets
- Export command shows detailed source → target mapping for better clarity
- Dependencies field retained as it's useful for CLI module ordering and prerequisites
- Eliminates duplication between YAML and dev files - dev files are the real truth
This commit is contained in:
Vijay Janapa Reddi
2025-07-12 00:05:45 -04:00
parent 2a4bbc6a09
commit ca8ec10427
6 changed files with 117 additions and 85 deletions

View File

@@ -2,25 +2,10 @@
# Essential system information for CLI tools and build systems
name: "activations"
title: "Activations"
description: "Activation functions (ReLU, Sigmoid, Tanh, etc.)"
title: "Activation Functions"
description: "Neural network activation functions (ReLU, Sigmoid, Tanh)"
# Dependencies
# Dependencies - Used by CLI for module ordering and prerequisites
dependencies:
prerequisites: ["setup", "tensor"]
enables: ["layers", "networks"]
# Package Export
exports_to: "tinytorch.core.activations"
# File Structure
files:
dev_file: "activations_dev.py"
test_file: "tests/test_activations.py"
readme: "README.md"
# Components
components:
- "ReLU"
- "Sigmoid"
- "Tanh"
prerequisites: ["tensor"]
enables: ["layers", "networks"]

View File

@@ -2,25 +2,10 @@
# Essential system information for CLI tools and build systems
name: "setup"
title: "Setup"
description: "Development environment setup and project configuration"
title: "Setup & Environment"
description: "Development environment setup and basic TinyTorch functionality"
# Dependencies
# Dependencies - Used by CLI for module ordering and prerequisites
dependencies:
prerequisites: []
enables: ["tensor", "activations", "layers"]
# Package Export
exports_to: "tinytorch.core.utils"
# File Structure
files:
dev_file: "setup_dev.py"
test_file: "tests/test_setup.py"
readme: "README.md"
# Components
components:
- "hello_tinytorch"
- "environment_check"
- "project_structure"
enables: ["tensor", "activations", "layers"]

View File

@@ -5,22 +5,7 @@ name: "tensor"
title: "Tensor"
description: "Core tensor data structure and operations"
# Dependencies
# Dependencies - Used by CLI for module ordering and prerequisites
dependencies:
prerequisites: ["setup"]
enables: ["activations", "layers", "autograd"]
# Package Export
exports_to: "tinytorch.core.tensor"
# File Structure
files:
dev_file: "tensor_dev.py"
test_file: "tests/test_tensor.py"
readme: "README.md"
# Components
components:
- "Tensor"
- "tensor_operations"
- "shape_manipulation"
enables: ["activations", "layers", "autograd"]

View File

@@ -3,9 +3,6 @@
# %% auto 0
__all__ = ['ReLU', 'Sigmoid', 'Tanh']
from tinytorch.core.tensor import Tensor
import numpy as np
# %% ../../modules/activations/activations_dev.ipynb 5
class ReLU:
"""

View File

@@ -1,9 +1,10 @@
"""
Sync command for TinyTorch CLI: exports notebook code to Python package using nbdev.
Export command for TinyTorch CLI: exports notebook code to Python package using nbdev.
"""
import subprocess
import sys
import re
from argparse import ArgumentParser, Namespace
from pathlib import Path
from rich.panel import Panel
@@ -23,6 +24,71 @@ class ExportCommand(BaseCommand):
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument("--module", help="Export specific module (e.g., setup, tensor)")
def _get_export_target(self, module_path: Path) -> str:
"""
Read the actual export target from the dev file's #| default_exp directive.
This is the source of truth, not the YAML file.
"""
dev_file = module_path / f"{module_path.name}_dev.py"
if not dev_file.exists():
return "unknown"
try:
with open(dev_file, 'r', encoding='utf-8') as f:
content = f.read()
# Look for #| default_exp directive with more flexible regex
match = re.search(r'#\|\s*default_exp\s+([^\n\r]+)', content)
if match:
return match.group(1).strip()
except Exception as e:
# Debug: print the error for troubleshooting
print(f"Debug: Error reading {dev_file}: {e}")
return "unknown"
def _show_export_details(self, console, module_name: str = None):
"""Show detailed export information including where each module exports to."""
exports_text = Text()
exports_text.append("📦 Export Details:\n", style="bold cyan")
if module_name:
# Single module export
module_path = Path(f"modules/{module_name}")
export_target = self._get_export_target(module_path)
if export_target != "unknown":
target_file = export_target.replace('.', '/') + '.py'
exports_text.append(f" 🔄 {module_name} → tinytorch/{target_file}\n", style="green")
exports_text.append(f" Source: modules/{module_name}/{module_name}_dev.py\n", style="dim")
exports_text.append(f" Target: tinytorch/{target_file}\n", style="dim")
else:
exports_text.append(f"{module_name} → export target not found\n", style="yellow")
else:
# All modules export
modules_dir = Path("modules")
if modules_dir.exists():
exclude_dirs = {'.quarto', '__pycache__', '.git', '.pytest_cache'}
for module_dir in modules_dir.iterdir():
if module_dir.is_dir() and module_dir.name not in exclude_dirs:
export_target = self._get_export_target(module_dir)
if export_target != "unknown":
target_file = export_target.replace('.', '/') + '.py'
exports_text.append(f" 🔄 {module_dir.name} → tinytorch/{target_file}\n", style="green")
# Show what was actually created
exports_text.append("\n📁 Generated Files:\n", style="bold cyan")
tinytorch_path = Path("tinytorch")
if tinytorch_path.exists():
for py_file in tinytorch_path.rglob("*.py"):
if py_file.name != "__init__.py" and py_file.stat().st_size > 100: # Non-empty files
rel_path = py_file.relative_to(tinytorch_path)
exports_text.append(f" ✅ tinytorch/{rel_path}\n", style="green")
exports_text.append("\n💡 Next steps:\n", style="bold yellow")
exports_text.append(" • Run: tito module test --all\n", style="white")
exports_text.append(" • Or: tito module test --module <name>\n", style="white")
console.print(Panel(exports_text, title="Export Summary", border_style="bright_green"))
def run(self, args: Namespace) -> int:
console = self.console
@@ -55,23 +121,8 @@ class ExportCommand(BaseCommand):
console.print(Panel("[green]✅ Successfully exported notebook code to tinytorch package![/green]",
title="Export Success", border_style="green"))
# Show what was exported
exports_text = Text()
exports_text.append("📦 Exported modules:\n", style="bold cyan")
# Check for exported files
tinytorch_path = Path("tinytorch")
if tinytorch_path.exists():
for py_file in tinytorch_path.rglob("*.py"):
if py_file.name != "__init__.py" and py_file.stat().st_size > 100: # Non-empty files
rel_path = py_file.relative_to(tinytorch_path)
exports_text.append(f" ✅ tinytorch/{rel_path}\n", style="green")
exports_text.append("\n💡 Next steps:\n", style="bold yellow")
exports_text.append(" • Run: tito test --module setup\n", style="white")
exports_text.append(" • Or: tito test --all\n", style="white")
console.print(Panel(exports_text, title="Export Summary", border_style="bright_green"))
# Show detailed export information
self._show_export_details(console, args.module if hasattr(args, 'module') else None)
else:
error_msg = result.stderr.strip() if result.stderr else "Unknown error"
@@ -84,7 +135,7 @@ class ExportCommand(BaseCommand):
help_text.append(" • Missing #| default_exp directive in notebook\n", style="white")
help_text.append(" • Syntax errors in exported code\n", style="white")
help_text.append(" • Missing settings.ini configuration\n", style="white")
help_text.append("\n🔧 Run 'tito doctor' for detailed diagnosis", style="cyan")
help_text.append("\n🔧 Run 'tito system doctor' for detailed diagnosis", style="cyan")
console.print(Panel(help_text, title="Troubleshooting", border_style="yellow"))

View File

@@ -5,6 +5,7 @@ Status command for TinyTorch CLI: checks status of all modules in modules/ direc
import subprocess
import sys
import yaml
import re
from argparse import ArgumentParser, Namespace
from pathlib import Path
from rich.panel import Panel
@@ -26,6 +27,27 @@ class StatusCommand(BaseCommand):
parser.add_argument("--details", action="store_true", help="Show detailed file structure")
parser.add_argument("--metadata", action="store_true", help="Show module metadata information")
def _get_export_target(self, module_path: Path) -> str:
"""
Read the actual export target from the dev file's #| default_exp directive.
Same logic as the export command.
"""
dev_file = module_path / f"{module_path.name}_dev.py"
if not dev_file.exists():
return "unknown"
try:
with open(dev_file, 'r', encoding='utf-8') as f:
content = f.read()
# Look for #| default_exp directive
match = re.search(r'#\|\s*default_exp\s+([^\n\r]+)', content)
if match:
return match.group(1).strip()
except Exception:
pass
return "unknown"
def run(self, args: Namespace) -> int:
console = self.console
@@ -59,7 +81,7 @@ class StatusCommand(BaseCommand):
if args.metadata:
status_table.add_column("Export", width=15, justify="center")
status_table.add_column("Components", width=15, justify="center")
status_table.add_column("Prerequisites", width=15, justify="center")
# Check each module
modules_status = []
@@ -80,9 +102,14 @@ class StatusCommand(BaseCommand):
# Add metadata columns if requested
if args.metadata:
metadata = status.get('metadata', {})
row.append(metadata.get('exports_to', 'unknown'))
components = metadata.get('components', [])
row.append(f"{len(components)} items" if components else 'none')
# Get export target from dev file (source of truth)
export_target = self._get_export_target(module_dir)
row.append(export_target if export_target != "unknown" else 'unknown')
# Show prerequisites from dependencies
deps = metadata.get('dependencies', {})
prereqs = deps.get('prerequisites', [])
row.append(', '.join(prereqs) if prereqs else 'none')
status_table.add_row(*row)
@@ -222,9 +249,11 @@ class StatusCommand(BaseCommand):
if metadata.get('description'):
console.print(f"📝 {metadata['description']}")
# Export info (no longer showing static status)
if metadata.get('exports_to'):
console.print(f"📦 Exports to: {metadata['exports_to']}")
# Export info (read from dev file - source of truth)
module_path = Path(f"modules/{module_name}")
export_target = self._get_export_target(module_path)
if export_target != "unknown":
console.print(f"📦 Exports to: {export_target}")
# Dependencies
if metadata.get('dependencies'):