mirror of
https://github.com/MLSysBook/TinyTorch.git
synced 2026-03-11 22:33:36 -05:00
Add submission command to tito community CLI
Implement tito community submit command for Module 20 capstone submissions with schema validation, metrics display, and error handling. Shows baseline and optimized model metrics with improvement calculations. Includes Coming soon message for future leaderboard integration.
This commit is contained in:
@@ -123,6 +123,16 @@ class CommunityCommand(BaseCommand):
|
||||
'compete',
|
||||
help='Join competitions and challenges (opens in browser)'
|
||||
)
|
||||
|
||||
# Submit command
|
||||
submit_parser = subparsers.add_parser(
|
||||
'submit',
|
||||
help='Submit your benchmark results to the leaderboard'
|
||||
)
|
||||
submit_parser.add_argument(
|
||||
'submission_file',
|
||||
help='Path to submission JSON file (e.g., submission.json)'
|
||||
)
|
||||
|
||||
def run(self, args: Namespace) -> int:
|
||||
"""Execute community command."""
|
||||
@@ -144,6 +154,8 @@ class CommunityCommand(BaseCommand):
|
||||
return self._open_leaderboard(args)
|
||||
elif args.community_command == 'compete':
|
||||
return self._open_compete(args)
|
||||
elif args.community_command == 'submit':
|
||||
return self._submit_benchmark(args)
|
||||
else:
|
||||
self.console.print(f"[red]Unknown community command: {args.community_command}[/red]")
|
||||
return 1
|
||||
@@ -706,3 +718,142 @@ class CommunityCommand(BaseCommand):
|
||||
self.console.print(f"[dim]Please visit: {compete_url}[/dim]")
|
||||
|
||||
return 0
|
||||
|
||||
def _submit_benchmark(self, args: Namespace) -> int:
|
||||
"""Submit benchmark results to the leaderboard."""
|
||||
console = self.console
|
||||
submission_file = Path(args.submission_file)
|
||||
|
||||
# Check if file exists
|
||||
if not submission_file.exists():
|
||||
console.print(Panel(
|
||||
f"[red]❌ Submission file not found[/red]\n\n"
|
||||
f"[dim]Path: {submission_file}[/dim]\n\n"
|
||||
f"Make sure you've generated a submission file first:\n"
|
||||
f" • Run Module 20 (Capstone)\n"
|
||||
f" • Generate submission: [cyan]save_submission(submission, 'submission.json')[/cyan]",
|
||||
title="File Not Found",
|
||||
border_style="red"
|
||||
))
|
||||
return 1
|
||||
|
||||
# Load and validate submission
|
||||
try:
|
||||
with open(submission_file, 'r') as f:
|
||||
submission = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
console.print(Panel(
|
||||
f"[red]❌ Invalid JSON file[/red]\n\n"
|
||||
f"[dim]Error: {e}[/dim]\n\n"
|
||||
f"Make sure your file contains valid JSON.",
|
||||
title="Invalid JSON",
|
||||
border_style="red"
|
||||
))
|
||||
return 1
|
||||
|
||||
# Validate submission schema
|
||||
console.print(Panel(
|
||||
"[cyan]🔍 Validating submission...[/cyan]",
|
||||
title="Validation",
|
||||
border_style="cyan"
|
||||
))
|
||||
|
||||
try:
|
||||
# Import validation function from Module 20
|
||||
import sys
|
||||
sys.path.insert(0, str(self.config.project_root / "src" / "20_capstone"))
|
||||
from importlib import import_module
|
||||
capstone = import_module("20_capstone")
|
||||
|
||||
# Validate
|
||||
capstone.validate_submission_schema(submission)
|
||||
|
||||
# Show validation success
|
||||
console.print()
|
||||
console.print("[green]✅ Submission validated successfully![/green]")
|
||||
console.print()
|
||||
|
||||
# Display submission summary
|
||||
table = Table(title="Submission Summary", show_header=True, box=None)
|
||||
table.add_column("Field", style="cyan", width=25)
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("TinyTorch Version", submission.get('tinytorch_version', 'N/A'))
|
||||
table.add_row("Submission Type", submission.get('submission_type', 'N/A'))
|
||||
table.add_row("Timestamp", submission.get('timestamp', 'N/A'))
|
||||
|
||||
baseline = submission.get('baseline', {})
|
||||
metrics = baseline.get('metrics', {})
|
||||
table.add_row("", "")
|
||||
table.add_row("[bold]Baseline Model[/bold]", "")
|
||||
table.add_row(" Model Name", baseline.get('model_name', 'N/A'))
|
||||
table.add_row(" Parameters", f"{metrics.get('parameter_count', 0):,}")
|
||||
table.add_row(" Size", f"{metrics.get('model_size_mb', 0):.2f} MB")
|
||||
table.add_row(" Accuracy", f"{metrics.get('accuracy', 0)*100:.1f}%")
|
||||
table.add_row(" Latency", f"{metrics.get('latency_ms_mean', 0):.2f} ms")
|
||||
|
||||
if 'optimized' in submission:
|
||||
optimized = submission['optimized']
|
||||
opt_metrics = optimized.get('metrics', {})
|
||||
improvements = submission.get('improvements', {})
|
||||
|
||||
table.add_row("", "")
|
||||
table.add_row("[bold]Optimized Model[/bold]", "")
|
||||
table.add_row(" Model Name", optimized.get('model_name', 'N/A'))
|
||||
table.add_row(" Parameters", f"{opt_metrics.get('parameter_count', 0):,}")
|
||||
table.add_row(" Size", f"{opt_metrics.get('model_size_mb', 0):.2f} MB")
|
||||
table.add_row(" Accuracy", f"{opt_metrics.get('accuracy', 0)*100:.1f}%")
|
||||
table.add_row(" Latency", f"{opt_metrics.get('latency_ms_mean', 0):.2f} ms")
|
||||
|
||||
table.add_row("", "")
|
||||
table.add_row("[bold]Improvements[/bold]", "")
|
||||
table.add_row(" Speedup", f"{improvements.get('speedup', 0):.2f}x")
|
||||
table.add_row(" Compression", f"{improvements.get('compression_ratio', 0):.2f}x")
|
||||
table.add_row(" Accuracy Delta", f"{improvements.get('accuracy_delta', 0)*100:+.1f}%")
|
||||
|
||||
console.print()
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
except AssertionError as e:
|
||||
console.print(Panel(
|
||||
f"[red]❌ Validation failed[/red]\n\n"
|
||||
f"[dim]Error: {e}[/dim]\n\n"
|
||||
f"Check your submission matches the required schema:\n"
|
||||
f" • Required fields: tinytorch_version, submission_type, timestamp, system_info, baseline\n"
|
||||
f" • Baseline metrics: parameter_count, model_size_mb, accuracy, latency_ms_mean\n"
|
||||
f" • Accuracy must be in [0, 1]\n"
|
||||
f" • All counts/sizes/latencies must be positive",
|
||||
title="Validation Failed",
|
||||
border_style="red"
|
||||
))
|
||||
return 1
|
||||
except Exception as e:
|
||||
console.print(Panel(
|
||||
f"[red]❌ Validation error[/red]\n\n"
|
||||
f"[dim]Error: {e}[/dim]",
|
||||
title="Error",
|
||||
border_style="red"
|
||||
))
|
||||
return 1
|
||||
|
||||
# Show "coming soon" message for actual submission
|
||||
console.print(Panel(
|
||||
"[bold yellow]🚧 Submission to Leaderboard - Coming Soon![/bold yellow]\n\n"
|
||||
"[dim]Your submission has been validated successfully![/dim]\n\n"
|
||||
"The TinyTorch community leaderboard is currently under development.\n"
|
||||
"Soon you'll be able to:\n"
|
||||
" • Submit your optimized models to the global leaderboard\n"
|
||||
" • Compare your results with other learners worldwide\n"
|
||||
" • Participate in TinyML optimization challenges\n"
|
||||
" • Earn badges and achievements\n\n"
|
||||
f"[green]✓[/green] Your submission file is ready: [cyan]{submission_file}[/cyan]\n"
|
||||
"[dim]Keep this file - you'll be able to submit it once the leaderboard launches![/dim]\n\n"
|
||||
"In the meantime:\n"
|
||||
" • View community: [cyan]tito community leaderboard[/cyan]\n"
|
||||
" • Join challenges: [cyan]tito community compete[/cyan]",
|
||||
title="🎯 Leaderboard Coming Soon",
|
||||
border_style="yellow"
|
||||
))
|
||||
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user