Add submission command to tito community CLI

Implement tito community submit command for Module 20 capstone submissions with schema validation, metrics display, and error handling. Shows baseline and optimized model metrics with improvement calculations. Includes Coming soon message for future leaderboard integration.
This commit is contained in:
Vijay Janapa Reddi
2025-11-29 18:42:51 -05:00
parent 6f9a9d156d
commit deea02f3b6

View File

@@ -123,6 +123,16 @@ class CommunityCommand(BaseCommand):
'compete',
help='Join competitions and challenges (opens in browser)'
)
# Submit command
submit_parser = subparsers.add_parser(
'submit',
help='Submit your benchmark results to the leaderboard'
)
submit_parser.add_argument(
'submission_file',
help='Path to submission JSON file (e.g., submission.json)'
)
def run(self, args: Namespace) -> int:
"""Execute community command."""
@@ -144,6 +154,8 @@ class CommunityCommand(BaseCommand):
return self._open_leaderboard(args)
elif args.community_command == 'compete':
return self._open_compete(args)
elif args.community_command == 'submit':
return self._submit_benchmark(args)
else:
self.console.print(f"[red]Unknown community command: {args.community_command}[/red]")
return 1
@@ -706,3 +718,142 @@ class CommunityCommand(BaseCommand):
self.console.print(f"[dim]Please visit: {compete_url}[/dim]")
return 0
def _submit_benchmark(self, args: Namespace) -> int:
"""Submit benchmark results to the leaderboard."""
console = self.console
submission_file = Path(args.submission_file)
# Check if file exists
if not submission_file.exists():
console.print(Panel(
f"[red]❌ Submission file not found[/red]\n\n"
f"[dim]Path: {submission_file}[/dim]\n\n"
f"Make sure you've generated a submission file first:\n"
f" • Run Module 20 (Capstone)\n"
f" • Generate submission: [cyan]save_submission(submission, 'submission.json')[/cyan]",
title="File Not Found",
border_style="red"
))
return 1
# Load and validate submission
try:
with open(submission_file, 'r') as f:
submission = json.load(f)
except json.JSONDecodeError as e:
console.print(Panel(
f"[red]❌ Invalid JSON file[/red]\n\n"
f"[dim]Error: {e}[/dim]\n\n"
f"Make sure your file contains valid JSON.",
title="Invalid JSON",
border_style="red"
))
return 1
# Validate submission schema
console.print(Panel(
"[cyan]🔍 Validating submission...[/cyan]",
title="Validation",
border_style="cyan"
))
try:
# Import validation function from Module 20
import sys
sys.path.insert(0, str(self.config.project_root / "src" / "20_capstone"))
from importlib import import_module
capstone = import_module("20_capstone")
# Validate
capstone.validate_submission_schema(submission)
# Show validation success
console.print()
console.print("[green]✅ Submission validated successfully![/green]")
console.print()
# Display submission summary
table = Table(title="Submission Summary", show_header=True, box=None)
table.add_column("Field", style="cyan", width=25)
table.add_column("Value", style="green")
table.add_row("TinyTorch Version", submission.get('tinytorch_version', 'N/A'))
table.add_row("Submission Type", submission.get('submission_type', 'N/A'))
table.add_row("Timestamp", submission.get('timestamp', 'N/A'))
baseline = submission.get('baseline', {})
metrics = baseline.get('metrics', {})
table.add_row("", "")
table.add_row("[bold]Baseline Model[/bold]", "")
table.add_row(" Model Name", baseline.get('model_name', 'N/A'))
table.add_row(" Parameters", f"{metrics.get('parameter_count', 0):,}")
table.add_row(" Size", f"{metrics.get('model_size_mb', 0):.2f} MB")
table.add_row(" Accuracy", f"{metrics.get('accuracy', 0)*100:.1f}%")
table.add_row(" Latency", f"{metrics.get('latency_ms_mean', 0):.2f} ms")
if 'optimized' in submission:
optimized = submission['optimized']
opt_metrics = optimized.get('metrics', {})
improvements = submission.get('improvements', {})
table.add_row("", "")
table.add_row("[bold]Optimized Model[/bold]", "")
table.add_row(" Model Name", optimized.get('model_name', 'N/A'))
table.add_row(" Parameters", f"{opt_metrics.get('parameter_count', 0):,}")
table.add_row(" Size", f"{opt_metrics.get('model_size_mb', 0):.2f} MB")
table.add_row(" Accuracy", f"{opt_metrics.get('accuracy', 0)*100:.1f}%")
table.add_row(" Latency", f"{opt_metrics.get('latency_ms_mean', 0):.2f} ms")
table.add_row("", "")
table.add_row("[bold]Improvements[/bold]", "")
table.add_row(" Speedup", f"{improvements.get('speedup', 0):.2f}x")
table.add_row(" Compression", f"{improvements.get('compression_ratio', 0):.2f}x")
table.add_row(" Accuracy Delta", f"{improvements.get('accuracy_delta', 0)*100:+.1f}%")
console.print()
console.print(table)
console.print()
except AssertionError as e:
console.print(Panel(
f"[red]❌ Validation failed[/red]\n\n"
f"[dim]Error: {e}[/dim]\n\n"
f"Check your submission matches the required schema:\n"
f" • Required fields: tinytorch_version, submission_type, timestamp, system_info, baseline\n"
f" • Baseline metrics: parameter_count, model_size_mb, accuracy, latency_ms_mean\n"
f" • Accuracy must be in [0, 1]\n"
f" • All counts/sizes/latencies must be positive",
title="Validation Failed",
border_style="red"
))
return 1
except Exception as e:
console.print(Panel(
f"[red]❌ Validation error[/red]\n\n"
f"[dim]Error: {e}[/dim]",
title="Error",
border_style="red"
))
return 1
# Show "coming soon" message for actual submission
console.print(Panel(
"[bold yellow]🚧 Submission to Leaderboard - Coming Soon![/bold yellow]\n\n"
"[dim]Your submission has been validated successfully![/dim]\n\n"
"The TinyTorch community leaderboard is currently under development.\n"
"Soon you'll be able to:\n"
" • Submit your optimized models to the global leaderboard\n"
" • Compare your results with other learners worldwide\n"
" • Participate in TinyML optimization challenges\n"
" • Earn badges and achievements\n\n"
f"[green]✓[/green] Your submission file is ready: [cyan]{submission_file}[/cyan]\n"
"[dim]Keep this file - you'll be able to submit it once the leaderboard launches![/dim]\n\n"
"In the meantime:\n"
" • View community: [cyan]tito community leaderboard[/cyan]\n"
" • Join challenges: [cyan]tito community compete[/cyan]",
title="🎯 Leaderboard Coming Soon",
border_style="yellow"
))
return 0