mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-05-01 18:19:18 -05:00
feat: integrate TinyTorch into MLSysBook repository
TinyTorch educational deep learning framework now lives at tinytorch/
Structure:
- tinytorch/src/ - Source modules (single source of truth)
- tinytorch/tito/ - CLI tool
- tinytorch/tests/ - Test suite
- tinytorch/site/ - Jupyter Book website
- tinytorch/milestones/ - Historical ML implementations
- tinytorch/datasets/ - Educational datasets (tinydigits, tinytalks)
- tinytorch/assignments/ - NBGrader assignments
- tinytorch/instructor/ - Teaching materials
Workflows (with tinytorch- prefix):
- tinytorch-ci.yml - CI/CD pipeline
- tinytorch-publish-dev.yml - Dev site deployment
- tinytorch-publish-live.yml - Live site deployment
- tinytorch-build-pdf.yml - PDF generation
- tinytorch-release-check.yml - Release validation
Repository Variables added:
- TINYTORCH_ROOT = tinytorch
- TINYTORCH_SRC = tinytorch/src
- TINYTORCH_SITE = tinytorch/site
- TINYTORCH_TESTS = tinytorch/tests
All workflows use \${{ vars.TINYTORCH_* }} for path configuration.
Note: tinytorch/site/_static/favicon.svg kept as SVG (valid for favicons)
This commit is contained in:
Vijay Janapa Reddi
247
tinytorch/tests/cli/test_cli_help_consistency.py
Normal file
247
tinytorch/tests/cli/test_cli_help_consistency.py
Normal file
@@ -0,0 +1,247 @@
|
||||
"""
|
||||
CLI Help Consistency Tests - Validate help text is consistent and complete
|
||||
|
||||
This test suite ensures:
|
||||
1. Help text uses consistent formatting and terminology
|
||||
2. All commands document their purpose clearly
|
||||
3. Examples are provided where helpful
|
||||
4. No broken references or outdated commands in help text
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add tito to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from tito.main import TinyTorchCLI
|
||||
|
||||
|
||||
class TestHelpConsistency:
|
||||
"""Test that help text is consistent across commands."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.project_root = Path(__file__).parent.parent.parent
|
||||
|
||||
def get_command_help(self, *args):
|
||||
"""Get help output for a command."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'] + list(args) + ['-h'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
return result.stdout + result.stderr
|
||||
|
||||
def test_all_command_helps_mention_tito(self):
|
||||
"""Verify all help texts mention 'tito' command."""
|
||||
commands = ['setup', 'system', 'module', 'checkpoint', 'milestones']
|
||||
|
||||
for cmd in commands:
|
||||
help_text = self.get_command_help(cmd)
|
||||
assert 'tito' in help_text.lower(), (
|
||||
f"Command '{cmd}' help doesn't mention 'tito'"
|
||||
)
|
||||
|
||||
def test_bare_tito_and_help_are_different(self):
|
||||
"""Verify bare 'tito' and 'tito -h' show different but related content."""
|
||||
# Get bare tito output
|
||||
bare_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
# Get help output
|
||||
help_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main', '-h'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
bare_output = bare_result.stdout
|
||||
help_output = help_result.stdout
|
||||
|
||||
# Should both mention key commands
|
||||
for cmd in ['module', 'milestones', 'setup']:
|
||||
assert cmd in bare_output, f"'{cmd}' missing from bare tito"
|
||||
assert cmd in help_output, f"'{cmd}' missing from tito -h"
|
||||
|
||||
# Bare should have welcome/logo
|
||||
assert any(word in bare_output for word in ['Welcome', 'TORCH', 'Quick Start']), \
|
||||
"Bare tito should show welcome screen"
|
||||
|
||||
# Help should have usage
|
||||
assert 'usage:' in help_output.lower(), "tito -h should show usage"
|
||||
|
||||
def test_no_references_to_removed_commands(self):
|
||||
"""Verify help doesn't reference commands that don't exist."""
|
||||
cli = TinyTorchCLI()
|
||||
registered_commands = set(cli.commands.keys())
|
||||
|
||||
# Get main help
|
||||
help_text = self.get_command_help()
|
||||
|
||||
# Common command-like words that might be false positives
|
||||
ignore_words = {
|
||||
'command', 'commands', 'option', 'options', 'argument',
|
||||
'arguments', 'help', 'version', 'verbose', 'color',
|
||||
'git', 'python', 'pip', 'jupyter', 'pytest', 'run',
|
||||
'build', 'install', 'create', 'delete', 'update',
|
||||
'show', 'list', 'view', 'open', 'close', 'start',
|
||||
'stop', 'export', 'import', 'output', 'input'
|
||||
}
|
||||
|
||||
# Extract words that look like commands (lowercase alphanumeric)
|
||||
import re
|
||||
potential_commands = set(re.findall(r'\b[a-z][a-z_-]*[a-z]\b', help_text.lower()))
|
||||
|
||||
# Filter to reasonable command-like words
|
||||
suspicious = potential_commands - registered_commands - ignore_words
|
||||
|
||||
# These are expected in help text but not commands
|
||||
expected_non_commands = {
|
||||
'system', 'module', 'first', 'time', 'complete', 'resume',
|
||||
'status', 'progress', 'journey', 'profile', 'timeline',
|
||||
'tinytorch', 'tiny', 'torch', 'cli', 'developer', 'student',
|
||||
'workflow', 'tracking', 'capabilities', 'achievements'
|
||||
}
|
||||
|
||||
truly_suspicious = suspicious - expected_non_commands
|
||||
|
||||
# Just warn if we find something, don't fail
|
||||
# (This test is informational)
|
||||
if truly_suspicious:
|
||||
print(f"\nInfo: Found potential command references: {sorted(truly_suspicious)[:10]}")
|
||||
|
||||
|
||||
class TestWelcomeScreen:
|
||||
"""Test the welcome screen shown by bare 'tito' command."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.project_root = Path(__file__).parent.parent.parent
|
||||
|
||||
def test_welcome_screen_shows_quick_start(self):
|
||||
"""Verify welcome screen has quick start section."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
assert 'Quick Start' in output or 'Getting Started' in output, \
|
||||
"Welcome screen should have Quick Start section"
|
||||
|
||||
def test_welcome_screen_shows_command_groups(self):
|
||||
"""Verify welcome screen organizes commands into groups."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
assert 'Quick Start:' in output or 'Commands:' in output, \
|
||||
"Welcome screen should show quick start or commands"
|
||||
|
||||
def test_welcome_screen_has_examples(self):
|
||||
"""Verify welcome screen shows example commands."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
|
||||
# Should have at least one example command
|
||||
assert 'tito setup' in output or 'tito module' in output, \
|
||||
"Welcome screen should show example commands"
|
||||
|
||||
|
||||
class TestCommandDocumentation:
|
||||
"""Test that commands are properly documented."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.project_root = Path(__file__).parent.parent.parent
|
||||
self.cli = TinyTorchCLI()
|
||||
|
||||
def test_all_registered_commands_in_welcome_or_help(self):
|
||||
"""Verify all registered commands appear in welcome screen or help."""
|
||||
# Get welcome screen
|
||||
welcome_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
# Get help
|
||||
help_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main', '-h'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
combined = welcome_result.stdout + help_result.stdout
|
||||
|
||||
missing = []
|
||||
for cmd in self.cli.commands.keys():
|
||||
if cmd not in combined:
|
||||
missing.append(cmd)
|
||||
|
||||
assert not missing, (
|
||||
f"Commands registered but not documented: {missing}\n"
|
||||
f"Add them to welcome screen or help text"
|
||||
)
|
||||
|
||||
def test_milestones_properly_documented(self):
|
||||
"""Specifically test that milestones command is documented."""
|
||||
# This addresses the user's concern about progress tracking
|
||||
welcome_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
help_result = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main', '-h'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
combined = welcome_result.stdout + help_result.stdout
|
||||
|
||||
assert 'milestones' in combined.lower(), \
|
||||
"milestones command should be documented"
|
||||
|
||||
# Check it has a description
|
||||
milestones_help = subprocess.run(
|
||||
[sys.executable, '-m', 'tito.main', 'milestones', '-h'],
|
||||
cwd=self.project_root,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
assert 'progress' in milestones_help.stdout.lower() or \
|
||||
'track' in milestones_help.stdout.lower() or \
|
||||
'milestone' in milestones_help.stdout.lower(), \
|
||||
"milestones help should explain its purpose"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
Reference in New Issue
Block a user