""" End-to-End User Journey Tests for TinyTorch These tests simulate the complete student experience: 1. Fresh start (setup) 2. Module workflow (start → work → complete) 3. Progress tracking 4. Milestone unlocking Run with: pytest tests/e2e/test_user_journey.py -v Categories: -k quick # Fast CLI verification (~30s) -k module_flow # Module workflow tests (~2min) -k full_journey # Complete journey test (~10min) """ import pytest import subprocess import sys import json import shutil import tempfile from pathlib import Path from typing import Optional, Tuple # Project root PROJECT_ROOT = Path(__file__).parent.parent.parent def run_tito(args: list, cwd: Optional[Path] = None, timeout: int = 60) -> Tuple[int, str, str]: """Run a tito command and return (exit_code, stdout, stderr).""" import os cmd = [sys.executable, "-m", "tito.main"] + args env = os.environ.copy() env["TITO_ALLOW_SYSTEM"] = "1" # Allow running outside venv for tests result = subprocess.run( cmd, cwd=cwd or PROJECT_ROOT, capture_output=True, text=True, timeout=timeout, env=env ) return result.returncode, result.stdout, result.stderr def run_python_script(script_path: Path, timeout: int = 120) -> Tuple[int, str, str]: """Run a Python script and return (exit_code, stdout, stderr).""" result = subprocess.run( [sys.executable, str(script_path)], cwd=PROJECT_ROOT, capture_output=True, text=True, timeout=timeout ) return result.returncode, result.stdout, result.stderr class TestQuickVerification: """Quick tests to verify CLI and structure (~30 seconds total).""" @pytest.mark.quick def test_tito_bare_command_works(self): """Bare 'tito' shows welcome screen.""" code, stdout, stderr = run_tito([]) assert code == 0, f"Bare tito failed: {stderr}" assert "Welcome" in stdout or "Quick Start" in stdout @pytest.mark.quick def test_tito_help_works(self): """'tito --help' shows help.""" code, stdout, stderr = run_tito(["--help"]) assert code == 0, f"tito --help failed: {stderr}" assert "usage" in stdout.lower() or "COMMAND" in stdout @pytest.mark.quick def test_tito_version_works(self): """'tito --version' shows version.""" code, stdout, stderr = run_tito(["--version"]) assert code == 0 assert "Tiny" in stdout or "CLI" in stdout @pytest.mark.quick def test_module_command_help(self): """'tito module' shows module help.""" code, stdout, stderr = run_tito(["module"]) assert code == 0 # Should show module subcommands assert "start" in stdout or "complete" in stdout @pytest.mark.quick def test_milestone_command_help(self): """'tito milestone' shows milestone help.""" code, stdout, stderr = run_tito(["milestone"]) assert code == 0 # Should show milestone subcommands assert "list" in stdout or "run" in stdout or "status" in stdout @pytest.mark.quick def test_module_status_works(self): """'tito module status' runs without error.""" code, stdout, stderr = run_tito(["module", "status"]) assert code == 0, f"module status failed: {stderr}" @pytest.mark.quick def test_system_info_works(self): """'tito system info' runs without error.""" code, stdout, stderr = run_tito(["system", "info"]) assert code == 0, f"system info failed: {stderr}" @pytest.mark.quick def test_milestone_list_works(self): """'tito milestones list' shows available milestones.""" code, stdout, stderr = run_tito(["milestone", "list", "--simple"]) assert code == 0, f"milestone list failed: {stderr}" # Should show milestone names assert "Perceptron" in stdout or "1958" in stdout @pytest.mark.quick def test_modules_directory_exists(self): """Modules directory structure exists.""" modules_dir = PROJECT_ROOT / "modules" assert modules_dir.exists(), "modules/ directory missing" # Check first few modules exist for num in ["01", "02", "03"]: module_dirs = list(modules_dir.glob(f"{num}_*")) assert len(module_dirs) > 0, f"Module {num} directory missing" @pytest.mark.quick def test_milestones_directory_exists(self): """Milestones directory structure exists.""" milestones_dir = PROJECT_ROOT / "milestones" assert milestones_dir.exists(), "milestones/ directory missing" # Check milestone directories assert (milestones_dir / "01_1958_perceptron").exists(), "Milestone 01 missing" @pytest.mark.quick def test_tinytorch_package_importable(self): """TinyTorch package can be imported.""" code, stdout, stderr = subprocess.run( [sys.executable, "-c", "import tinytorch; print('OK')"], cwd=PROJECT_ROOT, capture_output=True, text=True ).returncode, "", "" result = subprocess.run( [sys.executable, "-c", "import tinytorch; print('OK')"], cwd=PROJECT_ROOT, capture_output=True, text=True ) assert result.returncode == 0, f"Cannot import tinytorch: {result.stderr}" assert "OK" in result.stdout class TestModuleFlow: """Test module workflow: start → complete → progress tracking.""" @pytest.fixture(autouse=True) def backup_progress(self): """Backup and restore progress.json around tests.""" progress_file = PROJECT_ROOT / "progress.json" backup_file = PROJECT_ROOT / "progress.json.e2e_backup" # Backup existing progress if progress_file.exists(): shutil.copy(progress_file, backup_file) yield # Restore original progress if backup_file.exists(): shutil.copy(backup_file, progress_file) backup_file.unlink() elif progress_file.exists(): # If there was no original, remove the test progress # Actually, keep it - don't delete real progress pass @pytest.mark.module_flow def test_module_01_start_works(self): """'tito module start 01' works (first module, no prerequisites).""" # Note: This opens Jupyter, but should not block # We test the command doesn't error on already-started modules code, stdout, stderr = run_tito(["module", "status"]) assert code == 0 @pytest.mark.module_flow def test_module_02_start_responds(self): """'tito module start 02' gives a meaningful response about module state.""" # Note: This test checks that the command responds appropriately. # If module 01 is not completed, it should show "Locked" or prerequisites. # If module 01 is completed (from previous tests), it should show "Unlocked". code, stdout, stderr = run_tito(["module", "start", "02"]) combined = stdout + stderr # Should show either locked (needs prereqs) or unlocked (ready to start) assert "Locked" in combined or "Unlocked" in combined or "Module 02" in combined or code == 0 @pytest.mark.module_flow def test_module_complete_runs_tests(self): """'tito module complete 01 --skip-export' runs tests.""" # This tests that the complete command works (skip export to be faster) code, stdout, stderr = run_tito( ["module", "complete", "01", "--skip-export"], timeout=120 # Tests may take a while ) # Check that tests ran (may pass or fail depending on state) combined = stdout + stderr assert "Test" in combined or "test" in combined or code in [0, 1] @pytest.mark.module_flow def test_progress_tracking_persists(self): """Progress is saved and persisted across commands.""" progress_file = PROJECT_ROOT / "progress.json" # Set a known state progress_file.write_text(json.dumps({ "started_modules": ["01"], "completed_modules": [], "last_worked": "01" })) # Run status command code, stdout, stderr = run_tito(["module", "status"]) assert code == 0 # Check progress file still exists and has data assert progress_file.exists() data = json.loads(progress_file.read_text()) assert "started_modules" in data @pytest.mark.module_flow def test_module_test_command_works(self): """'tito module test 01' runs module tests.""" code, stdout, stderr = run_tito( ["module", "test", "01"], timeout=120 ) # Should run tests (may pass or fail) combined = stdout + stderr # Test command should produce some output assert len(combined) > 0 class TestMilestoneFlow: """Test milestone workflow: prerequisites → run → completion tracking.""" @pytest.mark.milestone_flow def test_milestone_list_shows_all(self): """Milestone list shows all available milestones.""" code, stdout, stderr = run_tito(["milestone", "list"]) assert code == 0 # Check for expected milestones expected = ["Perceptron", "XOR", "MLP", "CNN", "Transformer"] found = sum(1 for m in expected if m in stdout) assert found >= 3, f"Expected milestones not shown. Got: {stdout}" @pytest.mark.milestone_flow def test_milestone_info_works(self): """'tito milestones info 01' shows milestone details.""" code, stdout, stderr = run_tito(["milestone", "info", "01"]) assert code == 0 assert "Perceptron" in stdout or "1958" in stdout @pytest.mark.milestone_flow def test_milestone_status_works(self): """'tito milestones status' shows progress.""" code, stdout, stderr = run_tito(["milestone", "status"]) assert code == 0 @pytest.mark.milestone_flow def test_milestone_01_script_exists(self): """Milestone 01 script file exists.""" script_path = PROJECT_ROOT / "milestones" / "01_1958_perceptron" / "01_rosenblatt_forward.py" assert script_path.exists(), f"Milestone script missing: {script_path}" @pytest.mark.milestone_flow def test_milestone_run_checks_prerequisites(self): """'tito milestone run' checks prerequisites before running.""" # Create clean state with no completed modules tito_dir = PROJECT_ROOT / ".tito" tito_dir.mkdir(exist_ok=True) progress_file = tito_dir / "progress.json" progress_file.write_text(json.dumps({ "completed_modules": [] })) # Try to run milestone 03 (requires many modules) code, stdout, stderr = run_tito(["milestone", "run", "03", "--skip-checks"], timeout=5) # With --skip-checks it might try to run; without it should check prereqs # Either way, the command should not crash assert code in [0, 1, 130] # 130 = user interrupt class TestFullJourney: """Complete end-to-end journey test (slow, thorough).""" @pytest.mark.full_journey @pytest.mark.slow def test_complete_module_01_journey(self): """ Test complete journey for module 01: 1. Start module 2. Complete module (with tests) 3. Verify progress updated 4. Verify export worked """ # Step 1: Check initial state code, stdout, stderr = run_tito(["module", "status"]) assert code == 0 # Step 2: Test the module code, stdout, stderr = run_tito( ["module", "test", "01"], timeout=180 ) # Tests should run (may pass or fail based on implementation) combined = stdout + stderr assert "test" in combined.lower() or "Test" in combined # Step 3: Verify tinytorch imports work result = subprocess.run( [sys.executable, "-c", "from tinytorch import Tensor; print('OK')"], cwd=PROJECT_ROOT, capture_output=True, text=True ) # This tests that the package structure is correct # If Tensor is not exported, that's a test failure assert result.returncode == 0, ( f"Tensor not exported from tinytorch. " f"Run 'tito module complete 01' first. Error: {result.stderr}" ) @pytest.mark.full_journey @pytest.mark.slow def test_milestone_01_runs_successfully(self): """ Test that milestone 01 can run successfully. Requires: Module 01-08 completed and exported. """ # Check if prerequisite modules are available result = subprocess.run( [sys.executable, "-c", """ from tinytorch import Tensor, ReLU, Linear print('OK') """], cwd=PROJECT_ROOT, capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, ( f"Required modules (Tensor, ReLU, Linear) not exported from tinytorch. " f"Complete modules 01-08 first. Error: {result.stderr}" ) # Run milestone 01 with skip-checks (we verified prereqs above) script_path = PROJECT_ROOT / "milestones" / "01_1958_perceptron" / "01_rosenblatt_forward.py" assert script_path.exists(), f"Milestone script not found at {script_path}" code, stdout, stderr = run_python_script(script_path, timeout=120) # Should complete successfully or with informative error combined = stdout + stderr assert code == 0 or "Error" in combined, f"Milestone failed unexpectedly: {combined}" class TestErrorHandling: """Test that errors are handled gracefully.""" @pytest.mark.quick def test_invalid_command_shows_error(self): """Invalid commands show helpful error messages.""" code, stdout, stderr = run_tito(["nonexistent_command"]) assert code != 0 combined = stdout + stderr # Check for "not found" or "not a valid" (the actual error message text) assert "not found" in combined.lower() or "not a valid" in combined.lower() @pytest.mark.quick def test_invalid_module_number_handled(self): """Invalid module numbers are handled gracefully.""" code, stdout, stderr = run_tito(["module", "start", "99"]) assert code != 0 combined = stdout + stderr assert "not found" in combined.lower() or "invalid" in combined.lower() or "99" in combined @pytest.mark.quick def test_invalid_milestone_handled(self): """Invalid milestone IDs are handled gracefully.""" code, stdout, stderr = run_tito(["milestone", "info", "99"]) assert code != 0 combined = stdout + stderr assert "invalid" in combined.lower() or "not found" in combined.lower() class TestInstallationPaths: """Test different installation/usage paths.""" @pytest.mark.quick def test_src_directory_exists(self): """Source directory for development exists.""" src_dir = PROJECT_ROOT / "src" assert src_dir.exists(), "src/ directory missing" @pytest.mark.quick def test_pyproject_exists(self): """pyproject.toml exists for pip installation.""" pyproject = PROJECT_ROOT / "pyproject.toml" assert pyproject.exists(), "pyproject.toml missing" @pytest.mark.quick def test_requirements_exists(self): """requirements.txt exists for dependency installation.""" requirements = PROJECT_ROOT / "requirements.txt" assert requirements.exists(), "requirements.txt missing" if __name__ == "__main__": pytest.main([__file__, "-v"])