diff --git a/tinytorch/milestones/02_1969_xor/02_xor_solved.py b/tinytorch/milestones/02_1969_xor/02_xor_solved.py index 5569f85408..25396d6272 100755 --- a/tinytorch/milestones/02_1969_xor/02_xor_solved.py +++ b/tinytorch/milestones/02_1969_xor/02_xor_solved.py @@ -265,7 +265,25 @@ def train_network(model, X, y, epochs=500, lr=0.5): if (epoch + 1) % 100 == 0: live.console.print(f"Epoch {epoch+1:3d}/{epochs} Loss: {loss.data:.4f} Accuracy: {accuracy:.1%}") - console.print("\n[green]āœ… Training Complete - XOR Solved![/green]") + # Only declare success if the network actually converged on XOR. + # Anything below ~95% on a noiseless 4-pattern problem means the + # optimizer landed in a saddle point (classic 75% dead-ReLU symptom). + final_accuracy = history["accuracy"][-1] + if final_accuracy >= 0.95: + console.print("\n[green]āœ… Training Complete - XOR Solved![/green]") + else: + console.print( + f"\n[yellow]āš ļø Training Complete - but only {final_accuracy:.1%} accuracy.[/yellow]" + ) + console.print( + "[yellow] The network did not converge (likely stuck in a saddle point).[/yellow]" + ) + console.print( + "[yellow] Try re-running the milestone - random init can pin a 4-unit hidden[/yellow]" + ) + console.print( + "[yellow] layer at 75% on XOR. See issue #1614.[/yellow]" + ) return history @@ -412,8 +430,13 @@ def main(): # ACT 3: THE EXPERIMENT šŸ”¬ # ═══════════════════════════════════════════════════════════════════════ - # Set seed before model creation to guarantee reproducible 100% convergence - rng = np.random.default_rng(7) # The year backprop was published! + # Re-seed the layers' weight-init RNG to guarantee reproducible 100% + # convergence on XOR. With a 4-unit hidden layer, some random + # initializations land in a "dead ReLU" saddle point that pins accuracy + # at 75% (one of the four XOR cases stuck at pā‰ˆ0.5). 1986 (the year of the + # backprop paper) reliably escapes that saddle. + import tinytorch.core.layers as _layers + _layers.rng = np.random.default_rng(1986) model = XORNetwork(hidden_size=4) initial_preds = model(X) @@ -449,49 +472,88 @@ def main(): final_acc = history["accuracy"][-1] - console.print(Panel.fit( - "[bold green]šŸŽ‰ Success! You Ended the AI Winter![/bold green]\n\n" + # Convergence threshold: noiseless 4-pattern XOR should hit >=95% when + # training succeeds. Below that, the optimizer is stuck in a saddle + # point (the classic 75% dead-ReLU symptom). + XOR_CONVERGENCE_THRESHOLD = 0.95 - f"Final accuracy: [bold]{final_acc:.1%}[/bold] (Perfect XOR solution!)\n\n" + if final_acc >= XOR_CONVERGENCE_THRESHOLD: + console.print(Panel.fit( + "[bold green]šŸŽ‰ Success! You Ended the AI Winter![/bold green]\n\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + f"Final accuracy: [bold]{final_acc:.1%}[/bold] (Perfect XOR solution!)\n\n" - "[bold]šŸ’” What YOU Just Accomplished:[/bold]\n" - " āœ“ Solved the problem that killed AI for 17 years!\n" - " āœ“ Built multi-layer network with YOUR components\n" - " āœ“ Hidden layer learns non-linear features\n" - " āœ“ Backprop through multiple layers works perfectly!\n" - " āœ“ Proved that deep networks CAN work!\n\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + "[bold]šŸ’” What YOU Just Accomplished:[/bold]\n" + " āœ“ Solved the problem that killed AI for 17 years!\n" + " āœ“ Built multi-layer network with YOUR components\n" + " āœ“ Hidden layer learns non-linear features\n" + " āœ“ Backprop through multiple layers works perfectly!\n" + " āœ“ Proved that deep networks CAN work!\n\n" - "[bold]šŸŽ“ Why This Matters:[/bold]\n" - " This ENDED the 17-year AI Winter!\n" - " [bold red]1969:[/bold red] XOR crisis → single layers fail\n" - " [bold yellow]1970-1986:[/bold yellow] AI Winter - research funding dries up\n" - " [bold green]1986:[/bold green] Backprop + hidden layers solve it\n" - " [bold cyan]TODAY:[/bold cyan] YOU recreated this breakthrough!\n\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + "[bold]šŸŽ“ Why This Matters:[/bold]\n" + " This ENDED the 17-year AI Winter!\n" + " [bold red]1969:[/bold red] XOR crisis → single layers fail\n" + " [bold yellow]1970-1986:[/bold yellow] AI Winter - research funding dries up\n" + " [bold green]1986:[/bold green] Backprop + hidden layers solve it\n" + " [bold cyan]TODAY:[/bold cyan] YOU recreated this breakthrough!\n\n" - "[bold]šŸ“Œ The Key Insight:[/bold]\n" - " Hidden layers are the KEY to modern AI.\n" - " They learn new features that make problems solvable.\n" - " Every deep network (GPT, AlphaGo, etc.) uses this pattern!\n" - " \n" - " [green]Breakthrough:[/green] Non-linear activation functions (ReLU)\n" - " enable networks to learn non-linear decision boundaries.\n\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + "[bold]šŸ“Œ The Key Insight:[/bold]\n" + " Hidden layers are the KEY to modern AI.\n" + " They learn new features that make problems solvable.\n" + " Every deep network (GPT, AlphaGo, etc.) uses this pattern!\n" + " \n" + " [green]Breakthrough:[/green] Non-linear activation functions (ReLU)\n" + " enable networks to learn non-linear decision boundaries.\n\n" - "[bold]šŸš€ What's Next:[/bold]\n" - "[dim]Milestone 03 applies this to digit images with YOUR DataLoader!\n" - "Train on handwritten digits and see modern ML in action![/dim]", + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" - title="🌟 1986 AI Renaissance Complete", - border_style="green", - box=box.DOUBLE - )) + "[bold]šŸš€ What's Next:[/bold]\n" + "[dim]Milestone 03 applies this to digit images with YOUR DataLoader!\n" + "Train on handwritten digits and see modern ML in action![/dim]", + + title="🌟 1986 AI Renaissance Complete", + border_style="green", + box=box.DOUBLE + )) + else: + # Training did not converge - tell the student honestly and suggest + # how to recover, instead of falsely advertising XOR as solved. + console.print(Panel.fit( + "[bold yellow]āš ļø Training Did Not Converge[/bold yellow]\n\n" + + f"Final accuracy: [bold]{final_acc:.1%}[/bold] " + f"(below the {XOR_CONVERGENCE_THRESHOLD:.0%} convergence threshold)\n\n" + + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + + "[bold]šŸ” What Likely Happened:[/bold]\n" + " A 4-unit hidden layer trained on XOR can land in a\n" + " [italic]dead-ReLU saddle point[/italic] where one of the four XOR\n" + " cases stays pinned at probability ā‰ˆ 0.5 forever.\n" + " That gives the classic 75% accuracy plateau - the network\n" + " has [bold]not[/bold] solved XOR.\n\n" + + "[bold]šŸ› ļø How to Recover:[/bold]\n" + " • Re-run the milestone - a different random init usually escapes\n" + " • Check your ReLU / Linear / autograd implementations\n" + " • Try a larger hidden layer (e.g. hidden_size=8) for robustness\n\n" + + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n" + + "[dim]Do not move on to Milestone 03 (TinyDigits) until XOR\n" + "actually converges - otherwise you are debugging on top of a\n" + "broken foundation.[/dim]", + + title="āš ļø XOR Not Solved Yet", + border_style="yellow", + box=box.DOUBLE + )) press_enter_to_continue()