diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b96b1b1a3..6878ebc35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,12 +59,21 @@ repos: - id: format-python-in-qmd name: "Format Python code blocks (Black, 70 chars)" - entry: python tools/scripts/format_python_in_qmd.py + entry: python tools/scripts/content/format_python_in_qmd.py language: python additional_dependencies: [black>=23.0.0] pass_filenames: true files: ^quarto/contents/.*\.qmd$ + - id: check-list-formatting + name: "Fix markdown list formatting (require blank line before lists)" + entry: python tools/scripts/utilities/check_list_formatting.py --fix + language: python + additional_dependencies: [] + pass_filenames: true + files: \.qmd$ + description: "Ensure bullet lists are preceded by blank lines for proper markdown rendering" + # ============================================================================= # PHASE 2: BASIC VALIDATORS (Structure and syntax) # ============================================================================= @@ -168,7 +177,7 @@ repos: # --- Table Formatting Validation --- - id: check-table-formatting name: "Check table formatting (alignment, bolding, spacing)" - entry: python tools/scripts/format_tables.py --check -d quarto/contents/ + entry: python tools/scripts/content/format_tables.py --check -d quarto/contents/ language: python additional_dependencies: [] pass_filenames: false @@ -178,7 +187,7 @@ repos: # --- Part Key Validation --- - id: validate-part-keys name: "Validate part keys in .qmd files" - entry: python tools/scripts/validate_part_keys.py + entry: python tools/scripts/utilities/validate_part_keys.py language: python additional_dependencies: - pyyaml @@ -194,7 +203,7 @@ repos: # --- Image Validation --- - id: validate-images name: "Validate image files" - entry: python tools/scripts/utilities/manage_images.py + entry: python tools/scripts/images/manage_images.py language: python additional_dependencies: - pillow @@ -204,7 +213,7 @@ repos: - id: validate-external-images name: "Check for external images in Quarto files" - entry: python tools/scripts/manage_external_images.py --validate quarto/contents/ + entry: python tools/scripts/images/manage_external_images.py --validate quarto/contents/ language: python additional_dependencies: [requests] pass_filenames: false @@ -213,7 +222,7 @@ repos: - id: validate-image-references name: "Check that all image references exist on disk" - entry: python tools/scripts/validate_image_references.py -d quarto/contents/ --quiet + entry: python tools/scripts/images/validate_image_references.py -d quarto/contents/ --quiet language: python additional_dependencies: [] pass_filenames: false diff --git a/tools/scripts/.backup_20251009_133211.json b/tools/scripts/.backup_20251009_133211.json new file mode 100644 index 000000000..634982140 --- /dev/null +++ b/tools/scripts/.backup_20251009_133211.json @@ -0,0 +1,113 @@ +{ + "timestamp": "20251009_133211", + "files": [ + "validate_part_keys.py", + "cleanup_containers.py", + "remove_bg.py", + "test_publish_live.py", + "check_custom_extensions.py", + "cleanup_workflow_runs_gh.py", + "fix_mid_paragraph_bold.py", + "test_format_tables.py", + "manage_external_images.py", + "format_tables.py", + "__init__.py", + "reorganize_scripts.py", + "test_image_extraction.py", + "generate_release_notes.py", + "rename_downloaded_images.py", + "download_external_images.py", + "standardize_glossaries.py", + "rename_auto_images.py", + "validate_image_references.py", + "format_python_in_qmd.py", + "list_containers.py", + "preflight.py", + "content/check_unreferenced_labels.py", + "content/check_forbidden_footnotes.py", + "content/extract_headers.py", + "content/find_acronyms.py", + "content/fix_bibliography.py", + "content/check_fig_references.py", + "content/footnote_cleanup.py", + "content/fix_callout_titles.py", + "content/format_blank_lines.py", + "content/extract_concepts.py", + "content/check_duplicate_labels.py", + "content/clean_bibliographies.py", + "content/manage_captions.py", + "content/manage_section_ids.py", + "cross_refs/optimized_xrefs_generator.py", + "cross_refs/progressive_xref_generator.py", + "cross_refs/fine_tune_connections.py", + "cross_refs/academic_explanation_generator.py", + "cross_refs/contextual_explanation_generator.py", + "cross_refs/add_target_sections.py", + "cross_refs/fix_intro_explanations.py", + "cross_refs/regenerate_xrefs.py", + "cross_refs/refined_experiments.py", + "cross_refs/convert_to_lua_format.py", + "cross_refs/cognitive_load_optimized_generator.py", + "cross_refs/llm_enhanced_xrefs.py", + "cross_refs/improve_explanations.py", + "cross_refs/quality_analyzer.py", + "cross_refs/production_xref_generator.py", + "cross_refs/analyze_xrefs_coverage.py", + "cross_refs/context_aware_xref_generator.py", + "cross_refs/concept_xrefs_generator.py", + "cross_refs/experimental_framework.py", + "cross_refs/add_section_titles.py", + "cross_refs/fix_xref_issues.py", + "cross_refs/enhance_xref_explanations.py", + "cross_refs/enhance_all_xrefs.py", + "cross_refs/quick_enhance.py", + "cross_refs/manage_cross_references.py", + "utilities/analyze_git_usage.py", + "utilities/check_ascii.py", + "utilities/check_list_formatting.py", + "utilities/manage_sources.py", + "utilities/manage_images.py", + "utilities/convert_svg_to_png.py", + "utilities/git_cleanup.py", + "testing/test_manage_section_ids.py", + "testing/run_tests.py", + "common/config.py", + "common/logging_config.py", + "common/validators.py", + "common/base_classes.py", + "common/__init__.py", + "common/exceptions.py", + "publish/__init__.py", + "publish/render_compress_publish.py", + "publish/modify_dev_announcement.py", + "glossary/smart_consolidation.py", + "glossary/consolidate_similar_terms.py", + "glossary/rule_based_consolidation.py", + "glossary/clean_master_glossary.py", + "glossary/generate_glossary.py", + "glossary/build_global_glossary.py", + "maintenance/correct_image_references_case.py", + "maintenance/rename_images_to_snake_case.py", + "maintenance/analyze_image_sizes.py", + "maintenance/release_notes.py", + "maintenance/integrated_image_analyzer.py", + "maintenance/update_texlive_packages.py", + "maintenance/fix_changelog_years.py", + "maintenance/compress_images.py", + "maintenance/enforce_snake_case_images_final.py", + "maintenance/enforce_snake_case.py", + "maintenance/generate_casing_fix_script.py", + "maintenance/convert_images_to_lowercase.py", + "maintenance/fix_image_case.py", + "maintenance/repo_health_check.py", + "maintenance/change_log.py", + "maintenance/changelog-releasenotes.py", + "maintenance/cleanup_build_artifacts.py", + "maintenance/generate_release_content.py", + "build/generate_stats.py", + "genai/quizzes.py", + "genai/footnote_assistant.py", + "genai/header_update.py", + "genai/fix_dashes.py" + ] +} \ No newline at end of file diff --git a/tools/scripts/fix_mid_paragraph_bold.py b/tools/scripts/content/fix_mid_paragraph_bold.py similarity index 100% rename from tools/scripts/fix_mid_paragraph_bold.py rename to tools/scripts/content/fix_mid_paragraph_bold.py diff --git a/tools/scripts/format_python_in_qmd.py b/tools/scripts/content/format_python_in_qmd.py similarity index 100% rename from tools/scripts/format_python_in_qmd.py rename to tools/scripts/content/format_python_in_qmd.py diff --git a/tools/scripts/format_tables.py b/tools/scripts/content/format_tables.py similarity index 100% rename from tools/scripts/format_tables.py rename to tools/scripts/content/format_tables.py diff --git a/tools/scripts/genai/prompt.txt b/tools/scripts/genai/prompt.txt deleted file mode 100644 index 8627dc5ec..000000000 --- a/tools/scripts/genai/prompt.txt +++ /dev/null @@ -1,181 +0,0 @@ -# Academic Textbook Footnote Specialist Guidelines - -## Core Role -You are a footnote specialist for academic textbooks in machine learning, computer science, and engineering. Your goal is to improve reader comprehension through precise, academically rigorous footnotes—used **only when essential**. - -**Do not add new text.** Only analyze the original content and determine where a footnote is warranted. - -**Avoid footnoting any term already footnoted earlier in the chapter.** Assume readers can refer back to prior definitions or context. - -Footnotes are appropriate when added information would disrupt the main text’s flow—e.g., technical definitions, background context, or source references—especially if a reader might reasonably ask _"What does that mean?"_ or _"Says who?"_ - ---- - -## Footnote Usage - -### Format (`[^fn-keyword]`) -- **Purpose:** Define technical terms **or** provide key contextual insight (e.g., systems, datasets, historical events, architectural shifts). - -- **Footnote Style:** - - **Definition** (uses bold term): - ```markdown - [^fn-keyword]: **Term:** Concise definition (present tense, ≤ 50 words) - ``` - - **Contextual** (no bolding): - ```markdown - [^fn-keyword]: Concise explanation (past/present tense, ≤ 50 words) - ``` - -- **Examples:** - ```markdown - [^fn-asic]: **Application-Specific Integrated Circuit (ASIC):** A custom-built chip optimized for specific tasks in deep learning. - - [^fn-imagenet]: ImageNet was released in 2009 and significantly advanced large-scale supervised learning. - ``` - ---- - -## Decision Framework - -### When to Add Footnotes -- **Definitions:** Acronyms or terms not explained nearby -- **Contextual Notes:** Historical context, system lineage, benchmark origins -- **Cross-Disciplinary Concepts:** Borrowed ideas (e.g., from signal processing, optimization) - -### When to Avoid Footnotes -- Terms covered in course prerequisites or already defined nearby -- Widely known facts or self-evident information -- Nonessential trivia or asides -- Anything previously footnoted in the chapter - ---- - -## 🚨 CRITICAL: Placement Restrictions — BUILD-BREAKING RULES - -**NEVER add footnotes to the following locations** (they will break the build and fail pre-commit checks): - -### ❌ Forbidden Locations (STRICTLY ENFORCED) - -1. **Tables:** - - No footnotes in table content or table cells - - No footnotes in table headers - - Use inline explanations instead - -2. **Table Captions:** - - No footnotes in any table caption text - - Captions using `Table:` or `: Table description` syntax - - Even if the caption seems to need clarification - -3. **Figure Captions:** - - No footnotes in any figure caption text - - Markdown images: `![caption text](image.png)` - - Even for complex technical figures - -4. **Inside Div Blocks (:::):** - - **NO footnotes in ANY content inside `:::` div blocks** - - This includes: - - Callouts (`.callout-note`, `.callout-warning`, etc.) - - Examples (`.example`, `.proof`, etc.) - - Custom divs (`.definition`, `.theorem`, etc.) - - Figures with descriptions - - Any content between `:::` markers - - **Any text between opening `:::` and closing `:::`** is forbidden - -5. **Embedded Content:** - - No footnotes inside code block descriptions that are in divs - - No footnotes in margin content - - No footnotes in special formatting blocks - -### Why This Matters -These restrictions exist because: -- **Quarto's rendering engine CANNOT process footnotes in these contexts** -- Adding footnotes to captions or divs **causes immediate build failures** -- Tables with footnotes **create malformed LaTeX/PDF output** -- **Pre-commit hooks will reject the commit** if violations are found - -### ✅ Safe Locations ONLY -Footnotes are **only** safe in: -- Regular paragraph text (outside any div blocks) -- List items in regular text (outside divs) -- Inline code explanations in regular paragraphs -- Main body narrative text - -### Validation -Before suggesting ANY footnote: -1. Check if the text is inside a `:::` block → **SKIP** -2. Check if it's in a caption (`![...]` or `Table:`) → **SKIP** -3. Check if it's in a table cell → **SKIP** -4. If ANY doubt exists → **SKIP** - -**When in doubt, do NOT add the footnote.** Better to skip a footnote than break the build. - ---- - -## Formatting Guidelines - -### Inline Use -```markdown -The model uses quantization[^fn-quantization] to reduce memory usage. - -[^fn-quantization]: **Quantization:** A technique that reduces numerical precision to lower memory and compute costs. -``` - -### Style -- **Tense:** Present for definitions, past/present for context -- **Voice:** Active -- **Length:** ≤ 50 words; ideal range 10–25 -- **Tone:** Formal, academic - -### Bold Rule -- **Bold the term** in definition footnotes only -- **Do not bold** contextual footnotes - ---- - -## Quality Checklist - -1. **Necessity** - - Does this footnote aid comprehension? - - Is it adding true value? - -2. **Clarity** - - Is it short, readable, and precise? - -3. **Consistency** - - Correct formatting for definition vs. context? - - Bold only if it’s a definition? - -4. **Placement** - - Is the marker placed directly after the supported phrase? - - Is the location safe? (NOT in tables, captions, or divs) - -5. **Duplication** - - Has the term already been footnoted in this chapter? - - If yes, **do not repeat** it. - ---- - -## Best Practices - -- Max **2–3 footnotes per paragraph** -- No more than **1 per sentence**, unless critical -- Avoid circular definitions -- Reuse footnotes if referenced later, but **do not repeat the marker** - ---- - -## Response Format - -Return your response **strictly** as JSON (no extra text): - -```json -{ - "footnotes": [ - { - "marker": "[^fn-keyword]", - "insert_after": "Exact phrase in the text", - "footnote_text": "Definition or context (10–25 words, max 50)" - } - ] -} -``` \ No newline at end of file diff --git a/tools/scripts/standardize_glossaries.py b/tools/scripts/glossary/standardize_glossaries.py similarity index 100% rename from tools/scripts/standardize_glossaries.py rename to tools/scripts/glossary/standardize_glossaries.py diff --git a/tools/scripts/images/README.md b/tools/scripts/images/README.md new file mode 100644 index 000000000..53cbaaf42 --- /dev/null +++ b/tools/scripts/images/README.md @@ -0,0 +1,19 @@ +# Image Management Scripts + +Scripts for managing, processing, and validating images in the book. + +## Image Processing +- `compress_images.py` - Compress images to reduce file size +- `convert_svg_to_png.py` - Convert SVG files to PNG format +- `remove_bg.py` - Remove backgrounds from images + +## Image Management +- `manage_images.py` - Main image management utility +- `download_external_images.py` - Download external images +- `manage_external_images.py` - Manage external image references +- `rename_auto_images.py` - Rename automatically generated images +- `rename_downloaded_images.py` - Rename downloaded images + +## Validation +- `validate_image_references.py` - Ensure all image references are valid +- `analyze_image_sizes.py` - Analyze image sizes and suggest optimizations diff --git a/tools/scripts/images/__init__.py b/tools/scripts/images/__init__.py new file mode 100644 index 000000000..2df83be40 --- /dev/null +++ b/tools/scripts/images/__init__.py @@ -0,0 +1 @@ +"""Scripts for images management.""" diff --git a/tools/scripts/maintenance/analyze_image_sizes.py b/tools/scripts/images/analyze_image_sizes.py similarity index 100% rename from tools/scripts/maintenance/analyze_image_sizes.py rename to tools/scripts/images/analyze_image_sizes.py diff --git a/tools/scripts/maintenance/compress_images.py b/tools/scripts/images/compress_images.py similarity index 100% rename from tools/scripts/maintenance/compress_images.py rename to tools/scripts/images/compress_images.py diff --git a/tools/scripts/utilities/convert_svg_to_png.py b/tools/scripts/images/convert_svg_to_png.py similarity index 100% rename from tools/scripts/utilities/convert_svg_to_png.py rename to tools/scripts/images/convert_svg_to_png.py diff --git a/tools/scripts/download_external_images.py b/tools/scripts/images/download_external_images.py similarity index 100% rename from tools/scripts/download_external_images.py rename to tools/scripts/images/download_external_images.py diff --git a/tools/scripts/manage_external_images.py b/tools/scripts/images/manage_external_images.py similarity index 100% rename from tools/scripts/manage_external_images.py rename to tools/scripts/images/manage_external_images.py diff --git a/tools/scripts/utilities/manage_images.py b/tools/scripts/images/manage_images.py similarity index 100% rename from tools/scripts/utilities/manage_images.py rename to tools/scripts/images/manage_images.py diff --git a/tools/scripts/remove_bg.py b/tools/scripts/images/remove_bg.py similarity index 100% rename from tools/scripts/remove_bg.py rename to tools/scripts/images/remove_bg.py diff --git a/tools/scripts/rename_auto_images.py b/tools/scripts/images/rename_auto_images.py similarity index 100% rename from tools/scripts/rename_auto_images.py rename to tools/scripts/images/rename_auto_images.py diff --git a/tools/scripts/rename_downloaded_images.py b/tools/scripts/images/rename_downloaded_images.py similarity index 100% rename from tools/scripts/rename_downloaded_images.py rename to tools/scripts/images/rename_downloaded_images.py diff --git a/tools/scripts/validate_image_references.py b/tools/scripts/images/validate_image_references.py similarity index 100% rename from tools/scripts/validate_image_references.py rename to tools/scripts/images/validate_image_references.py diff --git a/tools/scripts/infrastructure/README.md b/tools/scripts/infrastructure/README.md new file mode 100644 index 000000000..4b918575e --- /dev/null +++ b/tools/scripts/infrastructure/README.md @@ -0,0 +1,10 @@ +# Infrastructure Scripts + +Scripts for managing CI/CD, containers, and workflow infrastructure. + +## Container Management +- `cleanup_containers.py` - Clean up Docker containers +- `list_containers.py` - List active containers + +## Workflow Management +- `cleanup_workflow_runs_gh.py` - Clean up old GitHub Actions workflow runs diff --git a/tools/scripts/infrastructure/__init__.py b/tools/scripts/infrastructure/__init__.py new file mode 100644 index 000000000..249345dfe --- /dev/null +++ b/tools/scripts/infrastructure/__init__.py @@ -0,0 +1 @@ +"""Scripts for infrastructure management.""" diff --git a/tools/scripts/cleanup_containers.py b/tools/scripts/infrastructure/cleanup_containers.py similarity index 100% rename from tools/scripts/cleanup_containers.py rename to tools/scripts/infrastructure/cleanup_containers.py diff --git a/tools/scripts/cleanup_workflow_runs_gh.py b/tools/scripts/infrastructure/cleanup_workflow_runs_gh.py similarity index 100% rename from tools/scripts/cleanup_workflow_runs_gh.py rename to tools/scripts/infrastructure/cleanup_workflow_runs_gh.py diff --git a/tools/scripts/list_containers.py b/tools/scripts/infrastructure/list_containers.py similarity index 100% rename from tools/scripts/list_containers.py rename to tools/scripts/infrastructure/list_containers.py diff --git a/tools/scripts/generate_release_notes.py b/tools/scripts/maintenance/generate_release_notes.py similarity index 100% rename from tools/scripts/generate_release_notes.py rename to tools/scripts/maintenance/generate_release_notes.py diff --git a/tools/scripts/preflight.py b/tools/scripts/maintenance/preflight.py similarity index 100% rename from tools/scripts/preflight.py rename to tools/scripts/maintenance/preflight.py diff --git a/tools/scripts/reorganize_scripts.py b/tools/scripts/reorganize_scripts.py new file mode 100755 index 000000000..3a979513b --- /dev/null +++ b/tools/scripts/reorganize_scripts.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +""" +Reorganize tools/scripts/ directory structure. + +This script: +1. Creates new subdirectories +2. Moves scripts to proper locations +3. Updates all references (pre-commit, imports, README) +4. Creates a rollback backup +""" + +import os +import shutil +import json +from pathlib import Path +from datetime import datetime + +# Define the migration plan +MIGRATION_PLAN = { + # Images subdirectory - consolidate all image-related scripts + 'images': [ + 'download_external_images.py', + 'manage_external_images.py', + 'remove_bg.py', + 'rename_auto_images.py', + 'rename_downloaded_images.py', + 'validate_image_references.py', + ], + + # Content subdirectory - add formatting scripts + 'content': [ + 'fix_mid_paragraph_bold.py', + 'format_python_in_qmd.py', + 'format_tables.py', + ], + + # Testing subdirectory - consolidate all tests + 'testing': [ + 'test_format_tables.py', + 'test_image_extraction.py', + 'test_publish_live.py', + ], + + # Infrastructure subdirectory - CI/CD and container management + 'infrastructure': [ + 'cleanup_containers.py', + 'list_containers.py', + 'cleanup_workflow_runs_gh.py', + ], + + # Glossary subdirectory - move glossary script + 'glossary': [ + 'standardize_glossaries.py', + ], + + # Maintenance subdirectory - add release and preflight + 'maintenance': [ + 'generate_release_notes.py', + 'preflight.py', + ], + + # Utilities subdirectory - validation scripts + 'utilities': [ + 'check_custom_extensions.py', + 'validate_part_keys.py', + ], +} + +# Files that also need to be moved from existing subdirectories +EXISTING_SUBDIRECTORY_MOVES = { + 'images': [ + ('utilities/manage_images.py', 'manage_images.py'), + ('utilities/convert_svg_to_png.py', 'convert_svg_to_png.py'), + ('maintenance/compress_images.py', 'compress_images.py'), + ('maintenance/analyze_image_sizes.py', 'analyze_image_sizes.py'), + ], +} + +# Pre-commit reference updates +PRECOMMIT_UPDATES = { + 'tools/scripts/format_python_in_qmd.py': 'tools/scripts/content/format_python_in_qmd.py', + 'tools/scripts/format_tables.py': 'tools/scripts/content/format_tables.py', + 'tools/scripts/validate_part_keys.py': 'tools/scripts/utilities/validate_part_keys.py', + 'tools/scripts/manage_external_images.py': 'tools/scripts/images/manage_external_images.py', + 'tools/scripts/validate_image_references.py': 'tools/scripts/images/validate_image_references.py', + 'tools/scripts/generate_release_notes.py': 'tools/scripts/maintenance/generate_release_notes.py', + 'tools/scripts/preflight.py': 'tools/scripts/maintenance/preflight.py', +} + + +def create_backup(): + """Create a backup of the current state.""" + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + backup_file = f'tools/scripts/.backup_{timestamp}.json' + + backup_data = { + 'timestamp': timestamp, + 'files': [] + } + + # Record all file locations + for root, dirs, files in os.walk('tools/scripts'): + for file in files: + if file.endswith('.py'): + rel_path = os.path.relpath(os.path.join(root, file), 'tools/scripts') + backup_data['files'].append(rel_path) + + with open(backup_file, 'w') as f: + json.dump(backup_data, f, indent=2) + + print(f"✅ Created backup: {backup_file}") + return backup_file + + +def create_directories(): + """Create new subdirectories.""" + print("\n📁 Creating new directories...") + + new_dirs = ['images', 'infrastructure'] + for dirname in new_dirs: + dirpath = f'tools/scripts/{dirname}' + if not os.path.exists(dirpath): + os.makedirs(dirpath) + # Create __init__.py + with open(f'{dirpath}/__init__.py', 'w') as f: + f.write(f'"""Scripts for {dirname} management."""\n') + print(f" ✅ Created {dirpath}/") + else: + print(f" ⚠️ {dirpath}/ already exists") + + +def move_files_from_root(): + """Move files from root to subdirectories.""" + print("\n📦 Moving files from root level...") + + moved_count = 0 + for target_dir, files in MIGRATION_PLAN.items(): + target_path = f'tools/scripts/{target_dir}' + + for filename in files: + source = f'tools/scripts/{filename}' + dest = f'{target_path}/{filename}' + + if os.path.exists(source): + shutil.move(source, dest) + print(f" ✅ {filename} → {target_dir}/") + moved_count += 1 + else: + print(f" ⚠️ {filename} not found (may already be moved)") + + print(f"\n Moved {moved_count} files from root") + + +def move_files_between_subdirs(): + """Move files between existing subdirectories.""" + print("\n🔄 Consolidating files between subdirectories...") + + moved_count = 0 + for target_dir, moves in EXISTING_SUBDIRECTORY_MOVES.items(): + target_path = f'tools/scripts/{target_dir}' + + for source_rel, dest_name in moves: + source = f'tools/scripts/{source_rel}' + dest = f'{target_path}/{dest_name}' + + if os.path.exists(source): + shutil.move(source, dest) + print(f" ✅ {source_rel} → {target_dir}/{dest_name}") + moved_count += 1 + else: + print(f" ⚠️ {source_rel} not found") + + print(f"\n Moved {moved_count} files between subdirectories") + + +def update_precommit_config(): + """Update .pre-commit-config.yaml with new paths.""" + print("\n⚙️ Updating .pre-commit-config.yaml...") + + config_path = '.pre-commit-config.yaml' + + with open(config_path, 'r') as f: + content = f.read() + + original_content = content + updates_made = 0 + + for old_path, new_path in PRECOMMIT_UPDATES.items(): + if old_path in content: + content = content.replace(old_path, new_path) + updates_made += 1 + print(f" ✅ Updated: {os.path.basename(old_path)}") + + if updates_made > 0: + with open(config_path, 'w') as f: + f.write(content) + print(f"\n Updated {updates_made} references in pre-commit config") + else: + print(" ℹ️ No updates needed") + + +def create_readme_files(): + """Create README files for new directories.""" + print("\n📝 Creating README files...") + + readmes = { + 'images': '''# Image Management Scripts + +Scripts for managing, processing, and validating images in the book. + +## Image Processing +- `compress_images.py` - Compress images to reduce file size +- `convert_svg_to_png.py` - Convert SVG files to PNG format +- `remove_bg.py` - Remove backgrounds from images + +## Image Management +- `manage_images.py` - Main image management utility +- `download_external_images.py` - Download external images +- `manage_external_images.py` - Manage external image references +- `rename_auto_images.py` - Rename automatically generated images +- `rename_downloaded_images.py` - Rename downloaded images + +## Validation +- `validate_image_references.py` - Ensure all image references are valid +- `analyze_image_sizes.py` - Analyze image sizes and suggest optimizations +''', + 'infrastructure': '''# Infrastructure Scripts + +Scripts for managing CI/CD, containers, and workflow infrastructure. + +## Container Management +- `cleanup_containers.py` - Clean up Docker containers +- `list_containers.py` - List active containers + +## Workflow Management +- `cleanup_workflow_runs_gh.py` - Clean up old GitHub Actions workflow runs +''', + } + + for dirname, content in readmes.items(): + readme_path = f'tools/scripts/{dirname}/README.md' + if not os.path.exists(readme_path): + with open(readme_path, 'w') as f: + f.write(content) + print(f" ✅ Created {dirname}/README.md") + + +def generate_summary(): + """Generate a summary of the reorganization.""" + print("\n" + "=" * 80) + print("📊 REORGANIZATION SUMMARY") + print("=" * 80) + + # Count files in each directory + summary = {} + for root, dirs, files in os.walk('tools/scripts'): + # Skip __pycache__ and hidden directories + if '__pycache__' in root or '/.backup' in root: + continue + + dirname = os.path.relpath(root, 'tools/scripts') + py_files = [f for f in files if f.endswith('.py') and f != '__init__.py'] + + if py_files: + summary[dirname] = len(py_files) + + print("\n📁 Files per directory:") + for dirname in sorted(summary.keys()): + count = summary[dirname] + print(f" {dirname + '/':<30} {count:>3} files") + + # Count root level files + root_files = [f for f in os.listdir('tools/scripts') + if f.endswith('.py') and os.path.isfile(f'tools/scripts/{f}')] + + print(f"\n🎯 Root level scripts remaining: {len(root_files)}") + if root_files: + for f in root_files: + print(f" - {f}") + + print("\n✅ Reorganization complete!") + print("\nNext steps:") + print(" 1. Test pre-commit hooks: pre-commit run --all-files") + print(" 2. Check for any broken imports") + print(" 3. Update any documentation references") + + +def main(): + """Main reorganization process.""" + import sys + + print("🔧 SCRIPT REORGANIZATION TOOL") + print("=" * 80) + print("\nThis will reorganize tools/scripts/ directory structure.") + print("\nChanges:") + print(" • Create new subdirectories (images/, infrastructure/)") + print(" • Move 21 scripts from root to appropriate subdirectories") + print(" • Consolidate scattered scripts (images, tests, etc.)") + print(" • Update .pre-commit-config.yaml references") + print(" • Create documentation") + + # Check for --yes flag + if '--yes' not in sys.argv: + response = input("\n⚠️ Proceed with reorganization? (yes/no): ") + if response.lower() != 'yes': + print("\n❌ Cancelled") + return 1 + else: + print("\n✅ Auto-confirmed with --yes flag") + + try: + # Create backup + backup_file = create_backup() + + # Execute reorganization + create_directories() + move_files_from_root() + move_files_between_subdirs() + update_precommit_config() + create_readme_files() + + # Summary + generate_summary() + + print(f"\n💾 Backup saved to: {backup_file}") + print(" (Can be used for rollback if needed)") + + return 0 + + except Exception as e: + print(f"\n❌ Error during reorganization: {e}") + print(" Please restore from backup if needed") + return 1 + + +if __name__ == '__main__': + exit(main()) + diff --git a/tools/scripts/test_format_tables.py b/tools/scripts/testing/test_format_tables.py similarity index 100% rename from tools/scripts/test_format_tables.py rename to tools/scripts/testing/test_format_tables.py diff --git a/tools/scripts/test_image_extraction.py b/tools/scripts/testing/test_image_extraction.py similarity index 100% rename from tools/scripts/test_image_extraction.py rename to tools/scripts/testing/test_image_extraction.py diff --git a/tools/scripts/test_publish_live.py b/tools/scripts/testing/test_publish_live.py similarity index 100% rename from tools/scripts/test_publish_live.py rename to tools/scripts/testing/test_publish_live.py diff --git a/tools/scripts/check_custom_extensions.py b/tools/scripts/utilities/check_custom_extensions.py similarity index 100% rename from tools/scripts/check_custom_extensions.py rename to tools/scripts/utilities/check_custom_extensions.py diff --git a/tools/scripts/validate_part_keys.py b/tools/scripts/utilities/validate_part_keys.py similarity index 100% rename from tools/scripts/validate_part_keys.py rename to tools/scripts/utilities/validate_part_keys.py