mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-04-28 16:48:30 -05:00
refactor(tools): reorganize scripts directory structure for better maintainability
Consolidated 21 root-level scripts into logical subdirectories: New structure: - images/: All image management scripts (10 files consolidated from 3 locations) - infrastructure/: CI/CD and container scripts (3 files) - content/: Added formatting scripts (3 files moved from root) - testing/: All test scripts (5 files consolidated) - glossary/: Added standardize_glossaries.py - maintenance/: Added generate_release_notes.py, preflight.py - utilities/: Added validation scripts Benefits: - Reduced root-level clutter (21 → 2 files) - Related scripts grouped logically - Easier to find and maintain scripts - Follows standard project organization patterns Changes: - Created new subdirectories: images/, infrastructure/ - Moved scripts from root to appropriate subdirectories - Consolidated scattered scripts (images were in 3 places) - Updated all pre-commit hook references - Created README files for new directories - Included backup file for rollback if needed Tool: tools/scripts/reorganize_scripts.py (for future reference)
This commit is contained in:
Vijay Janapa Reddi
@@ -59,12 +59,21 @@ repos:
|
||||
|
||||
- id: format-python-in-qmd
|
||||
name: "Format Python code blocks (Black, 70 chars)"
|
||||
entry: python tools/scripts/format_python_in_qmd.py
|
||||
entry: python tools/scripts/content/format_python_in_qmd.py
|
||||
language: python
|
||||
additional_dependencies: [black>=23.0.0]
|
||||
pass_filenames: true
|
||||
files: ^quarto/contents/.*\.qmd$
|
||||
|
||||
- id: check-list-formatting
|
||||
name: "Fix markdown list formatting (require blank line before lists)"
|
||||
entry: python tools/scripts/utilities/check_list_formatting.py --fix
|
||||
language: python
|
||||
additional_dependencies: []
|
||||
pass_filenames: true
|
||||
files: \.qmd$
|
||||
description: "Ensure bullet lists are preceded by blank lines for proper markdown rendering"
|
||||
|
||||
# =============================================================================
|
||||
# PHASE 2: BASIC VALIDATORS (Structure and syntax)
|
||||
# =============================================================================
|
||||
@@ -168,7 +177,7 @@ repos:
|
||||
# --- Table Formatting Validation ---
|
||||
- id: check-table-formatting
|
||||
name: "Check table formatting (alignment, bolding, spacing)"
|
||||
entry: python tools/scripts/format_tables.py --check -d quarto/contents/
|
||||
entry: python tools/scripts/content/format_tables.py --check -d quarto/contents/
|
||||
language: python
|
||||
additional_dependencies: []
|
||||
pass_filenames: false
|
||||
@@ -178,7 +187,7 @@ repos:
|
||||
# --- Part Key Validation ---
|
||||
- id: validate-part-keys
|
||||
name: "Validate part keys in .qmd files"
|
||||
entry: python tools/scripts/validate_part_keys.py
|
||||
entry: python tools/scripts/utilities/validate_part_keys.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- pyyaml
|
||||
@@ -194,7 +203,7 @@ repos:
|
||||
# --- Image Validation ---
|
||||
- id: validate-images
|
||||
name: "Validate image files"
|
||||
entry: python tools/scripts/utilities/manage_images.py
|
||||
entry: python tools/scripts/images/manage_images.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- pillow
|
||||
@@ -204,7 +213,7 @@ repos:
|
||||
|
||||
- id: validate-external-images
|
||||
name: "Check for external images in Quarto files"
|
||||
entry: python tools/scripts/manage_external_images.py --validate quarto/contents/
|
||||
entry: python tools/scripts/images/manage_external_images.py --validate quarto/contents/
|
||||
language: python
|
||||
additional_dependencies: [requests]
|
||||
pass_filenames: false
|
||||
@@ -213,7 +222,7 @@ repos:
|
||||
|
||||
- id: validate-image-references
|
||||
name: "Check that all image references exist on disk"
|
||||
entry: python tools/scripts/validate_image_references.py -d quarto/contents/ --quiet
|
||||
entry: python tools/scripts/images/validate_image_references.py -d quarto/contents/ --quiet
|
||||
language: python
|
||||
additional_dependencies: []
|
||||
pass_filenames: false
|
||||
|
||||
113
tools/scripts/.backup_20251009_133211.json
Normal file
113
tools/scripts/.backup_20251009_133211.json
Normal file
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"timestamp": "20251009_133211",
|
||||
"files": [
|
||||
"validate_part_keys.py",
|
||||
"cleanup_containers.py",
|
||||
"remove_bg.py",
|
||||
"test_publish_live.py",
|
||||
"check_custom_extensions.py",
|
||||
"cleanup_workflow_runs_gh.py",
|
||||
"fix_mid_paragraph_bold.py",
|
||||
"test_format_tables.py",
|
||||
"manage_external_images.py",
|
||||
"format_tables.py",
|
||||
"__init__.py",
|
||||
"reorganize_scripts.py",
|
||||
"test_image_extraction.py",
|
||||
"generate_release_notes.py",
|
||||
"rename_downloaded_images.py",
|
||||
"download_external_images.py",
|
||||
"standardize_glossaries.py",
|
||||
"rename_auto_images.py",
|
||||
"validate_image_references.py",
|
||||
"format_python_in_qmd.py",
|
||||
"list_containers.py",
|
||||
"preflight.py",
|
||||
"content/check_unreferenced_labels.py",
|
||||
"content/check_forbidden_footnotes.py",
|
||||
"content/extract_headers.py",
|
||||
"content/find_acronyms.py",
|
||||
"content/fix_bibliography.py",
|
||||
"content/check_fig_references.py",
|
||||
"content/footnote_cleanup.py",
|
||||
"content/fix_callout_titles.py",
|
||||
"content/format_blank_lines.py",
|
||||
"content/extract_concepts.py",
|
||||
"content/check_duplicate_labels.py",
|
||||
"content/clean_bibliographies.py",
|
||||
"content/manage_captions.py",
|
||||
"content/manage_section_ids.py",
|
||||
"cross_refs/optimized_xrefs_generator.py",
|
||||
"cross_refs/progressive_xref_generator.py",
|
||||
"cross_refs/fine_tune_connections.py",
|
||||
"cross_refs/academic_explanation_generator.py",
|
||||
"cross_refs/contextual_explanation_generator.py",
|
||||
"cross_refs/add_target_sections.py",
|
||||
"cross_refs/fix_intro_explanations.py",
|
||||
"cross_refs/regenerate_xrefs.py",
|
||||
"cross_refs/refined_experiments.py",
|
||||
"cross_refs/convert_to_lua_format.py",
|
||||
"cross_refs/cognitive_load_optimized_generator.py",
|
||||
"cross_refs/llm_enhanced_xrefs.py",
|
||||
"cross_refs/improve_explanations.py",
|
||||
"cross_refs/quality_analyzer.py",
|
||||
"cross_refs/production_xref_generator.py",
|
||||
"cross_refs/analyze_xrefs_coverage.py",
|
||||
"cross_refs/context_aware_xref_generator.py",
|
||||
"cross_refs/concept_xrefs_generator.py",
|
||||
"cross_refs/experimental_framework.py",
|
||||
"cross_refs/add_section_titles.py",
|
||||
"cross_refs/fix_xref_issues.py",
|
||||
"cross_refs/enhance_xref_explanations.py",
|
||||
"cross_refs/enhance_all_xrefs.py",
|
||||
"cross_refs/quick_enhance.py",
|
||||
"cross_refs/manage_cross_references.py",
|
||||
"utilities/analyze_git_usage.py",
|
||||
"utilities/check_ascii.py",
|
||||
"utilities/check_list_formatting.py",
|
||||
"utilities/manage_sources.py",
|
||||
"utilities/manage_images.py",
|
||||
"utilities/convert_svg_to_png.py",
|
||||
"utilities/git_cleanup.py",
|
||||
"testing/test_manage_section_ids.py",
|
||||
"testing/run_tests.py",
|
||||
"common/config.py",
|
||||
"common/logging_config.py",
|
||||
"common/validators.py",
|
||||
"common/base_classes.py",
|
||||
"common/__init__.py",
|
||||
"common/exceptions.py",
|
||||
"publish/__init__.py",
|
||||
"publish/render_compress_publish.py",
|
||||
"publish/modify_dev_announcement.py",
|
||||
"glossary/smart_consolidation.py",
|
||||
"glossary/consolidate_similar_terms.py",
|
||||
"glossary/rule_based_consolidation.py",
|
||||
"glossary/clean_master_glossary.py",
|
||||
"glossary/generate_glossary.py",
|
||||
"glossary/build_global_glossary.py",
|
||||
"maintenance/correct_image_references_case.py",
|
||||
"maintenance/rename_images_to_snake_case.py",
|
||||
"maintenance/analyze_image_sizes.py",
|
||||
"maintenance/release_notes.py",
|
||||
"maintenance/integrated_image_analyzer.py",
|
||||
"maintenance/update_texlive_packages.py",
|
||||
"maintenance/fix_changelog_years.py",
|
||||
"maintenance/compress_images.py",
|
||||
"maintenance/enforce_snake_case_images_final.py",
|
||||
"maintenance/enforce_snake_case.py",
|
||||
"maintenance/generate_casing_fix_script.py",
|
||||
"maintenance/convert_images_to_lowercase.py",
|
||||
"maintenance/fix_image_case.py",
|
||||
"maintenance/repo_health_check.py",
|
||||
"maintenance/change_log.py",
|
||||
"maintenance/changelog-releasenotes.py",
|
||||
"maintenance/cleanup_build_artifacts.py",
|
||||
"maintenance/generate_release_content.py",
|
||||
"build/generate_stats.py",
|
||||
"genai/quizzes.py",
|
||||
"genai/footnote_assistant.py",
|
||||
"genai/header_update.py",
|
||||
"genai/fix_dashes.py"
|
||||
]
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
# Academic Textbook Footnote Specialist Guidelines
|
||||
|
||||
## Core Role
|
||||
You are a footnote specialist for academic textbooks in machine learning, computer science, and engineering. Your goal is to improve reader comprehension through precise, academically rigorous footnotes—used **only when essential**.
|
||||
|
||||
**Do not add new text.** Only analyze the original content and determine where a footnote is warranted.
|
||||
|
||||
**Avoid footnoting any term already footnoted earlier in the chapter.** Assume readers can refer back to prior definitions or context.
|
||||
|
||||
Footnotes are appropriate when added information would disrupt the main text’s flow—e.g., technical definitions, background context, or source references—especially if a reader might reasonably ask _"What does that mean?"_ or _"Says who?"_
|
||||
|
||||
---
|
||||
|
||||
## Footnote Usage
|
||||
|
||||
### Format (`[^fn-keyword]`)
|
||||
- **Purpose:** Define technical terms **or** provide key contextual insight (e.g., systems, datasets, historical events, architectural shifts).
|
||||
|
||||
- **Footnote Style:**
|
||||
- **Definition** (uses bold term):
|
||||
```markdown
|
||||
[^fn-keyword]: **Term:** Concise definition (present tense, ≤ 50 words)
|
||||
```
|
||||
- **Contextual** (no bolding):
|
||||
```markdown
|
||||
[^fn-keyword]: Concise explanation (past/present tense, ≤ 50 words)
|
||||
```
|
||||
|
||||
- **Examples:**
|
||||
```markdown
|
||||
[^fn-asic]: **Application-Specific Integrated Circuit (ASIC):** A custom-built chip optimized for specific tasks in deep learning.
|
||||
|
||||
[^fn-imagenet]: ImageNet was released in 2009 and significantly advanced large-scale supervised learning.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Decision Framework
|
||||
|
||||
### When to Add Footnotes
|
||||
- **Definitions:** Acronyms or terms not explained nearby
|
||||
- **Contextual Notes:** Historical context, system lineage, benchmark origins
|
||||
- **Cross-Disciplinary Concepts:** Borrowed ideas (e.g., from signal processing, optimization)
|
||||
|
||||
### When to Avoid Footnotes
|
||||
- Terms covered in course prerequisites or already defined nearby
|
||||
- Widely known facts or self-evident information
|
||||
- Nonessential trivia or asides
|
||||
- Anything previously footnoted in the chapter
|
||||
|
||||
---
|
||||
|
||||
## 🚨 CRITICAL: Placement Restrictions — BUILD-BREAKING RULES
|
||||
|
||||
**NEVER add footnotes to the following locations** (they will break the build and fail pre-commit checks):
|
||||
|
||||
### ❌ Forbidden Locations (STRICTLY ENFORCED)
|
||||
|
||||
1. **Tables:**
|
||||
- No footnotes in table content or table cells
|
||||
- No footnotes in table headers
|
||||
- Use inline explanations instead
|
||||
|
||||
2. **Table Captions:**
|
||||
- No footnotes in any table caption text
|
||||
- Captions using `Table:` or `: Table description` syntax
|
||||
- Even if the caption seems to need clarification
|
||||
|
||||
3. **Figure Captions:**
|
||||
- No footnotes in any figure caption text
|
||||
- Markdown images: ``
|
||||
- Even for complex technical figures
|
||||
|
||||
4. **Inside Div Blocks (:::):**
|
||||
- **NO footnotes in ANY content inside `:::` div blocks**
|
||||
- This includes:
|
||||
- Callouts (`.callout-note`, `.callout-warning`, etc.)
|
||||
- Examples (`.example`, `.proof`, etc.)
|
||||
- Custom divs (`.definition`, `.theorem`, etc.)
|
||||
- Figures with descriptions
|
||||
- Any content between `:::` markers
|
||||
- **Any text between opening `:::` and closing `:::`** is forbidden
|
||||
|
||||
5. **Embedded Content:**
|
||||
- No footnotes inside code block descriptions that are in divs
|
||||
- No footnotes in margin content
|
||||
- No footnotes in special formatting blocks
|
||||
|
||||
### Why This Matters
|
||||
These restrictions exist because:
|
||||
- **Quarto's rendering engine CANNOT process footnotes in these contexts**
|
||||
- Adding footnotes to captions or divs **causes immediate build failures**
|
||||
- Tables with footnotes **create malformed LaTeX/PDF output**
|
||||
- **Pre-commit hooks will reject the commit** if violations are found
|
||||
|
||||
### ✅ Safe Locations ONLY
|
||||
Footnotes are **only** safe in:
|
||||
- Regular paragraph text (outside any div blocks)
|
||||
- List items in regular text (outside divs)
|
||||
- Inline code explanations in regular paragraphs
|
||||
- Main body narrative text
|
||||
|
||||
### Validation
|
||||
Before suggesting ANY footnote:
|
||||
1. Check if the text is inside a `:::` block → **SKIP**
|
||||
2. Check if it's in a caption (`![...]` or `Table:`) → **SKIP**
|
||||
3. Check if it's in a table cell → **SKIP**
|
||||
4. If ANY doubt exists → **SKIP**
|
||||
|
||||
**When in doubt, do NOT add the footnote.** Better to skip a footnote than break the build.
|
||||
|
||||
---
|
||||
|
||||
## Formatting Guidelines
|
||||
|
||||
### Inline Use
|
||||
```markdown
|
||||
The model uses quantization[^fn-quantization] to reduce memory usage.
|
||||
|
||||
[^fn-quantization]: **Quantization:** A technique that reduces numerical precision to lower memory and compute costs.
|
||||
```
|
||||
|
||||
### Style
|
||||
- **Tense:** Present for definitions, past/present for context
|
||||
- **Voice:** Active
|
||||
- **Length:** ≤ 50 words; ideal range 10–25
|
||||
- **Tone:** Formal, academic
|
||||
|
||||
### Bold Rule
|
||||
- **Bold the term** in definition footnotes only
|
||||
- **Do not bold** contextual footnotes
|
||||
|
||||
---
|
||||
|
||||
## Quality Checklist
|
||||
|
||||
1. **Necessity**
|
||||
- Does this footnote aid comprehension?
|
||||
- Is it adding true value?
|
||||
|
||||
2. **Clarity**
|
||||
- Is it short, readable, and precise?
|
||||
|
||||
3. **Consistency**
|
||||
- Correct formatting for definition vs. context?
|
||||
- Bold only if it’s a definition?
|
||||
|
||||
4. **Placement**
|
||||
- Is the marker placed directly after the supported phrase?
|
||||
- Is the location safe? (NOT in tables, captions, or divs)
|
||||
|
||||
5. **Duplication**
|
||||
- Has the term already been footnoted in this chapter?
|
||||
- If yes, **do not repeat** it.
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Max **2–3 footnotes per paragraph**
|
||||
- No more than **1 per sentence**, unless critical
|
||||
- Avoid circular definitions
|
||||
- Reuse footnotes if referenced later, but **do not repeat the marker**
|
||||
|
||||
---
|
||||
|
||||
## Response Format
|
||||
|
||||
Return your response **strictly** as JSON (no extra text):
|
||||
|
||||
```json
|
||||
{
|
||||
"footnotes": [
|
||||
{
|
||||
"marker": "[^fn-keyword]",
|
||||
"insert_after": "Exact phrase in the text",
|
||||
"footnote_text": "Definition or context (10–25 words, max 50)"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
19
tools/scripts/images/README.md
Normal file
19
tools/scripts/images/README.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# Image Management Scripts
|
||||
|
||||
Scripts for managing, processing, and validating images in the book.
|
||||
|
||||
## Image Processing
|
||||
- `compress_images.py` - Compress images to reduce file size
|
||||
- `convert_svg_to_png.py` - Convert SVG files to PNG format
|
||||
- `remove_bg.py` - Remove backgrounds from images
|
||||
|
||||
## Image Management
|
||||
- `manage_images.py` - Main image management utility
|
||||
- `download_external_images.py` - Download external images
|
||||
- `manage_external_images.py` - Manage external image references
|
||||
- `rename_auto_images.py` - Rename automatically generated images
|
||||
- `rename_downloaded_images.py` - Rename downloaded images
|
||||
|
||||
## Validation
|
||||
- `validate_image_references.py` - Ensure all image references are valid
|
||||
- `analyze_image_sizes.py` - Analyze image sizes and suggest optimizations
|
||||
1
tools/scripts/images/__init__.py
Normal file
1
tools/scripts/images/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Scripts for images management."""
|
||||
10
tools/scripts/infrastructure/README.md
Normal file
10
tools/scripts/infrastructure/README.md
Normal file
@@ -0,0 +1,10 @@
|
||||
# Infrastructure Scripts
|
||||
|
||||
Scripts for managing CI/CD, containers, and workflow infrastructure.
|
||||
|
||||
## Container Management
|
||||
- `cleanup_containers.py` - Clean up Docker containers
|
||||
- `list_containers.py` - List active containers
|
||||
|
||||
## Workflow Management
|
||||
- `cleanup_workflow_runs_gh.py` - Clean up old GitHub Actions workflow runs
|
||||
1
tools/scripts/infrastructure/__init__.py
Normal file
1
tools/scripts/infrastructure/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Scripts for infrastructure management."""
|
||||
338
tools/scripts/reorganize_scripts.py
Executable file
338
tools/scripts/reorganize_scripts.py
Executable file
@@ -0,0 +1,338 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reorganize tools/scripts/ directory structure.
|
||||
|
||||
This script:
|
||||
1. Creates new subdirectories
|
||||
2. Moves scripts to proper locations
|
||||
3. Updates all references (pre-commit, imports, README)
|
||||
4. Creates a rollback backup
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# Define the migration plan
|
||||
MIGRATION_PLAN = {
|
||||
# Images subdirectory - consolidate all image-related scripts
|
||||
'images': [
|
||||
'download_external_images.py',
|
||||
'manage_external_images.py',
|
||||
'remove_bg.py',
|
||||
'rename_auto_images.py',
|
||||
'rename_downloaded_images.py',
|
||||
'validate_image_references.py',
|
||||
],
|
||||
|
||||
# Content subdirectory - add formatting scripts
|
||||
'content': [
|
||||
'fix_mid_paragraph_bold.py',
|
||||
'format_python_in_qmd.py',
|
||||
'format_tables.py',
|
||||
],
|
||||
|
||||
# Testing subdirectory - consolidate all tests
|
||||
'testing': [
|
||||
'test_format_tables.py',
|
||||
'test_image_extraction.py',
|
||||
'test_publish_live.py',
|
||||
],
|
||||
|
||||
# Infrastructure subdirectory - CI/CD and container management
|
||||
'infrastructure': [
|
||||
'cleanup_containers.py',
|
||||
'list_containers.py',
|
||||
'cleanup_workflow_runs_gh.py',
|
||||
],
|
||||
|
||||
# Glossary subdirectory - move glossary script
|
||||
'glossary': [
|
||||
'standardize_glossaries.py',
|
||||
],
|
||||
|
||||
# Maintenance subdirectory - add release and preflight
|
||||
'maintenance': [
|
||||
'generate_release_notes.py',
|
||||
'preflight.py',
|
||||
],
|
||||
|
||||
# Utilities subdirectory - validation scripts
|
||||
'utilities': [
|
||||
'check_custom_extensions.py',
|
||||
'validate_part_keys.py',
|
||||
],
|
||||
}
|
||||
|
||||
# Files that also need to be moved from existing subdirectories
|
||||
EXISTING_SUBDIRECTORY_MOVES = {
|
||||
'images': [
|
||||
('utilities/manage_images.py', 'manage_images.py'),
|
||||
('utilities/convert_svg_to_png.py', 'convert_svg_to_png.py'),
|
||||
('maintenance/compress_images.py', 'compress_images.py'),
|
||||
('maintenance/analyze_image_sizes.py', 'analyze_image_sizes.py'),
|
||||
],
|
||||
}
|
||||
|
||||
# Pre-commit reference updates
|
||||
PRECOMMIT_UPDATES = {
|
||||
'tools/scripts/format_python_in_qmd.py': 'tools/scripts/content/format_python_in_qmd.py',
|
||||
'tools/scripts/format_tables.py': 'tools/scripts/content/format_tables.py',
|
||||
'tools/scripts/validate_part_keys.py': 'tools/scripts/utilities/validate_part_keys.py',
|
||||
'tools/scripts/manage_external_images.py': 'tools/scripts/images/manage_external_images.py',
|
||||
'tools/scripts/validate_image_references.py': 'tools/scripts/images/validate_image_references.py',
|
||||
'tools/scripts/generate_release_notes.py': 'tools/scripts/maintenance/generate_release_notes.py',
|
||||
'tools/scripts/preflight.py': 'tools/scripts/maintenance/preflight.py',
|
||||
}
|
||||
|
||||
|
||||
def create_backup():
|
||||
"""Create a backup of the current state."""
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
backup_file = f'tools/scripts/.backup_{timestamp}.json'
|
||||
|
||||
backup_data = {
|
||||
'timestamp': timestamp,
|
||||
'files': []
|
||||
}
|
||||
|
||||
# Record all file locations
|
||||
for root, dirs, files in os.walk('tools/scripts'):
|
||||
for file in files:
|
||||
if file.endswith('.py'):
|
||||
rel_path = os.path.relpath(os.path.join(root, file), 'tools/scripts')
|
||||
backup_data['files'].append(rel_path)
|
||||
|
||||
with open(backup_file, 'w') as f:
|
||||
json.dump(backup_data, f, indent=2)
|
||||
|
||||
print(f"✅ Created backup: {backup_file}")
|
||||
return backup_file
|
||||
|
||||
|
||||
def create_directories():
|
||||
"""Create new subdirectories."""
|
||||
print("\n📁 Creating new directories...")
|
||||
|
||||
new_dirs = ['images', 'infrastructure']
|
||||
for dirname in new_dirs:
|
||||
dirpath = f'tools/scripts/{dirname}'
|
||||
if not os.path.exists(dirpath):
|
||||
os.makedirs(dirpath)
|
||||
# Create __init__.py
|
||||
with open(f'{dirpath}/__init__.py', 'w') as f:
|
||||
f.write(f'"""Scripts for {dirname} management."""\n')
|
||||
print(f" ✅ Created {dirpath}/")
|
||||
else:
|
||||
print(f" ⚠️ {dirpath}/ already exists")
|
||||
|
||||
|
||||
def move_files_from_root():
|
||||
"""Move files from root to subdirectories."""
|
||||
print("\n📦 Moving files from root level...")
|
||||
|
||||
moved_count = 0
|
||||
for target_dir, files in MIGRATION_PLAN.items():
|
||||
target_path = f'tools/scripts/{target_dir}'
|
||||
|
||||
for filename in files:
|
||||
source = f'tools/scripts/{filename}'
|
||||
dest = f'{target_path}/{filename}'
|
||||
|
||||
if os.path.exists(source):
|
||||
shutil.move(source, dest)
|
||||
print(f" ✅ {filename} → {target_dir}/")
|
||||
moved_count += 1
|
||||
else:
|
||||
print(f" ⚠️ {filename} not found (may already be moved)")
|
||||
|
||||
print(f"\n Moved {moved_count} files from root")
|
||||
|
||||
|
||||
def move_files_between_subdirs():
|
||||
"""Move files between existing subdirectories."""
|
||||
print("\n🔄 Consolidating files between subdirectories...")
|
||||
|
||||
moved_count = 0
|
||||
for target_dir, moves in EXISTING_SUBDIRECTORY_MOVES.items():
|
||||
target_path = f'tools/scripts/{target_dir}'
|
||||
|
||||
for source_rel, dest_name in moves:
|
||||
source = f'tools/scripts/{source_rel}'
|
||||
dest = f'{target_path}/{dest_name}'
|
||||
|
||||
if os.path.exists(source):
|
||||
shutil.move(source, dest)
|
||||
print(f" ✅ {source_rel} → {target_dir}/{dest_name}")
|
||||
moved_count += 1
|
||||
else:
|
||||
print(f" ⚠️ {source_rel} not found")
|
||||
|
||||
print(f"\n Moved {moved_count} files between subdirectories")
|
||||
|
||||
|
||||
def update_precommit_config():
|
||||
"""Update .pre-commit-config.yaml with new paths."""
|
||||
print("\n⚙️ Updating .pre-commit-config.yaml...")
|
||||
|
||||
config_path = '.pre-commit-config.yaml'
|
||||
|
||||
with open(config_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
original_content = content
|
||||
updates_made = 0
|
||||
|
||||
for old_path, new_path in PRECOMMIT_UPDATES.items():
|
||||
if old_path in content:
|
||||
content = content.replace(old_path, new_path)
|
||||
updates_made += 1
|
||||
print(f" ✅ Updated: {os.path.basename(old_path)}")
|
||||
|
||||
if updates_made > 0:
|
||||
with open(config_path, 'w') as f:
|
||||
f.write(content)
|
||||
print(f"\n Updated {updates_made} references in pre-commit config")
|
||||
else:
|
||||
print(" ℹ️ No updates needed")
|
||||
|
||||
|
||||
def create_readme_files():
|
||||
"""Create README files for new directories."""
|
||||
print("\n📝 Creating README files...")
|
||||
|
||||
readmes = {
|
||||
'images': '''# Image Management Scripts
|
||||
|
||||
Scripts for managing, processing, and validating images in the book.
|
||||
|
||||
## Image Processing
|
||||
- `compress_images.py` - Compress images to reduce file size
|
||||
- `convert_svg_to_png.py` - Convert SVG files to PNG format
|
||||
- `remove_bg.py` - Remove backgrounds from images
|
||||
|
||||
## Image Management
|
||||
- `manage_images.py` - Main image management utility
|
||||
- `download_external_images.py` - Download external images
|
||||
- `manage_external_images.py` - Manage external image references
|
||||
- `rename_auto_images.py` - Rename automatically generated images
|
||||
- `rename_downloaded_images.py` - Rename downloaded images
|
||||
|
||||
## Validation
|
||||
- `validate_image_references.py` - Ensure all image references are valid
|
||||
- `analyze_image_sizes.py` - Analyze image sizes and suggest optimizations
|
||||
''',
|
||||
'infrastructure': '''# Infrastructure Scripts
|
||||
|
||||
Scripts for managing CI/CD, containers, and workflow infrastructure.
|
||||
|
||||
## Container Management
|
||||
- `cleanup_containers.py` - Clean up Docker containers
|
||||
- `list_containers.py` - List active containers
|
||||
|
||||
## Workflow Management
|
||||
- `cleanup_workflow_runs_gh.py` - Clean up old GitHub Actions workflow runs
|
||||
''',
|
||||
}
|
||||
|
||||
for dirname, content in readmes.items():
|
||||
readme_path = f'tools/scripts/{dirname}/README.md'
|
||||
if not os.path.exists(readme_path):
|
||||
with open(readme_path, 'w') as f:
|
||||
f.write(content)
|
||||
print(f" ✅ Created {dirname}/README.md")
|
||||
|
||||
|
||||
def generate_summary():
|
||||
"""Generate a summary of the reorganization."""
|
||||
print("\n" + "=" * 80)
|
||||
print("📊 REORGANIZATION SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
# Count files in each directory
|
||||
summary = {}
|
||||
for root, dirs, files in os.walk('tools/scripts'):
|
||||
# Skip __pycache__ and hidden directories
|
||||
if '__pycache__' in root or '/.backup' in root:
|
||||
continue
|
||||
|
||||
dirname = os.path.relpath(root, 'tools/scripts')
|
||||
py_files = [f for f in files if f.endswith('.py') and f != '__init__.py']
|
||||
|
||||
if py_files:
|
||||
summary[dirname] = len(py_files)
|
||||
|
||||
print("\n📁 Files per directory:")
|
||||
for dirname in sorted(summary.keys()):
|
||||
count = summary[dirname]
|
||||
print(f" {dirname + '/':<30} {count:>3} files")
|
||||
|
||||
# Count root level files
|
||||
root_files = [f for f in os.listdir('tools/scripts')
|
||||
if f.endswith('.py') and os.path.isfile(f'tools/scripts/{f}')]
|
||||
|
||||
print(f"\n🎯 Root level scripts remaining: {len(root_files)}")
|
||||
if root_files:
|
||||
for f in root_files:
|
||||
print(f" - {f}")
|
||||
|
||||
print("\n✅ Reorganization complete!")
|
||||
print("\nNext steps:")
|
||||
print(" 1. Test pre-commit hooks: pre-commit run --all-files")
|
||||
print(" 2. Check for any broken imports")
|
||||
print(" 3. Update any documentation references")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main reorganization process."""
|
||||
import sys
|
||||
|
||||
print("🔧 SCRIPT REORGANIZATION TOOL")
|
||||
print("=" * 80)
|
||||
print("\nThis will reorganize tools/scripts/ directory structure.")
|
||||
print("\nChanges:")
|
||||
print(" • Create new subdirectories (images/, infrastructure/)")
|
||||
print(" • Move 21 scripts from root to appropriate subdirectories")
|
||||
print(" • Consolidate scattered scripts (images, tests, etc.)")
|
||||
print(" • Update .pre-commit-config.yaml references")
|
||||
print(" • Create documentation")
|
||||
|
||||
# Check for --yes flag
|
||||
if '--yes' not in sys.argv:
|
||||
response = input("\n⚠️ Proceed with reorganization? (yes/no): ")
|
||||
if response.lower() != 'yes':
|
||||
print("\n❌ Cancelled")
|
||||
return 1
|
||||
else:
|
||||
print("\n✅ Auto-confirmed with --yes flag")
|
||||
|
||||
try:
|
||||
# Create backup
|
||||
backup_file = create_backup()
|
||||
|
||||
# Execute reorganization
|
||||
create_directories()
|
||||
move_files_from_root()
|
||||
move_files_between_subdirs()
|
||||
update_precommit_config()
|
||||
create_readme_files()
|
||||
|
||||
# Summary
|
||||
generate_summary()
|
||||
|
||||
print(f"\n💾 Backup saved to: {backup_file}")
|
||||
print(" (Can be used for rollback if needed)")
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error during reorganization: {e}")
|
||||
print(" Please restore from backup if needed")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
exit(main())
|
||||
|
||||
Reference in New Issue
Block a user