diff --git a/.github/workflows/book-build-container.yml b/.github/workflows/book-build-container.yml index e28e6f328..ff65d35cd 100644 --- a/.github/workflows/book-build-container.yml +++ b/.github/workflows/book-build-container.yml @@ -130,26 +130,6 @@ on: linux_epub_vol2_artifact: description: "Linux EPUB artifact name (Volume II)" value: ${{ jobs.collect-outputs.outputs.linux_epub_vol2_artifact }} - # Windows Volume I artifacts - windows_html_vol1_artifact: - description: "Windows HTML artifact name (Volume I)" - value: ${{ jobs.collect-outputs.outputs.windows_html_vol1_artifact }} - windows_pdf_vol1_artifact: - description: "Windows PDF artifact name (Volume I)" - value: ${{ jobs.collect-outputs.outputs.windows_pdf_vol1_artifact }} - windows_epub_vol1_artifact: - description: "Windows EPUB artifact name (Volume I)" - value: ${{ jobs.collect-outputs.outputs.windows_epub_vol1_artifact }} - # Windows Volume II artifacts - windows_html_vol2_artifact: - description: "Windows HTML artifact name (Volume II)" - value: ${{ jobs.collect-outputs.outputs.windows_html_vol2_artifact }} - windows_pdf_vol2_artifact: - description: "Windows PDF artifact name (Volume II)" - value: ${{ jobs.collect-outputs.outputs.windows_pdf_vol2_artifact }} - windows_epub_vol2_artifact: - description: "Windows EPUB artifact name (Volume II)" - value: ${{ jobs.collect-outputs.outputs.windows_epub_vol2_artifact }} permissions: contents: read @@ -377,19 +357,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: ๐Ÿณ Ensure Docker daemon (Windows) - if: matrix.platform == 'windows' && matrix.enabled - shell: pwsh - run: | - $ErrorActionPreference = 'Stop' - $dockerService = Get-Service -Name docker -ErrorAction Stop - - if ($dockerService.Status -ne 'Running') { - Write-Host "Starting Docker service..." - Start-Service -Name docker - } else { - Write-Host "Docker service already running." - } $maxAttempts = 12 for ($i = 1; $i -le $maxAttempts; $i++) { @@ -405,9 +372,6 @@ jobs: } } - - name: ๐Ÿณ Pull Docker Image - if: matrix.platform == 'windows' && matrix.enabled - run: docker pull ${{ env.CONTAINER_IMAGE }} - name: ๐Ÿงช Preflight toolchain (Linux) if: matrix.platform == 'linux' && matrix.enabled @@ -454,84 +418,6 @@ jobs: echo "โœ… Linux toolchain preflight checks passed" - - name: ๐Ÿงช Preflight toolchain (Windows) - if: matrix.platform == 'windows' && matrix.enabled - shell: pwsh - run: | - Write-Host "๐Ÿงช Running Windows toolchain preflight checks..." - $preflightScript = @" - - `$ErrorActionPreference = 'Stop' - `$PSNativeCommandUseErrorActionPreference = `$true - - function Invoke-Check { - param( - [string]`$Name, - [scriptblock]`$Action - ) - Write-Host \"โ–ถ preflight: `$Name\" - try { - & `$Action - if (-not `$?) { throw 'Command returned non-zero status' } - Write-Host \"โœ… preflight: `$Name\" - } catch { - `$errorMessage = `$_.Exception.Message - Write-Error (\"โŒ preflight failed: `$Name -- {0}\" -f `$errorMessage) - throw - } - } - - Invoke-Check 'quarto on PATH' { `$resolved = Get-Command quarto -ErrorAction Stop; Write-Host (\" quarto -> {0}\" -f `$resolved.Source) } - Invoke-Check 'quarto version' { quarto --version | Select-Object -First 1 } - Invoke-Check 'pandoc available' { - `$pandocCmd = Get-Command pandoc -ErrorAction SilentlyContinue - if (`$pandocCmd) { - Write-Host (\" pandoc -> {0}\" -f `$pandocCmd.Source) - } else { - quarto pandoc --version | Select-Object -First 1 | Out-Null - Write-Host \" pandoc -> bundled via quarto\" - } - } - Invoke-Check 'pandoc version' { - `$pandocCmd = Get-Command pandoc -ErrorAction SilentlyContinue - if (`$pandocCmd) { - pandoc --version | Select-Object -First 1 - } else { - quarto pandoc --version | Select-Object -First 1 - } - } - Invoke-Check 'python on PATH' { `$resolved = Get-Command python -ErrorAction Stop; Write-Host (\" python -> {0}\" -f `$resolved.Source) } - Invoke-Check 'python version' { python --version } - Invoke-Check 'python3 on PATH' { `$resolved = Get-Command python3 -ErrorAction Stop; Write-Host (\" python3 -> {0}\" -f `$resolved.Source) } - Invoke-Check 'python3 version' { python3 --version } - Invoke-Check 'mlsysim import' { python -c 'import mlsysim,sys; print(\"mlsysim:\", mlsysim.__file__); print(\"python:\", sys.executable)' } - Invoke-Check 'Rscript on PATH' { `$resolved = Get-Command Rscript -ErrorAction Stop; Write-Host (\" Rscript -> {0}\" -f `$resolved.Source) } - Invoke-Check 'Rscript version' { Rscript --version | Select-Object -First 1 } - Invoke-Check 'inkscape on PATH' { `$resolved = Get-Command inkscape -ErrorAction Stop; Write-Host (\" inkscape -> {0}\" -f `$resolved.Source) } - Invoke-Check 'inkscape version' { inkscape --version | Select-Object -First 1 } - - if ('${{ matrix.format_name }}' -eq 'PDF') { - Invoke-Check 'lualatex on PATH' { `$resolved = Get-Command lualatex -ErrorAction Stop; Write-Host (\" lualatex -> {0}\" -f `$resolved.Source) } - Invoke-Check 'lualatex version' { lualatex --version | Select-Object -First 1 } - Invoke-Check 'ghostscript on PATH' { - `$gsCmd = Get-Command gs -ErrorAction SilentlyContinue - if (-not `$gsCmd) { `$gsCmd = Get-Command gswin64c -ErrorAction SilentlyContinue } - if (-not `$gsCmd) { throw 'Ghostscript not found (expected gs or gswin64c)' } - Write-Host (\" ghostscript -> {0}\" -f `$gsCmd.Source) - } - Invoke-Check 'ghostscript version' { - `$gsCmd = Get-Command gs -ErrorAction SilentlyContinue - if (-not `$gsCmd) { `$gsCmd = Get-Command gswin64c -ErrorAction SilentlyContinue } - & `$gsCmd.Source --version - } - } - - if ('${{ matrix.format_name }}' -eq 'EPUB') { - Invoke-Check 'Pillow import' { python -c 'import PIL; print(\"Pillow:\", PIL.__version__)' } - } - "@ - $preflightScript | docker run --rm -e PYTHONPATH=C:\workspace -v "$($PWD.Path):C:\workspace" -w "C:\workspace\book\quarto" ${{ env.CONTAINER_IMAGE }} -i pwsh -NoLogo -Command - - Write-Host "โœ… Windows toolchain preflight checks passed" - name: ๐Ÿ”จ Build ${{ matrix.format_name }} (Linux) if: matrix.platform == 'linux' && matrix.enabled @@ -545,19 +431,6 @@ jobs: quarto render --to ${{ matrix.render_target }} --output-dir "${{ matrix.output_dir }}" echo "โœ… ${{ matrix.format_name }} build completed" - - name: ๐Ÿ”จ Build ${{ matrix.format_name }} (Windows) - if: matrix.platform == 'windows' && matrix.enabled - shell: pwsh - run: | - Write-Host "๐Ÿ”จ Building ${{ matrix.format_name }} on Windows container..." - $buildScript = @" - - if (Test-Path '_quarto.yml') { Remove-Item '_quarto.yml' -Force } - Copy-Item 'config\${{ matrix.config }}' '_quarto.yml' -Force - quarto render --to ${{ matrix.render_target }} --output-dir '${{ matrix.output_dir }}' - "@ - $buildScript | docker run --rm -e PYTHONPATH=C:\workspace -v "$($PWD.Path):C:\workspace" -w "C:\workspace\book\quarto" ${{ env.CONTAINER_IMAGE }} -i pwsh -NoLogo -Command - - Write-Host "โœ… ${{ matrix.format_name }} build completed" - name: ๐Ÿ“‰ Compress PDF (Linux) if: matrix.platform == 'linux' && matrix.format_name == 'PDF' && matrix.enabled @@ -594,45 +467,6 @@ jobs: echo "โš ๏ธ PDF file not found for compression" fi - - name: ๐Ÿ“‰ Compress PDF (Windows) - if: matrix.platform == 'windows' && matrix.format_name == 'PDF' && matrix.enabled - shell: pwsh - run: | - Write-Host "๐Ÿ”จ Compressing PDF on Windows container..." - $compressPdfScript = @" - - if (Test-Path 'Machine-Learning-Systems.pdf') { - Write-Host '๐Ÿ“‰ Compressing PDF with professional compression tool...' - Write-Host '๐Ÿ” DEBUG: Current directory:' - Get-Location - Write-Host '๐Ÿ” DEBUG: Windows environment info:' - Write-Host ("OS: {0}" -f [System.Environment]::OSVersion.VersionString) - Write-Host ("Architecture: {0}" -f [System.Environment]::Is64BitOperatingSystem) - Write-Host '๐Ÿ” DEBUG: Verifying Python and Pillow installation:' - `$py_version = (python --version 2>&1).Trim() - Write-Host " ๐Ÿ Python Version: `$py_version" - `$py_path = (python -c 'import sys; print(sys.executable)').Trim() - Write-Host " ๐Ÿ Python Path: `$py_path" - `$pillow_version = (python -c 'import PIL; print(PIL.__version__)').Trim() - Write-Host " โœ… Pillow Version: `$pillow_version" - Write-Host '๐Ÿ” DEBUG: Checking for script at relative path:' - if (Test-Path '..\\..\\publish\\compress_pdf.py') { - Write-Host 'โœ… Found script at ..\\..\\publish\\compress_pdf.py' - python ..\..\publish\compress_pdf.py --input 'Machine-Learning-Systems.pdf' --output 'compressed.pdf' --quality minimal --verbose - } else { - Write-Host '๐Ÿ”„ Trying fallback path: C:\workspace\book\quarto\publish\compress_pdf.py' - python C:\workspace\book\quarto\publish\compress_pdf.py --input 'Machine-Learning-Systems.pdf' --output 'compressed.pdf' --quality minimal --verbose - } - if (Test-Path 'compressed.pdf') { - Move-Item -Force 'compressed.pdf' 'Machine-Learning-Systems.pdf' - Write-Host 'โœ… PDF compression completed' - } - } else { - Write-Warning 'โš ๏ธ Machine-Learning-Systems.pdf not found for compression.' - } - "@ - $compressPdfScript | docker run --rm -e PYTHONPATH=C:\workspace -v "$($PWD.Path):C:\workspace" -w "C:\workspace\book\quarto\${{ matrix.output_dir }}" ${{ env.CONTAINER_IMAGE }} -i pwsh -NoLogo -Command - - Write-Host "โœ… PDF compression completed." - name: ๐Ÿ“š Compress EPUB (Linux) if: matrix.platform == 'linux' && matrix.format_name == 'EPUB' && matrix.enabled @@ -671,45 +505,6 @@ jobs: echo "โš ๏ธ EPUB file not found for compression" fi - - name: ๐Ÿ“š Compress EPUB (Windows) - if: matrix.platform == 'windows' && matrix.format_name == 'EPUB' && matrix.enabled - shell: pwsh - run: | - Write-Host "๐Ÿ”จ Compressing EPUB on Windows container..." - $compressEpubScript = @" - - if (Test-Path 'Machine-Learning-Systems.epub') { - Write-Host '๐Ÿ“š Compressing EPUB with optimized compression tool...' - Write-Host '๐Ÿ” DEBUG: Current directory:' - Get-Location - Write-Host '๐Ÿ” DEBUG: Windows environment info:' - Write-Host ("OS: {0}" -f [System.Environment]::OSVersion.VersionString) - Write-Host ("Architecture: {0}" -f [System.Environment]::Is64BitOperatingSystem) - Write-Host '๐Ÿ” DEBUG: Verifying Python and Pillow installation:' - `$py_version = (python --version 2>&1).Trim() - Write-Host " ๐Ÿ Python Version: `$py_version" - `$py_path = (python -c 'import sys; print(sys.executable)').Trim() - Write-Host " ๐Ÿ Python Path: `$py_path" - `$pillow_version = (python -c 'import PIL; print(PIL.__version__)').Trim() - Write-Host " โœ… Pillow Version: `$pillow_version" - Write-Host '๐Ÿ” DEBUG: Checking for script at relative path:' - if (Test-Path '..\\..\\publish\\compress_epub.py') { - Write-Host 'โœ… Found script at ..\\..\\publish\\compress_epub.py' - python ..\..\publish\compress_epub.py --input 'Machine-Learning-Systems.epub' --output 'compressed.epub' --verbose - } else { - Write-Host '๐Ÿ”„ Trying fallback path: C:\workspace\book\quarto\publish\compress_epub.py' - python C:\workspace\book\quarto\publish\compress_epub.py --input 'Machine-Learning-Systems.epub' --output 'compressed.epub' --verbose - } - if (Test-Path 'compressed.epub') { - Move-Item -Force 'compressed.epub' 'Machine-Learning-Systems.epub' - Write-Host 'โœ… EPUB compression completed (using optimized defaults: quality=50, max-size=1000px)' - } - } else { - Write-Warning 'โš ๏ธ Machine-Learning-Systems.epub not found for compression.' - } - "@ - $compressEpubScript | docker run --rm -e PYTHONPATH=C:\workspace -v "$($PWD.Path):C:\workspace" -w "C:\workspace\book\quarto\${{ matrix.output_dir }}" ${{ env.CONTAINER_IMAGE }} -i pwsh -NoLogo -Command - - Write-Host "โœ… EPUB compression completed." - name: ๐Ÿ“ค Upload artifact uses: actions/upload-artifact@v6 diff --git a/.github/workflows/book-publish-live.yml b/.github/workflows/book-publish-live.yml index c98ed0f1b..3a662ddbc 100644 --- a/.github/workflows/book-publish-live.yml +++ b/.github/workflows/book-publish-live.yml @@ -38,9 +38,6 @@ env: # main-html-linux: Contains build/html/ (web version) # main-pdf-linux: Contains build/pdf/Machine-Learning-Systems.pdf # main-epub-linux: Contains build/epub/Machine-Learning-Systems.epub -# main-html-windows: Contains build/html/ (web version) -# main-pdf-windows: Contains build/pdf/Machine-Learning-Systems.pdf -# main-epub-windows: Contains build/epub/Machine-Learning-Systems.epub # # This workflow downloads the artifact manifest first to get the exact names, # then downloads the HTML, PDF, and EPUB artifacts using those names for coordination. @@ -757,7 +754,6 @@ jobs: uses: ./.github/workflows/book-build-container.yml with: build_linux: true # Production builds Linux only for now - build_windows: false build_html: true # HTML + PDF + EPUB for production build_pdf: true build_epub: true diff --git a/.github/workflows/book-validate-dev.yml b/.github/workflows/book-validate-dev.yml index 506528af1..e175daedf 100644 --- a/.github/workflows/book-validate-dev.yml +++ b/.github/workflows/book-validate-dev.yml @@ -111,7 +111,6 @@ jobs: uses: ./.github/workflows/infra-health-check.yml # Shared workflow - no book- prefix with: test_linux: true - test_windows: false container_registry: 'ghcr.io' container_tag: 'latest' @@ -217,7 +216,6 @@ jobs: uses: ./.github/workflows/book-build-container.yml with: build_linux: ${{ needs.build-config.outputs.build_linux == 'true' }} - build_windows: false build_html: ${{ needs.build-config.outputs.build_html == 'true' }} build_pdf: ${{ needs.build-config.outputs.build_pdf == 'true' }} build_epub: ${{ needs.build-config.outputs.build_epub == 'true' }} diff --git a/.github/workflows/infra-container-windows.yml b/.github/workflows/infra-container-windows.yml deleted file mode 100644 index 45e2d1228..000000000 --- a/.github/workflows/infra-container-windows.yml +++ /dev/null @@ -1,245 +0,0 @@ -name: '๐Ÿ”ง Infra ยท ๐Ÿณ Container (Windows)' - -# This workflow builds the Windows Quarto build container -# Windows containers are more complex but provide performance benefits - -# Prevent multiple builds running simultaneously -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - inputs: - force_rebuild: - description: 'Force rebuild even if no changes' - required: false - default: false - type: boolean - no_cache: - description: 'Disable Docker build cache (fresh build)' - required: false - default: false - type: boolean - container_registry: - description: 'Container registry URL' - required: false - default: 'ghcr.io' - type: string - container_name: - description: 'Container image name' - required: false - default: 'quarto-windows' - type: string - container_tag: - description: 'Container tag' - required: false - default: 'latest' - type: string - workflow_call: - inputs: - force_rebuild: - required: false - default: false - type: boolean - no_cache: - required: false - default: false - type: boolean - container_registry: - required: false - default: 'ghcr.io' - type: string - container_name: - required: false - default: 'quarto-windows' - type: string - container_tag: - required: false - default: 'latest' - type: string - - outputs: - build-status: - description: "Container build status (success/failure/skipped)" - value: ${{ jobs.build.outputs.build-status }} - image-name: - description: "Full container image name with registry" - value: ${{ jobs.build.outputs.image-name }} - image-digest: - description: "Container image digest (SHA256)" - value: ${{ jobs.build.outputs.image-digest }} - cache-hit: - description: "Whether build used cache (true/false)" - value: ${{ jobs.build.outputs.cache-hit }} - - # Re-enable automatic triggers - schedule: - - cron: '0 2 * * 0' # Weekly rebuild (Sunday at 2am - after Linux container) - push: - branches: [dev] # Only trigger on dev branch, not main - paths: - - 'book/tools/dependencies/**' - - 'book/docker/windows/**' - - '.github/workflows/book-build-windows-container.yml' - -env: - # ============================================================================= - # PATH CONFIGURATION - Uses GitHub Repository Variables (Settings > Variables) - # ============================================================================= - # MLSysBook content lives under book/ to accommodate TinyTorch at root - # Use ${{ vars.BOOK_ROOT }}, ${{ vars.BOOK_DOCKER }}, etc. in workflow steps - # Variables: BOOK_ROOT, BOOK_DOCKER, BOOK_TOOLS, BOOK_QUARTO, BOOK_DEPS - - # Container Registry Configuration (configurable via inputs) - REGISTRY: ${{ (github.event_name == 'workflow_dispatch' && inputs.container_registry) || 'ghcr.io' }} - IMAGE_NAME: ${{ github.repository }}/${{ (github.event_name == 'workflow_dispatch' && inputs.container_name) || 'quarto-windows' }} - CONTAINER_TAG: ${{ (github.event_name == 'workflow_dispatch' && inputs.container_tag) || 'latest' }} - - # Container Build Configuration - # Using vars.BOOK_DOCKER (repository variable) - works in all contexts - DOCKERFILE_PATH: ./${{ vars.BOOK_DOCKER }}/windows/Dockerfile - CONTEXT_PATH: . - -jobs: - build: - runs-on: windows-latest - if: github.repository_owner == 'harvard-edge' - timeout-minutes: 180 # takes about 2 hours to build on Windows - permissions: - contents: read - packages: write - - outputs: - build-status: ${{ steps.build.outputs.build-status }} - image-name: ${{ steps.build.outputs.image-name }} - image-digest: ${{ steps.build.outputs.image-digest }} - cache-hit: ${{ steps.build.outputs.cache-hit }} - - steps: - - name: ๐Ÿ“ฅ Checkout repository - uses: actions/checkout@v6 - - # Skip Docker Buildx for Windows containers - use native Docker engine - # Buildx doesn't properly support Windows containers on GitHub Actions - # - name: ๐Ÿ› ๏ธ Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - - name: ๐Ÿ” Log in to Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: ๐Ÿท๏ธ Extract metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=${{ env.CONTAINER_TAG }} - - - name: ๐Ÿณ Build and Push Windows container - id: build - shell: pwsh - # Use native Docker instead of book/docker/build-push-action for Windows containers - # Buildx has compatibility issues with Windows containers on GitHub Actions - run: | - # Extract image name and tag from metadata - $IMAGE_TAG = "${{ steps.meta.outputs.tags }}" - $NO_CACHE = "${{ github.event_name == 'workflow_dispatch' && inputs.no_cache || false }}" - - Write-Host "๐Ÿ”จ Building Windows container..." - Write-Host "๐Ÿ“Š Image: $IMAGE_TAG" - Write-Host "๐Ÿ“Š Context: ${{ env.CONTEXT_PATH }}" - Write-Host "๐Ÿ“Š Dockerfile: ${{ env.DOCKERFILE_PATH }}" - Write-Host "๐Ÿ“Š No Cache: $NO_CACHE" - - # Build the container using native Docker - $buildArgs = @( - "build", - "--file", "${{ env.DOCKERFILE_PATH }}", - "--tag", $IMAGE_TAG - ) - - # Add no-cache flag if requested - if ($NO_CACHE -eq "true") { - $buildArgs += "--no-cache" - Write-Host "๐Ÿšซ Cache disabled - building from scratch" - } else { - Write-Host "๐Ÿ’พ Using Docker cache" - } - - # Add labels from metadata - $labels = "${{ steps.meta.outputs.labels }}" - if ($labels) { - $labels -split "`n" | ForEach-Object { - if ($_.Trim()) { - $buildArgs += "--label", $_.Trim() - } - } - } - - # Add context path - $buildArgs += "${{ env.CONTEXT_PATH }}" - - Write-Host "๐Ÿ”จ Running: docker $($buildArgs -join ' ')" - & docker @buildArgs - - if ($LASTEXITCODE -ne 0) { - Write-Host "โŒ Docker build failed with exit code $LASTEXITCODE" - exit $LASTEXITCODE - } - - Write-Host "โœ… Build completed successfully" - - # Push the container - Write-Host "๐Ÿ“ค Pushing container to registry..." - & docker push $IMAGE_TAG - - if ($LASTEXITCODE -ne 0) { - Write-Host "โŒ Docker push failed with exit code $LASTEXITCODE" - exit $LASTEXITCODE - } - - Write-Host "โœ… Push completed successfully" - - # Get image digest for output - $DIGEST = & docker inspect --format='{{index .RepoDigests 0}}' $IMAGE_TAG 2>$null - if ($DIGEST -match '@(.+)$') { - $DIGEST = $matches[1] - } else { - $DIGEST = "unknown" - } - - # Set outputs for build summary - "digest=$DIGEST" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - "cache-hit=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - - - name: ๐Ÿ“Š Build Summary - id: build-summary - if: always() - shell: pwsh - run: | - # Determine build status - if ("${{ steps.build.outcome }}" -eq "success") { - $BUILD_STATUS = "success" - } else { - $BUILD_STATUS = "failure" - } - - # Extract build information - $IMAGE_NAME = "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.CONTAINER_TAG }}" - $IMAGE_DIGEST = "${{ steps.build.outputs.digest }}" - $CACHE_HIT = "${{ steps.build.outputs.cache-hit }}" - - "build-status=$BUILD_STATUS" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - "image-name=$IMAGE_NAME" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - "image-digest=$IMAGE_DIGEST" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - "cache-hit=$CACHE_HIT" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - - Write-Host "๐Ÿ“Š Build Status: $BUILD_STATUS" - Write-Host "๐Ÿณ Image: $IMAGE_NAME" - Write-Host "๐Ÿ” Digest: $IMAGE_DIGEST" - Write-Host "๐Ÿ’พ Cache Hit: $CACHE_HIT" diff --git a/.github/workflows/infra-health-check.yml b/.github/workflows/infra-health-check.yml index 86445dab0..6512e10e0 100644 --- a/.github/workflows/infra-health-check.yml +++ b/.github/workflows/infra-health-check.yml @@ -66,10 +66,8 @@ env: REGISTRY: ${{ inputs.container_registry || 'ghcr.io' }} CONTAINER_TAG: ${{ inputs.container_tag || 'latest' }} LINUX_CONTAINER_NAME: 'quarto-linux' - WINDOWS_CONTAINER_NAME: 'quarto-windows' # Computed full image names LINUX_IMAGE: ${{ inputs.container_registry || 'ghcr.io' }}/${{ github.repository }}/quarto-linux:${{ inputs.container_tag || 'latest' }} - WINDOWS_IMAGE: ${{ inputs.container_registry || 'ghcr.io' }}/${{ github.repository }}/quarto-windows:${{ inputs.container_tag || 'latest' }} jobs: # Matrix strategy for both container platforms @@ -85,9 +83,6 @@ jobs: platform: linux container_name: quarto-linux shell: bash - - os: windows-latest - platform: windows - container_name: quarto-windows shell: pwsh env: @@ -100,7 +95,6 @@ jobs: - name: ๐Ÿ“ฅ Checkout repository if: | (matrix.platform == 'linux' && inputs.test_linux != false) || - (matrix.platform == 'windows' && inputs.test_windows != false) uses: actions/checkout@v6 with: fetch-depth: 0 @@ -108,7 +102,6 @@ jobs: - name: ๐Ÿ”‘ Log in to GitHub Container Registry if: | (matrix.platform == 'linux' && inputs.test_linux != false) || - (matrix.platform == 'windows' && inputs.test_windows != false) uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} @@ -118,13 +111,11 @@ jobs: - name: ๐Ÿณ Pull Docker Image if: | (matrix.platform == 'linux' && inputs.test_linux != false) || - (matrix.platform == 'windows' && inputs.test_windows != false) run: docker pull ${{ env.CONTAINER_IMAGE }} - name: ๐Ÿ“Š Container Information if: | (matrix.platform == 'linux' && inputs.test_linux != false) || - (matrix.platform == 'windows' && inputs.test_windows != false) run: | echo "๐Ÿ“Š === CONTAINER INFORMATION ===" echo "๐Ÿ“‹ Platform: ${{ matrix.platform }}" @@ -299,239 +290,6 @@ jobs: echo "๐Ÿ“‹ Linux tool versions saved to linux-tool-versions.log" - - name: ๐ŸชŸ Windows Container - Tool Version Check - if: matrix.platform == 'windows' && inputs.test_windows != false - run: | - Write-Output "๐ŸชŸ === WINDOWS CONTAINER TOOL VERSIONS ===" - Write-Output "๐Ÿ“‹ Testing Windows tools with enhanced path detection using 'scoop which':" - Write-Output "==============================================" - - docker run --rm ${{ env.CONTAINER_IMAGE }} pwsh -Command " - Write-Output '๐Ÿ” === WINDOWS CONTAINER TOOL VERSIONS ===' - Write-Output '' - - # Check Scoop environment - Write-Output '๐Ÿ“ฆ SCOOP ENVIRONMENT:' - Write-Output '--------------------' - if (Test-Path 'C:\Users\ContainerAdministrator\scoop') { - Write-Output 'โœ… Scoop directory found' - if (Test-Path 'C:\Users\ContainerAdministrator\scoop\shims') { - `$shimCount = (Get-ChildItem 'C:\Users\ContainerAdministrator\scoop\shims' -File | Measure-Object).Count - Write-Output `"๐Ÿ“ Scoop shims directory: `$shimCount files`" - } - } else { - Write-Output 'โŒ Scoop directory not found' - } - Write-Output '' - - # --- Tool Checks --- - - Write-Output '๐Ÿ“Š QUARTO:' - Write-Output '----------' - try { - `$quartoPath = (scoop which quarto 2>`$null).Trim() - if (`$LASTEXITCODE -ne 0) { throw } - Write-Output `"๐Ÿ“ Location: `$quartoPath`" - Write-Output '๐Ÿ“ฆ Method: scoop which' - `$quartoVersion = & `$quartoPath --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$quartoVersion`" - Write-Output 'โœ… Status: OK' - } catch { - try { - `$quartoPath = (Get-Command quarto -ErrorAction Stop).Source - Write-Output `"๐Ÿ“ Location: `$quartoPath`" - Write-Output '๐Ÿ“ฆ Method: Get-Command (PATH)' - `$quartoVersion = & `$quartoPath --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$quartoVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - } - Write-Output '' - - Write-Output '๐Ÿ“Š PYTHON:' - Write-Output '-----------' - try { - `$pythonPath = (scoop which python 2>`$null).Trim() - if (`$LASTEXITCODE -ne 0) { throw } - Write-Output `"๐Ÿ“ Location: `$pythonPath`" - Write-Output '๐Ÿ“ฆ Method: scoop which' - `$pythonVersion = & `$pythonPath --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$pythonVersion`" - Write-Output 'โœ… Status: OK' - } catch { - try { - `$pythonPath = (Get-Command python -ErrorAction Stop).Source - Write-Output `"๐Ÿ“ Location: `$pythonPath`" - Write-Output '๐Ÿ“ฆ Method: Get-Command (PATH)' - `$pythonVersion = & `$pythonPath --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$pythonVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - } - Write-Output '' - - Write-Output '๐Ÿ“Š R:' - Write-Output '------' - try { - `$rCmd = Get-Command R.exe -ErrorAction SilentlyContinue - if (-not `$rCmd) { `$rCmd = Get-Command R -ErrorAction Stop } - `$rPath = `$rCmd.Source - Write-Output `"๐Ÿ“ Location: `$rPath`" - Write-Output '๐Ÿ“ฆ Method: Get-Command (PATH)' - `$rVersion = & `$rCmd.Name --version 2>&1 | Select-Object -First 1 - Write-Output `"๐Ÿ“‹ Version: `$rVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - Write-Output '' - - Write-Output '๐Ÿ“Š LUALATEX:' - Write-Output '------------' - try { - `$lualatexPath = (Get-Command lualatex -ErrorAction Stop).Source - Write-Output `"๐Ÿ“ Location: `$lualatexPath`" - `$lualatexVersion = & `$lualatexPath --version 2>&1 | Select-Object -First 1 - Write-Output `"๐Ÿ“‹ Version: `$lualatexVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - Write-Output '' - - Write-Output '๐Ÿ“Š GHOSTSCRIPT:' - Write-Output '---------------' - try { - `$gsPath = (scoop which gswin64c 2>`$null).Trim() - if (`$LASTEXITCODE -ne 0) { `$gsPath = (scoop which gs 2>`$null).Trim() } - if (`$LASTEXITCODE -ne 0) { throw } - Write-Output `"๐Ÿ“ Location: `$gsPath`" - Write-Output '๐Ÿ“ฆ Method: scoop which' - `$gsVersion = & `$gsPath --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$gsVersion`" - Write-Output 'โœ… Status: OK' - } catch { - try { - `$gsCmd = Get-Command gswin64c -ErrorAction SilentlyContinue - if (-not `$gsCmd) { `$gsCmd = Get-Command gs -ErrorAction Stop } - `$gsPath = `$gsCmd.Source - Write-Output `"๐Ÿ“ Location: `$gsPath`" - Write-Output '๐Ÿ“ฆ Method: Get-Command (PATH)' - `$gsVersion = & `$gsCmd.Name --version 2>&1 - Write-Output `"๐Ÿ“‹ Version: `$gsVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - } - Write-Output '' - - Write-Output '๐Ÿ“Š INKSCAPE:' - Write-Output '------------' - try { - `$inkscapePath = (scoop which inkscape 2>`$null).Trim() - if (`$LASTEXITCODE -ne 0) { throw } - Write-Output `"๐Ÿ“ Location: `$inkscapePath`" - Write-Output '๐Ÿ“ฆ Method: scoop which' - `$inkscapeVersion = & `$inkscapePath --version 2>&1 | Select-Object -First 1 - Write-Output `"๐Ÿ“‹ Version: `$inkscapeVersion`" - Write-Output 'โœ… Status: OK' - } catch { - try { - `$inkscapePath = (Get-Command inkscape -ErrorAction Stop).Source - Write-Output `"๐Ÿ“ Location: `$inkscapePath`" - Write-Output '๐Ÿ“ฆ Method: Get-Command (PATH)' - `$inkscapeVersion = & `$inkscapePath --version 2>&1 | Select-Object -First 1 - Write-Output `"๐Ÿ“‹ Version: `$inkscapeVersion`" - Write-Output 'โœ… Status: OK' - } catch { - Write-Output 'โŒ Status: NOT FOUND' - } - } - Write-Output '' - - Write-Output '๐ŸŽฏ === WINDOWS TOOL CHECK COMPLETE ===' - Write-Output '' - - # Summary of all tool statuses - Write-Output '๐Ÿ“‹ WINDOWS TOOL STATUS SUMMARY:' - Write-Output '===============================' - - # Check each tool and show status - if ((scoop which quarto 2>`$null) -or (Get-Command quarto -ErrorAction SilentlyContinue)) { - Write-Output 'โœ… Quarto: AVAILABLE' - } else { - Write-Output 'โŒ Quarto: MISSING' - } - if ((scoop which python 2>`$null) -or (Get-Command python -ErrorAction SilentlyContinue)) { - Write-Output 'โœ… Python: AVAILABLE' - } else { - Write-Output 'โŒ Python: MISSING' - } - if ((Get-Command R.exe -ErrorAction SilentlyContinue) -or (Get-Command R -ErrorAction SilentlyContinue)) { - Write-Output 'โœ… R: AVAILABLE' - } else { - Write-Output 'โŒ R: MISSING' - } - if (Get-Command lualatex -ErrorAction SilentlyContinue) { - Write-Output 'โœ… LuaLaTeX: AVAILABLE' - } else { - Write-Output 'โŒ LuaLaTeX: MISSING' - } - if ((scoop which gswin64c 2>`$null) -or (scoop which gs 2>`$null) -or (Get-Command gswin64c -ErrorAction SilentlyContinue) -or (Get-Command gs -ErrorAction SilentlyContinue)) { - Write-Output 'โœ… Ghostscript: AVAILABLE' - } else { - Write-Output 'โŒ Ghostscript: MISSING' - } - if ((scoop which inkscape 2>`$null) -or (Get-Command inkscape -ErrorAction SilentlyContinue)) { - Write-Output 'โœ… Inkscape: AVAILABLE' - } else { - Write-Output 'โŒ Inkscape: MISSING' - } - - Write-Output '' - - # Check if any tools are missing and fail if so - `$FailedTools = 0 - - if (-not ((scoop which quarto 2>`$null) -or (Get-Command quarto -ErrorAction SilentlyContinue))) { `$FailedTools++ } - if (-not ((scoop which python 2>`$null) -or (Get-Command python -ErrorAction SilentlyContinue))) { `$FailedTools++ } - if (-not ((Get-Command R.exe -ErrorAction SilentlyContinue) -or (Get-Command R -ErrorAction SilentlyContinue))) { `$FailedTools++ } - if (-not (Get-Command lualatex -ErrorAction SilentlyContinue)) { `$FailedTools++ } - if (-not ((scoop which gswin64c 2>`$null) -or (scoop which gs 2>`$null) -or (Get-Command gswin64c -ErrorAction SilentlyContinue) -or (Get-Command gs -ErrorAction SilentlyContinue))) { `$FailedTools++ } - if (-not ((scoop which inkscape 2>`$null) -or (Get-Command inkscape -ErrorAction SilentlyContinue))) { `$FailedTools++ } - - if (`$FailedTools -eq 0) { - Write-Output '๐ŸŽฏ โœ… Windows container tool validation: PASSED' - Write-Output 'All essential Quarto build tools are available!' - } else { - Write-Output '๐ŸŽฏ โŒ Windows container tool validation: FAILED' - Write-Output "Missing `$FailedTools essential tools - container not ready!" - Write-Output 'This container is NOT ready for Quarto builds.' - exit 1 - } - Write-Output '' - - # List all Scoop apps for debugging - Write-Output '๐Ÿ“ฆ SCOOP INSTALLED APPS:' - Write-Output '------------------------' - if (Get-Command scoop -ErrorAction SilentlyContinue) { - try { - `$scoopApps = & scoop list 2>&1 - Write-Output `$scoopApps - } catch { - Write-Output 'Failed to list Scoop apps' - } - } else { - Write-Output 'Scoop command not found' - } - " | Tee-Object -FilePath "windows-tool-versions.log" - - Write-Output "๐Ÿ“‹ Windows tool versions saved to windows-tool-versions.log" - name: ๐Ÿง Linux Container - Quarto Check if: matrix.platform == 'linux' && inputs.test_linux != false @@ -599,146 +357,6 @@ jobs: fi fi - - name: ๐ŸชŸ Windows Container - Quarto Check - if: matrix.platform == 'windows' && inputs.test_windows != false - continue-on-error: true # Don't fail workflow if quarto check fails - run: | - Write-Output "๐ŸชŸ === WINDOWS QUARTO CHECK (COMPREHENSIVE) ===" - Write-Output "๐Ÿ“‹ Running quarto check to validate full installation:" - Write-Output "=====================================================" - - # 1) Emit the inner script to a file (no escaping headaches) - $inner = @' - Write-Output "๐Ÿ” Running enhanced Quarto diagnostics..." - Write-Output "๐Ÿ“‹ Capturing comprehensive diagnostic information..." - Write-Output "" - - Write-Output '--- SYSTEM DIAGNOSTIC INFO START ---' - Write-Output '๐Ÿ”ง Environment Variables:' - Write-Output ("PATH: {0}" -f $env:PATH) - Write-Output ("QUARTO_LOG_LEVEL: {0}" -f $env:QUARTO_LOG_LEVEL) - Write-Output "" - - Write-Output '๐Ÿ”ง Visual C++ Redistributable DLLs:' - Get-ChildItem 'C:\Windows\System32' -Filter 'msvcp*.dll' -ErrorAction SilentlyContinue | - Select-Object Name, Length | - Format-Table -AutoSize | Out-String | Write-Output - Get-ChildItem 'C:\Windows\System32' -Filter 'vcruntime*.dll' -ErrorAction SilentlyContinue | - Select-Object Name, Length | - Format-Table -AutoSize | Out-String | Write-Output - Write-Output "" - - Write-Output '๐Ÿ”ง Quarto Installation Info:' - try { - $quartoCmd = Get-Command quarto.exe -ErrorAction Stop - Write-Output ("Quarto.exe location: {0}" -f $quartoCmd.Source) - Write-Output ("Quarto.exe file size: {0} bytes" -f (Get-Item $quartoCmd.Source).Length) - $quartoVersion = & $quartoCmd.Source --version 2>&1 - Write-Output ("Quarto version: {0}" -f $quartoVersion.Trim()) - } catch { - Write-Output 'quarto.exe not found in PATH' - } - Write-Output '--- SYSTEM DIAGNOSTIC INFO END ---' - Write-Output "" - - # Run quarto check with enhanced error handling and raw output capture - $quartoExitCode = 0 - try { - Write-Output '--- QUARTO CHECK OUTPUT START ---' - Write-Output '๐Ÿ” Setting maximum verbosity for raw output capture...' - $env:QUARTO_LOG_LEVEL = 'DEBUG' - $env:QUARTO_PRINT_STACK = 'true' - - # Capture raw output with timestamps using a more robust method - Write-Output "โฐ Quarto check started at: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" - - $stdOutPath = [System.IO.Path]::GetTempFileName() - $stdErrPath = [System.IO.Path]::GetTempFileName() - - $quartoExitCode = 0 - - try { - $process = Start-Process -FilePath "quarto.exe" -ArgumentList "check", "--log-level=debug" -Wait -PassThru -RedirectStandardOutput $stdOutPath -RedirectStandardError $stdErrPath - $quartoExitCode = $process.ExitCode - } catch { - $quartoExitCode = 1 # Indicate failure - Write-Output "โ€ผ๏ธ Error executing Start-Process: $($_.Exception.Message)" - } - - $rawOutput = Get-Content $stdOutPath -Raw -ErrorAction SilentlyContinue - $errorOutput = Get-Content $stdErrPath -Raw -ErrorAction SilentlyContinue - - # Combine stdout and stderr for the log - if (![string]::IsNullOrEmpty($errorOutput)) { - $rawOutput += "`n--- STDERR ---`n" + $errorOutput - } - - Remove-Item $stdOutPath, $stdErrPath -Force -ErrorAction SilentlyContinue - - Write-Output "โฐ Quarto check completed at: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" - - # Display raw output line by line with prefixes for clarity - Write-Output '๐Ÿ“‹ RAW QUARTO CHECK OUTPUT:' - Write-Output 'โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”' - if (-not [string]::IsNullOrEmpty($rawOutput)) { - foreach ($line in $rawOutput.Split("`n")) { - Write-Output "โ”‚ $line" - } - } - Write-Output 'โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜' - - Write-Output '--- QUARTO CHECK OUTPUT END ---' - } catch { - Write-Output '--- QUARTO CHECK EXCEPTION ---' - Write-Output ("Exception Type: {0}" -f $($_.Exception.GetType().FullName)) - Write-Output ("Exception Message: {0}" -f $($_.Exception.Message)) - if ($_.Exception.InnerException) { - Write-Output ("Inner Exception: {0}" -f $($_.Exception.InnerException.Message)) - } - Write-Output '--- QUARTO CHECK EXCEPTION END ---' - $quartoExitCode = 1 - } - - Write-Output "" - if ($quartoExitCode -eq 0) { - Write-Output 'โœ… Quarto check: PASSED - All components verified!' - $QuartoStatus = 'PASSED' - } else { - Write-Output 'โŒ Quarto check: FAILED - Issues detected!' - Write-Output '๐Ÿ” This indicates potential container configuration issues.' - Write-Output '๐Ÿ“‹ Check the diagnostic information above for details.' - Write-Output ("๐Ÿ“Š Exit Code: {0}" -f $quartoExitCode) - switch ($quartoExitCode) { - -1073741515 { Write-Output '๐Ÿ” Exit code -1073741515 (0xC0000135): DLL not found - Missing Visual C++ Runtime or dependencies' } - 1 { Write-Output '๐Ÿ” Exit code 1: General error - Check Quarto dependencies' } - default { Write-Output ("๐Ÿ” Exit code {0}: Unknown error - Check logs above" -f $quartoExitCode) } - } - $QuartoStatus = 'FAILED' - } - - Write-Output "" - Write-Output ("๐Ÿ“Š QUARTO CHECK SUMMARY: {0}" -f $QuartoStatus) - '@ - - $scriptPath = Join-Path $PWD 'windows-quarto-check.ps1' - Set-Content -Path $scriptPath -Value $inner -Encoding UTF8 - - # 2) Run it inside the Windows container and tee the output - docker run --rm ` - -v "${PWD}:C:\work" ` - ${{ env.CONTAINER_IMAGE }} ` - pwsh -NoProfile -ExecutionPolicy Bypass -File "C:\work\windows-quarto-check.ps1" ` - | Tee-Object -FilePath "windows-quarto-check.log" - - Write-Output "๐Ÿ“‹ Windows quarto check results saved to windows-quarto-check.log" - - # Also save just the raw quarto check output to a separate file for easy viewing - if (Test-Path "windows-quarto-check.log") { - $logContent = Get-Content "windows-quarto-check.log" -Raw - if ($logContent -match '๐Ÿ“‹ RAW QUARTO CHECK OUTPUT:(.*?)--- QUARTO CHECK OUTPUT END ---') { - $rawOutput = $matches[1].Trim() - $rawOutput | Out-File "windows-quarto-raw-output.txt" -Encoding UTF8 - Write-Output "๐Ÿ“„ Raw Quarto output extracted to windows-quarto-raw-output.txt" } } @@ -753,27 +371,16 @@ jobs: echo "โœ… Tool validation enforced - will FAIL if any essential tool missing" echo "LINUX CONTAINER TESTS COMPLETE" - - name: ๐Ÿ“Š Windows Container Analysis - if: matrix.platform == 'windows' && inputs.test_windows != false - run: | - Write-Output "WINDOWS CONTAINER TEST SUMMARY" - Write-Output "โœ… Container pulled successfully" - Write-Output "โœ… Essential tools validated with pass/fail status (Quarto, Python, R, LaTeX, Ghostscript, Inkscape)" - Write-Output "โœ… Quarto check completed" - Write-Output "โœ… Container size displayed" - Write-Output "โœ… Tool validation enforced - will FAIL if any essential tool missing" - Write-Output "WINDOWS CONTAINER TESTS COMPLETE" - name: ๐Ÿ“ค Upload Test Artifacts - if: always() && ((matrix.platform == 'linux' && inputs.test_linux != false) || (matrix.platform == 'windows' && inputs.test_windows != false)) + if: always() && matrix.platform == 'linux' && inputs.test_linux != false uses: actions/upload-artifact@v6 with: name: ${{ matrix.platform }}-container-test-results path: | - ${{ matrix.platform == 'linux' && 'linux-tool-versions.log' || 'windows-tool-versions.log' }} - ${{ matrix.platform == 'linux' && 'linux-quarto-check.log' || 'windows-quarto-check.log' }} - ${{ matrix.platform == 'linux' && 'linux-quarto-raw-output.txt' || 'windows-quarto-raw-output.txt' }} - ${{ matrix.platform == 'windows' && 'windows-quarto-check.ps1' || '' }} + linux-tool-versions.log + linux-quarto-check.log + linux-quarto-raw-output.txt if-no-files-found: warn # Final summary job @@ -791,14 +398,11 @@ jobs: # Get container sizes LINUX_IMAGE="${{ inputs.container_registry || 'ghcr.io' }}/${{ github.repository }}/quarto-linux:${{ inputs.container_tag || 'latest' }}" - WINDOWS_IMAGE="${{ inputs.container_registry || 'ghcr.io' }}/${{ github.repository }}/quarto-windows:${{ inputs.container_tag || 'latest' }}" LINUX_SIZE=$(docker images "$LINUX_IMAGE" --format "{{.Size}}" 2>/dev/null || echo "Not available") - WINDOWS_SIZE=$(docker images "$WINDOWS_IMAGE" --format "{{.Size}}" 2>/dev/null || echo "Not available") echo "๐Ÿ“ฆ CONTAINER SIZES:" echo "------------------" echo "๐Ÿง Linux: $LINUX_SIZE" - echo "๐ŸชŸ Windows: $WINDOWS_SIZE" echo "" echo "๐Ÿ” TEST RESULTS:" diff --git a/book/docker/windows/.dockerignore b/book/docker/windows/.dockerignore deleted file mode 100644 index f1db2ea80..000000000 --- a/book/docker/windows/.dockerignore +++ /dev/null @@ -1,71 +0,0 @@ -# Exclude unnecessary files from Docker build context -# This reduces build time and image size - -# Build artifacts -build/ -_book/ -_site/ -*.pdf -*.html - -# Git and version control -.git/ -.gitignore - -# Documentation -docs/ -*.md -!docker/quarto-build-windows/README.md - -# IDE and editor files -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS files -.DS_Store -Thumbs.db - -# Logs and temporary files -*.log -*.tmp -*.temp - -# Node modules (if any) -node_modules/ - -# Python cache -__pycache__/ -*.pyc -*.pyo - -# R cache -.Rhistory -.RData - -# Large media files (not needed for build) -assets/media/ -*.mp4 -*.avi -*.mov - -# Test files -test-* -*.test.* - -# Backup files -*.bak -*.backup - -# Large data files -data/ -*.csv -*.json -*.xml - -# Keep only essential files for build -# - tools/dependencies/ (needed for package installation) -# - book/ (needed for build testing) -# - .github/workflows/ (needed for workflow files) diff --git a/book/docker/windows/Dockerfile b/book/docker/windows/Dockerfile deleted file mode 100644 index da5250192..000000000 --- a/book/docker/windows/Dockerfile +++ /dev/null @@ -1,456 +0,0 @@ -# escape=` -# MLSysBook Windows Quarto Build Container (Windows Server 2022) -# - PowerShell 7 via ZIP (no MSI) -# - Quarto via Scoop (extras bucket) -# - Python 3.13.1 + requirements -# - Ghostscript + Inkscape (Scoop) -# - TeX Live (latest by default; overridable via TEXLIVE_VERSION build arg) + packages from tl_packages -# - R 4.5.2 + packages via install_packages.R -# - Verifications: versions, kpsewhich font files, TikZ smoke test - -FROM mcr.microsoft.com/windows/server:ltsc2022 -ARG TEXLIVE_VERSION=latest -ENV TEXLIVE_VERSION=${TEXLIVE_VERSION} - -# Use Windows PowerShell initially -SHELL ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command"] - -# === PATH CONFIGURATION === -# Paths are hardcoded since they're stable and ARG scope causes issues in multi-layer builds - -# ------------------------------------------------------------ -# PHASE 0: Base dirs and env (same as quarto-build workflow) -# ------------------------------------------------------------ -ENV R_LIBS_USER="C:/r-lib" -ENV QUARTO_LOG_LEVEL="INFO" -ENV PYTHONIOENCODING="utf-8" -ENV LANG="en_US.UTF-8" -ENV LC_ALL="en_US.UTF-8" -# Explicitly set TMP/TEMP so the TeX Live Perl installer always has a valid temp dir -ENV TMP="C:/temp" -ENV TEMP="C:/temp" - -RUN Write-Host '=== STARTING BASE SETUP ===' ; ` - Write-Host 'Creating base directories...' ; ` - New-Item -ItemType Directory -Force -Path 'C:\temp' | Out-Null ; ` - Write-Host '๐Ÿ“ Created C:\temp' ; ` - New-Item -ItemType Directory -Force -Path 'C:\r-lib' | Out-Null ; ` - Write-Host '๐Ÿ“ Created C:\r-lib' ; ` - Write-Host 'Environment variables set:' ; ` - Write-Host " R_LIBS_USER: $env:R_LIBS_USER" ; ` - Write-Host " QUARTO_LOG_LEVEL: $env:QUARTO_LOG_LEVEL" ; ` - Write-Host " PYTHONIOENCODING: $env:PYTHONIOENCODING" ; ` - Write-Host " LANG: $env:LANG" ; ` - Write-Host " LC_ALL: $env:LC_ALL" ; ` - Write-Host 'โœ… Base setup complete' - -# ------------------------------------------------------------ -# PHASE 1: PowerShell 7 (ZIP install, container-safe) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING POWERSHELL 7 INSTALLATION ===' ; ` - Write-Host 'Using ZIP install for container compatibility' ; ` - Write-Host 'Download URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.1/PowerShell-7.4.1-win-x64.zip' ; ` - $Url = 'https://github.com/PowerShell/PowerShell/releases/download/v7.4.1/PowerShell-7.4.1-win-x64.zip' ; ` - $Zip = 'C:\PowerShell-7.4.1.zip' ; ` - Write-Host "Downloading PowerShell 7 to: $Zip" ; ` - Invoke-WebRequest -Uri $Url -OutFile $Zip -UseBasicParsing ; ` - Write-Host '๐Ÿ“ฅ Download completed' ; ` - Write-Host 'Creating PowerShell directory...' ; ` - New-Item -ItemType Directory -Force -Path 'C:\Program Files\PowerShell\7' | Out-Null ; ` - Write-Host '๐Ÿ“ Directory created' ; ` - Write-Host 'Extracting ZIP file...' ; ` - Expand-Archive -Path $Zip -DestinationPath 'C:\Program Files\PowerShell\7' -Force ; ` - Write-Host '๐Ÿ“ฆ Extraction completed' ; ` - Write-Host 'Cleaning up ZIP file...' ; ` - Remove-Item $Zip -Force ; ` - Write-Host '๐Ÿงน Cleanup completed' ; ` - Write-Host 'Adding PowerShell to PATH...' ; ` - $mach = [Environment]::GetEnvironmentVariable('PATH','Machine') ; ` - Write-Host "Current PATH: $mach" ; ` - if ($mach -notmatch [regex]::Escape('C:\Program Files\PowerShell\7')) { ` - [Environment]::SetEnvironmentVariable('PATH', ('C:\Program Files\PowerShell\7;' + $mach), 'Machine') ; ` - Write-Host '๐Ÿ”— PowerShell added to PATH' ; ` - } else { ` - Write-Host 'โš ๏ธ PowerShell already in PATH' ; ` - } ; ` - Write-Host 'Verifying PowerShell installation...' ; ` - & 'C:\Program Files\PowerShell\7\pwsh.exe' -NoLogo -Command '$PSVersionTable.PSVersion ; Write-Host ''PowerShell 7 installation verified โœ…''' - -# Switch to PowerShell 7 for subsequent layers -SHELL ["C:\\Program Files\\PowerShell\\7\\pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command"] - -# ------------------------------------------------------------ -# PHASE 2: Chocolatey (package manager for Windows) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING CHOCOLATEY INSTALLATION ===' ; ` - Write-Host 'Installing Chocolatey package manager...' ; ` - Write-Host 'Setting TLS 1.2 for download...' ; ` - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; ` - Write-Host '๐Ÿ”’ TLS 1.2 enabled' ; ` - Write-Host 'Downloading and executing Chocolatey install script...' ; ` - iex ((New-Object Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) ; ` - Write-Host '๐Ÿ“ฆ Chocolatey install script executed' ; ` - Write-Host 'Verifying Chocolatey installation...' ; ` - choco --version ; ` - Write-Host '๐Ÿ”ง Configuring Chocolatey for CI stability...' ; ` - choco config set --name commandExecutionTimeoutSeconds --value 14400 ; ` - choco feature disable --name showDownloadProgress ; ` - Write-Host 'โœ… Chocolatey timeout/progress settings applied' ; ` - Write-Host 'โœ… Chocolatey installation complete' - -# ------------------------------------------------------------ -# PHASE 3: Copy dependency files (same as quarto-build workflow) -# ------------------------------------------------------------ -COPY book/tools/dependencies/requirements.txt C:/temp/requirements.txt -COPY book/tools/dependencies/install_packages.R C:/temp/install_packages.R -COPY book/tools/dependencies/tl_packages C:/temp/tl_packages -COPY book/docker/windows/verify_r_packages.R C:/temp/verify_r_packages.R -COPY book/docker/windows/install_texlive.ps1 C:/temp/install_texlive.ps1 -RUN Write-Host 'โœ… Dependency file copy complete' - -# ------------------------------------------------------------ -# PHASE 4: Install Scoop (Package manager setup) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING SCOOP INSTALLATION ===' ; ` - Write-Host 'Setting UTF-8 encoding...' ; ` - [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 ; ` - $OutputEncoding = [System.Text.Encoding]::UTF8 ; ` - Write-Host '๐Ÿ”ค UTF-8 encoding set' ; ` - Write-Host 'Setting execution policy...' ; ` - Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser -Force ; ` - Write-Host '๐Ÿ” Execution policy set' ; ` - Write-Host 'Installing Scoop package manager...' ; ` - Invoke-WebRequest -useb get.scoop.sh -outfile 'install.ps1' ; ` - Write-Host '๐Ÿ“ฅ Scoop install script downloaded' ; ` - & .\install.ps1 -RunAsAdmin ; ` - Write-Host '๐Ÿ“ฆ Scoop installed' ; ` - Write-Host 'Adding Scoop shims to PATH...' ; ` - $scoopShims = Join-Path (Resolve-Path ~).Path 'scoop\shims' ; ` - Write-Host "Scoop shims path: $scoopShims" ; ` - $mach = [Environment]::GetEnvironmentVariable('PATH','Machine') ; ` - [Environment]::SetEnvironmentVariable('PATH', ($scoopShims + ';' + $mach), 'Machine') ; ` - Write-Host '๐Ÿ”— Added Scoop shims to PATH' ; ` - Write-Host 'Installing Git (required for buckets)...' ; ` - scoop install git ; ` - Write-Host '๐Ÿ“ฆ Git installed' ; ` - Write-Host 'Adding r-bucket...' ; ` - scoop bucket add r-bucket https://github.com/cderv/r-bucket.git ; ` - Write-Host '๐Ÿ“ฆ r-bucket added' ; ` - Write-Host 'Adding extras bucket...' ; ` - scoop bucket add extras ; ` - Write-Host '๐Ÿ“ฆ extras bucket added' ; ` - Write-Host 'โœ… Scoop installation completed!' - -# ------------------------------------------------------------ -# PHASE 5: Install Quarto via Scoop (consistent PATH with other tools) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING QUARTO INSTALLATION ===' ; ` - Write-Host 'Installing Quarto via Scoop (extras bucket)...' ; ` - scoop install extras/quarto ; ` - if (-not (Get-Command quarto -ErrorAction SilentlyContinue)) { ` - Write-Host 'โŒ Quarto install failed: executable not found in PATH' ; ` - exit 1 ; ` - } ; ` - $qv = quarto --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ Quarto install failed: version check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ Quarto installed: $qv" ; ` - Write-Host 'โœ… Quarto installation completed!' - -# ------------------------------------------------------------ -# PHASE 6: Install Ghostscript (required for PDF generation) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING GHOSTSCRIPT INSTALLATION ===' ; ` - Write-Host 'Installing Ghostscript via Scoop...' ; ` - scoop install main/ghostscript ; ` - $gsCmd = Get-Command gs -ErrorAction SilentlyContinue ; ` - if (-not $gsCmd) { ` - Write-Host 'โŒ Ghostscript install failed: gs shim not found in PATH' ; ` - exit 1 ; ` - } ; ` - Write-Host "Resolved Ghostscript: $($gsCmd.Source)" ; ` - Write-Host '๐Ÿ“ฆ Ghostscript installed (runtime check deferred to final verification)' ; ` - Write-Host 'โœ… Ghostscript installation complete' - -# ------------------------------------------------------------ -# PHASE 7: Install Inkscape and rsvg-convert (required for SVG processing) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING INKSCAPE INSTALLATION ===' ; ` - Write-Host 'Installing Inkscape via Scoop...' ; ` - scoop install inkscape ; ` - if (-not (Get-Command inkscape -ErrorAction SilentlyContinue)) { ` - Write-Host 'โŒ Inkscape install failed: executable not found in PATH' ; ` - exit 1 ; ` - } ; ` - $iv = inkscape --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ Inkscape install failed: version check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ Inkscape installed: $iv" ; ` - Write-Host 'โœ… Inkscape installation complete' - -RUN Write-Host '=== STARTING RSVG-CONVERT INSTALLATION ===' ; ` - Write-Host 'Installing rsvg-convert via Scoop (required by Quarto for SVG-to-PDF)...' ; ` - scoop install rsvg-convert ; ` - if (-not (Get-Command rsvg-convert -ErrorAction SilentlyContinue)) { ` - Write-Host 'โš ๏ธ Scoop install failed or package unavailable from current buckets; trying Chocolatey...' ; ` - choco install rsvg-convert -y ; ` - } ; ` - if (-not (Get-Command rsvg-convert -ErrorAction SilentlyContinue)) { ` - Write-Host 'โŒ rsvg-convert install failed via both Scoop and Chocolatey' ; ` - exit 1 ; ` - } ; ` - $rv = rsvg-convert --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ rsvg-convert install failed: version check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ rsvg-convert installed: $rv" ; ` - Write-Host 'โœ… rsvg-convert installation complete' - -# ------------------------------------------------------------ -# PHASE 8: Install Python (Medium complexity) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING PYTHON INSTALLATION ===' ; ` - Write-Host 'Installing Python via Scoop (same as quarto-build workflow)...' ; ` - Write-Host 'Installing Python from main bucket...' ; ` - scoop install main/python ; ` - if (-not (Get-Command python -ErrorAction SilentlyContinue)) { ` - Write-Host 'โŒ Python install failed: executable not found in PATH' ; ` - exit 1 ; ` - } ; ` - $pv = python --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ Python install failed: version check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - $pyExe = (python -c 'import sys; print(sys.executable)').Trim() ; ` - $pyDir = Split-Path $pyExe -Parent ; ` - $py3Exe = Join-Path $pyDir 'python3.exe' ; ` - if (-not (Test-Path $py3Exe)) { ` - Copy-Item $pyExe $py3Exe -Force ; ` - } ; ` - $pv3 = python3 --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ Python install failed: python3 alias check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ Python installed: $pv" ; ` - Write-Host "๐Ÿ“ฆ Python3 alias verified: $pv3" ; ` - Write-Host 'โœ… Python installation complete' - -# ------------------------------------------------------------ -# PHASE 9: Install Python packages (Medium complexity) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING PYTHON PACKAGE INSTALLATION ===' ; ` - Write-Host 'Installing Python packages from requirements.txt (same as quarto-build workflow)...' ; ` - Write-Host 'Using bundled pip (skip upgrade to avoid WinError 3 shim lock)...' ; ` - Write-Host 'Installing packages from requirements.txt...' ; ` - Write-Host 'Requirements file contents:' ; ` - Get-Content C:/temp/requirements.txt | Write-Host ; ` - python -m pip install -r C:/temp/requirements.txt ; ` - Write-Host 'โœ… Python package installation complete' - -# ------------------------------------------------------------ -# PHASE 10: Install Visual C++ Redistributable (Required for Quarto DLLs) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING VISUAL C++ REDISTRIBUTABLE INSTALLATION ===' ; ` - Write-Host 'Installing Microsoft Visual C++ Redistributable...' ; ` - Write-Host 'This is required for Quarto DLL dependencies on Windows' ; ` - choco install vcredist-all -y ; ` - Write-Host '๐Ÿ“ฆ Visual C++ Redistributable installed' ; ` - Write-Host 'โœ… Visual C++ Redistributable installation complete' - -# ------------------------------------------------------------ -# PHASE 11: Install R (Medium complexity) -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING R INSTALLATION ===' ; ` - Write-Host 'Installing R 4.5.2 via Scoop (main bucket)...' ; ` - scoop install main/r@4.5.2 ; ` - if (-not (Get-Command Rscript -ErrorAction SilentlyContinue)) { ` - Write-Host 'โŒ R install failed: Rscript not found in PATH' ; ` - exit 1 ; ` - } ; ` - $rv = Rscript --version ; ` - if ($LASTEXITCODE -ne 0) { ` - Write-Host "โŒ R install failed: version check exited $LASTEXITCODE" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ R installed: $rv" ; ` - Write-Host 'โœ… R installation complete' - -# ------------------------------------------------------------ -# PHASE 12: Install R packages (Medium complexity) -# ------------------------------------------------------------ -RUN Write-Host '=== INSTALLING R PACKAGES ===' ; ` - Write-Host 'Installing R packages from install_packages.R (same as quarto-build workflow)...' ; ` - Write-Host 'Setting up R environment...' ; ` - Write-Host "R_LIBS_USER: $env:R_LIBS_USER" ; ` - Write-Host 'Installing R packages...' ; ` - Rscript -e 'options(repos=c(CRAN=\"https://cran.rstudio.com\"))' ; ` - Rscript -e 'dir.create(Sys.getenv(\"R_LIBS_USER\"), recursive=TRUE, showWarnings=FALSE)' ; ` - Rscript -e '.libPaths(Sys.getenv(\"R_LIBS_USER\"))' ; ` - Rscript -e 'install.packages(\"remotes\")' ; ` - if (Test-Path 'C:/temp/install_packages.R') { ` - Write-Host 'Found install_packages.R, sourcing it...' ; ` - Rscript 'C:/temp/install_packages.R' ; ` - } else { ` - Write-Host 'No install_packages.R found, installing basic packages...' ; ` - Rscript -e 'install.packages(c(\"rmarkdown\",\"knitr\",\"ggplot2\"))' ; ` - } ; ` - Rscript -e 'for (p in c(\"rmarkdown\",\"knitr\")) if (!require(p, character.only=TRUE, quietly=TRUE)) stop(\"missing: \", p)' ; ` - Write-Host '๐Ÿ“ฆ R packages installed' ; ` - Write-Host 'Verifying R packages...' ; ` - Rscript C:/temp/verify_r_packages.R ; ` - Write-Host 'โœ… R package installation complete' - -# ------------------------------------------------------------ -# PHASE 13: Install TeX Live LAST (largest/slowest install) -# Direct install-tl approach via standalone script: -# - Bypasses Chocolatey's ErrorActionPreference=Stop wrapper -# - Pins mirror in profile, tries 3 mirrors in order -# ------------------------------------------------------------ -RUN & 'C:\temp\install_texlive.ps1' - -RUN Write-Host '=== VERIFY TEX LIVE INSTALLATION ===' ; ` - $cmd = Get-Command lualatex -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { ` - Write-Host 'โŒ TeX Live verification failed: lualatex not found in PATH' ; ` - exit 1 ; ` - } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $lvAll = & lualatex --version 2>&1 ; ` - $lv = $lvAll | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { ` - Write-Host "โŒ TeX Live verification failed: lualatex --version exited $exitCode" ; ` - exit 1 ; ` - } ; ` - Write-Host "๐Ÿ“ฆ LaTeX verified: $lv" ; ` - Write-Host 'โœ… TeX Live verification complete' - -# ------------------------------------------------------------ -# PHASE 14: Cleanup and Environment Setup -# ------------------------------------------------------------ -RUN Write-Host '=== STARTING CLEANUP AND ENVIRONMENT SETUP ===' ; ` - Write-Host 'Cleaning temporary files and setting up environment...' ; ` - Write-Host 'Removing temporary files...' ; ` - Remove-Item C:/temp/requirements.txt -ErrorAction SilentlyContinue ; ` - Write-Host '๐Ÿ—‘๏ธ requirements.txt removed' ; ` - Remove-Item C:/temp/install_packages.R -ErrorAction SilentlyContinue ; ` - Write-Host '๐Ÿ—‘๏ธ install_packages.R removed' ; ` - Remove-Item C:/temp/verify_r_packages.R -ErrorAction SilentlyContinue ; ` - Write-Host '๐Ÿ—‘๏ธ verify_r_packages.R removed' ; ` - Remove-Item C:/temp/tl_packages -ErrorAction SilentlyContinue ; ` - Write-Host '๐Ÿ—‘๏ธ tl_packages removed' ; ` - Remove-Item C:/temp/requirements/ -Recurse -Force -ErrorAction SilentlyContinue ; ` - Write-Host '๐Ÿ—‘๏ธ requirements/ directory removed' ; ` - Write-Host 'Setting up environment variables for Quarto...' ; ` - $env:QUARTO_LOG_LEVEL = 'DEBUG' ; ` - [Environment]::SetEnvironmentVariable('QUARTO_LOG_LEVEL', 'DEBUG', 'Machine') ; ` - Write-Host '๐Ÿ”ง QUARTO_LOG_LEVEL set to DEBUG' ; ` - Write-Host 'โœ… Cleanup and environment setup complete' - -# ------------------------------------------------------------ -# PHASE 15: Commit PATH into Docker image metadata -# [Environment]::SetEnvironmentVariable writes to the registry but Docker does -# not re-read HKLM between RUN steps โ€” each new layer inherits the *previous -# layer's* Docker ENV, not registry changes made mid-layer. Using the Docker -# ENV directive here guarantees these paths are committed into the image and -# visible to every subsequent RUN step and to containers at runtime. -# ------------------------------------------------------------ -ENV PATH="C:\\Users\\ContainerAdministrator\\scoop\\shims;C:\\Users\\ContainerAdministrator\\scoop\\apps\\python\\current\\Scripts;C:\\Users\\ContainerAdministrator\\scoop\\apps\\python\\current;C:\\Users\\ContainerAdministrator\\scoop\\apps\\ghostscript\\current\\lib;C:\\Users\\ContainerAdministrator\\scoop\\apps\\r\\current\\bin;C:\\Users\\ContainerAdministrator\\scoop\\apps\\git\\current\\cmd;C:\\texlive\\bin\\windows;C:\\Program Files\\PowerShell\\7;C:\\ProgramData\\chocolatey\\bin;C:\\Windows\\system32;C:\\Windows;C:\\Windows\\System32\\Wbem;C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\Windows\\System32\\OpenSSH\\" - -# ------------------------------------------------------------ -# FINAL CHECKS: Comprehensive verification with diagnostics -# ------------------------------------------------------------ -WORKDIR C:/workspace -RUN Write-Host '=== FINAL VERIFICATION ===' ; ` - Write-Host 'PATH environment variable:' ; ` - Write-Host $env:PATH ; ` - Write-Host '' ; ` - Write-Host 'Tool checks run in separate Docker steps below for isolated reporting' - -RUN Write-Host '=== VERIFY: Quarto ===' ; ` - $cmd = Get-Command quarto -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ Quarto FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & quarto --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ Quarto FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… Quarto verified' - -RUN Write-Host '=== VERIFY: Python ===' ; ` - $cmd = Get-Command python -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ Python FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & python --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ Python FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… Python verified' - -RUN Write-Host '=== VERIFY: Python3 ===' ; ` - $cmd = Get-Command python3 -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ Python3 FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & python3 --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ Python3 FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… Python3 verified' - -RUN Write-Host '=== VERIFY: R ===' ; ` - $cmd = Get-Command Rscript -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ R FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & Rscript --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ R FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… R verified' - -RUN Write-Host '=== VERIFY: LaTeX ===' ; ` - $cmd = Get-Command lualatex -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ LaTeX FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & lualatex --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ LaTeX FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… LaTeX verified' - -RUN Write-Host '=== VERIFY: Ghostscript ===' ; ` - $cmd = Get-Command gs -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { $cmd = Get-Command gswin64c -ErrorAction SilentlyContinue } ; ` - if (-not $cmd) { Write-Host "โŒ Ghostscript FAILED: gs/gswin64c not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & $cmd.Source --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ Ghostscript FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… Ghostscript verified' - -RUN Write-Host '=== VERIFY: Inkscape ===' ; ` - $cmd = Get-Command inkscape -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ Inkscape FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & inkscape --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ Inkscape FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… Inkscape verified' - -RUN Write-Host '=== VERIFY: rsvg-convert ===' ; ` - $cmd = Get-Command rsvg-convert -ErrorAction SilentlyContinue ; ` - if (-not $cmd) { Write-Host "โŒ rsvg-convert FAILED: command not found"; exit 1 } ; ` - Write-Host "Resolved: $($cmd.Source)" ; ` - $out = & rsvg-convert --version 2>&1 | Select-Object -First 1 ; ` - $exitCode = if ($null -eq $LASTEXITCODE) { if ($?) { 0 } else { 1 } } else { [int]$LASTEXITCODE } ; ` - if ($exitCode -ne 0) { Write-Host "โŒ rsvg-convert FAILED: exited $exitCode"; exit 1 } ; ` - Write-Host " $out" ; ` - Write-Host 'โœ… rsvg-convert verified' diff --git a/book/docker/windows/README.md b/book/docker/windows/README.md deleted file mode 100644 index 5126c23ab..000000000 --- a/book/docker/windows/README.md +++ /dev/null @@ -1,179 +0,0 @@ -# Windows Quarto Build Container - -This directory contains the Windows Server 2022 container configuration for building the MLSysBook with Quarto. - -## ๐Ÿณ Container Features - -- **Base Image**: Windows Server 2022 LTSC -- **PowerShell**: 7.4.1 (ZIP install, container-safe) -- **Quarto**: 1.9.27 (Scoop) -- **Python**: 3.13.1 + production dependencies -- **TeX Live**: 2025 snapshot with required packages -- **R**: 4.5.2 + R Markdown packages -- **Graphics**: Ghostscript + Inkscape (via Chocolatey) - -## ๐Ÿ”ง Key Fixes Applied - -### 1. PowerShell 7 Path Issues -- **Problem**: Using `pwsh` shorthand can fail in containers -- **Fix**: Use full path `C:\Program Files\PowerShell\7\pwsh.exe` - -### 2. TeX Live Installation -- **Problem**: `Start-Process` without `-NoNewWindow` can hang -- **Fix**: Added `-NoNewWindow` flag for container compatibility -- **Problem**: Comments in `tl_packages` file -- **Fix**: Filter out comment lines when installing packages - -### 3. TikZ Test Document -- **Problem**: Complex here-string with backticks -- **Fix**: Simplified to standard multi-line string - -### 4. Package Installation -- **Problem**: Silent failures in package installation -- **Fix**: Added verbose output and better error handling - -## ๐Ÿš€ Building the Container - -### Prerequisites -- Windows Docker Desktop or Windows Server with Docker -- At least 8GB RAM available for Docker -- 20GB+ free disk space - -## Local Build -To build the Windows container locally, run the following command from the repository root: -```powershell -docker build -f docker/windows/Dockerfile -t mlsysbook-windows . -``` - -### Testing -To test the Dockerfile before building, you can use the provided PowerShell script: -```powershell -./docker/windows/test_dockerfile.ps1 -``` - -## Workflow -The container is built and pushed to the GitHub Container Registry via the [`.github/workflows/build-windows-container.yml`](../../.github/workflows/build-windows-container.yml) workflow. -This workflow is triggered manually or on a weekly schedule. - -## Notes -- Building the Windows container can take a significant amount of time (often over 2 hours). -- The image is large due to the comprehensive set of pre-installed dependencies. - -## ๐Ÿ“‹ Build Phases - -1. **Base Setup**: Directories, environment variables -2. **PowerShell 7**: ZIP installation (container-safe) -3. **Chocolatey**: Package manager installation -4. **Dependencies**: Copy required files -5. **Quarto**: ZIP installation with PATH setup -6. **Python**: 3.13.1 + production requirements -7. **Graphics**: Ghostscript + Inkscape -8. **TeX Live**: 2025 snapshot + packages -9. **R**: 4.5.2 + R Markdown packages -10. **Cleanup**: Remove temporary files - -## ๐Ÿ” Verification Steps - -The container includes comprehensive verification: - -- **PowerShell 7**: Version check -- **Quarto**: Version and command availability -- **Python**: Version and pip functionality -- **TeX Live**: Package verification with `kpsewhich` -- **Fonts**: Helvetica font files verification -- **TikZ**: Smoke test with PDF generation -- **R**: Package installation verification - -## โš ๏ธ Common Issues & Solutions - -### 1. Build Timeouts -- **Cause**: Large downloads (TeX Live, Python packages) -- **Solution**: Increased timeout values in Dockerfile - -### 2. PATH Issues -- **Cause**: Windows PATH not properly updated -- **Solution**: Explicit PATH manipulation with regex escaping - -### 3. Package Installation Failures -- **Cause**: Network issues or missing dependencies -- **Solution**: Added verbose output and error checking - -### 4. Memory Issues -- **Cause**: TeX Live installation requires significant memory -- **Solution**: Use `scheme-infraonly` for minimal installation - -## ๐Ÿงช Testing - -### Run Container -```powershell -docker run -it mlsysbook-windows pwsh -``` - -### Test Quarto -```powershell -quarto --version -quarto check -``` - -### Test Python -```powershell -python --version -python -c "import nltk; print('NLTK available')" -``` - -### Test R -```powershell -R --version -Rscript -e "library(rmarkdown); print('R Markdown available')" -``` - -### Test TeX Live -```powershell -lualatex --version -kpsewhich pgf.sty -``` - -## ๐Ÿ“Š Performance Notes - -- **Build Time**: ~45-60 minutes on typical hardware -- **Image Size**: ~8-12GB (includes TeX Live, R, Python) -- **Memory Usage**: 4-6GB during build, 2-3GB runtime -- **Disk Space**: 15-20GB for build cache - -## ๐Ÿ”ง Troubleshooting - -### Build Fails on TeX Live -```powershell -# Check available memory -docker system df -docker system prune -f -``` - -### PowerShell Issues -```powershell -# Verify PowerShell 7 installation -docker run mlsysbook-windows pwsh -Command "Get-Host" -``` - -### Package Installation Issues -```powershell -# Check Chocolatey installation -docker run mlsysbook-windows choco --version -``` - -## ๐Ÿ“ Maintenance - -### Updating Dependencies -1. Update version numbers in Dockerfile -2. Test with validation script -3. Rebuild and verify all components - -### Adding New Packages -1. Add to appropriate phase in Dockerfile -2. Update verification steps -3. Test thoroughly - -### Security Updates -- Regularly update base image -- Monitor for CVE reports -- Update package versions as needed diff --git a/book/docker/windows/install_texlive.ps1 b/book/docker/windows/install_texlive.ps1 deleted file mode 100644 index ab7ea2033..000000000 --- a/book/docker/windows/install_texlive.ps1 +++ /dev/null @@ -1,152 +0,0 @@ -param( - [string]$TexLiveRoot = 'C:\texlive', - [string]$TexInstallDir = 'C:\texlive-install' -) - -$ErrorActionPreference = 'Continue' - -$mirrors = @( - 'https://ctan.math.illinois.edu/systems/texlive/tlnet', - 'https://mirrors.mit.edu/CTAN/systems/texlive/tlnet', - 'https://mirror.ctan.org/systems/texlive/tlnet' -) - -Write-Host '=== STARTING TEX LIVE INSTALLATION ===' -Write-Host '๐Ÿ“ฅ Downloading install-tl...' -$installTlZip = 'C:\temp\install-tl.zip' -Invoke-WebRequest -Uri 'https://mirror.ctan.org/systems/texlive/tlnet/install-tl.zip' ` - -OutFile $installTlZip -UseBasicParsing -Write-Host '๐Ÿ“ฆ Extracting install-tl...' -Expand-Archive -Path $installTlZip -DestinationPath $TexInstallDir -Force -Remove-Item $installTlZip -Force - -$installTlDir = Get-ChildItem $TexInstallDir -Directory | - Where-Object { $_.Name -match '^install-tl' } | - Select-Object -First 1 -Write-Host "๐Ÿ“ install-tl directory: $($installTlDir.FullName)" - -$installed = $false -foreach ($mirror in $mirrors) { - Write-Host "๐Ÿ”„ Trying mirror: $mirror" - $profile = 'C:\temp\texlive.profile' - @( - "selected_scheme scheme-basic", - "TEXDIR $TexLiveRoot", - "TEXMFLOCAL $TexLiveRoot/texmf-local", - "TEXMFSYSCONFIG $TexLiveRoot/texmf-config", - "TEXMFSYSVAR $TexLiveRoot/texmf-var", - "instopt_adjustrepo 0" - ) | Set-Content $profile - $env:TEXLIVE_INSTALL_NO_WELCOME = '1' - $batPath = Join-Path $installTlDir.FullName 'install-tl-windows.bat' - cmd /c """$batPath"" -no-gui -profile ""$profile"" -repository $mirror" - if ($LASTEXITCODE -eq 0) { - $installed = $true - Write-Host "โœ… TeX Live installed from $mirror" - break - } - Write-Host "โš ๏ธ Mirror $mirror failed (exit $LASTEXITCODE), trying next..." -} - -Remove-Item $TexInstallDir -Recurse -Force -ErrorAction SilentlyContinue - -if (-not $installed) { - Write-Host 'โŒ TeX Live installation failed on all mirrors' - exit 1 -} - -Write-Host '๐Ÿ” Finding TeX Live bin directory...' -Write-Host " TexLiveRoot contents:" -Get-ChildItem $TexLiveRoot -Directory | ForEach-Object { Write-Host " $_" } - -# Strategy 1: look for a year-numbered directory (e.g. 2025) -$texYearDir = Get-ChildItem $TexLiveRoot -Directory | - Where-Object { $_.Name -match '^\d{4}$' } | - Sort-Object Name -Descending | - Select-Object -First 1 - -if ($texYearDir) { - $texLiveBin = Join-Path $texYearDir.FullName 'bin\windows' -} else { - # Strategy 2: search recursively for tlmgr.bat - Write-Host ' โš ๏ธ No year directory found, searching recursively for tlmgr.bat...' - $tlmgr = Get-ChildItem $TexLiveRoot -Recurse -Filter 'tlmgr.bat' -ErrorAction SilentlyContinue | - Select-Object -First 1 - if ($tlmgr) { - $texLiveBin = $tlmgr.DirectoryName - } else { - Write-Host 'โŒ Cannot find TeX Live bin directory' - exit 1 - } -} - -if (-not (Test-Path $texLiveBin)) { - Write-Host "โŒ TeX Live bin directory does not exist: $texLiveBin" - exit 1 -} -Write-Host "๐Ÿ“ TeX Live bin: $texLiveBin" - -# Create a stable symlink so ENV PATH in Docker can use C:\texlive\bin\windows -$stableBin = Join-Path $TexLiveRoot 'bin\windows' -if ($texLiveBin -ne $stableBin -and -not (Test-Path $stableBin)) { - Write-Host "๐Ÿ”— Creating stable path: $stableBin -> $texLiveBin" - New-Item -ItemType Directory -Force -Path (Join-Path $TexLiveRoot 'bin') | Out-Null - cmd /c mklink /J "$stableBin" "$texLiveBin" - if (Test-Path $stableBin) { - Write-Host "โœ… Stable symlink created" - } else { - Write-Host "โš ๏ธ Symlink failed, using discovered path directly" - } -} - -[Environment]::SetEnvironmentVariable('PATH', ($stableBin + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine')), 'Machine') -Write-Host 'โœ… PATH updated' - -Write-Host '๐Ÿ”ง Pinning tlmgr repository to stable mirror...' -$tlmgrMirror = 'https://ctan.math.illinois.edu/systems/texlive/tlnet' -& "$texLiveBin\tlmgr.bat" option repository $tlmgrMirror -if ($LASTEXITCODE -eq 0) { - Write-Host "โœ… tlmgr repository: $tlmgrMirror" -} else { - Write-Host 'โš ๏ธ Could not pin tlmgr repository, continuing with defaults' -} - -Write-Host '๐Ÿ“‹ Reading collections from tl_packages...' -if (Test-Path 'C:\temp\tl_packages') { - $collections = Get-Content 'C:\temp\tl_packages' | - Where-Object { $_.Trim() -ne '' -and -not $_.Trim().StartsWith('#') } - Write-Host "๐Ÿ“ฆ Found $($collections.Count) collections to install" - $i = 1 - foreach ($collection in $collections) { - Write-Host "๐Ÿ“ฆ [$i/$($collections.Count)] Installing $collection..." - & "$texLiveBin\tlmgr.bat" install $collection - if ($LASTEXITCODE -eq 0) { - Write-Host "โœ… $collection installed successfully" - } else { - Write-Host "โš ๏ธ Failed to install $collection, continuing..." - } - $i++ - } - Write-Host 'โœ… Collection installation complete' -} else { - Write-Host 'โš ๏ธ No tl_packages file found, skipping collection installation' -} - -Write-Host '๐Ÿ”„ Updating tlmgr...' -& "$texLiveBin\tlmgr.bat" update --self --all -if ($LASTEXITCODE -ne 0) { - Write-Host 'โš ๏ธ tlmgr update returned non-zero, continuing...' -} else { - Write-Host 'โœ… tlmgr updated' -} - -Write-Host '๐Ÿ” Verifying lualatex installation...' -$lualatexPath = Join-Path $texLiveBin 'lualatex.exe' -if (-not (Test-Path $lualatexPath)) { - Write-Host "โŒ lualatex.exe not found at: $lualatexPath" - Write-Host " Contents of bin dir:" - Get-ChildItem $texLiveBin -ErrorAction SilentlyContinue | Select-Object -First 20 | ForEach-Object { Write-Host " $_" } - exit 1 -} -& $lualatexPath --version -Write-Host 'โœ… TeX Live installation verified' diff --git a/book/docker/windows/verify_r_packages.R b/book/docker/windows/verify_r_packages.R deleted file mode 100644 index 500b6fff2..000000000 --- a/book/docker/windows/verify_r_packages.R +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env Rscript - -# Verify R package installation -source('C:/temp/install_packages.R') - -missing_packages <- required_packages[!sapply(required_packages, requireNamespace, quietly = TRUE)] - -if(length(missing_packages) > 0) { - cat('โŒ Missing packages:', paste(missing_packages, collapse = ', '), '\n') - quit(status = 1) -} else { - cat('โœ… All required R packages installed successfully\n') -}