mirror of
https://github.com/harvard-edge/cs249r_book.git
synced 2026-08-03 12:14:39 -05:00
Vijay Janapa Reddi
2a61ece3f5
Both reusable workflows used `group: ${{ github.workflow }}-...`, but
when GitHub runs a workflow via `workflow_call`, github.workflow resolves
to the CALLER'S workflow name. So when staffml-preview-dev calls both
staffml-validate-dev and staffml-validate-vault via `uses:` from the
same parent run, the two reusable workflows collapsed into the same
concurrency group (parent-name + parent-run-id). With
`cancel-in-progress: true`, whichever queued first got cancelled by the
later one.
Concretely, on every push run since 6ddb82a71b (2026-05-02):
- Validate (Vault) jobs queue at parent+~3s with no runner assigned
- Validate (Dev) jobs queue at parent+~5s
- Vault jobs cancel ~1s later (cancel-in-progress fires when the
second occupant of the shared group enters)
Net effect: vault validation never ran but the StaffML preview-dev run
overall reported 'cancelled', flipping the README badge red despite
build + Validate (Dev) all green. 9 push runs in a row affected.
Fix: replace ${{ github.workflow }} with a literal workflow-identifying
string in each group key so the two reusable workflows live in disjoint
groups regardless of caller. The fallback to head_ref/run_id is kept,
so PR cancel-on-amend and standalone-vs-uses uniqueness still work.
Tested by dispatching staffml-validate-vault standalone before this
commit (run 25351824595): both jobs ran cleanly to success, confirming
the failure was purely the concurrency interaction between the two
reusable workflows in the same parent, not anything in the validation
logic itself.
386 lines
15 KiB
YAML
386 lines
15 KiB
YAML
name: '🎯 StaffML · ✅ Validate (Dev)'
|
|
|
|
# =============================================================================
|
|
# StaffML — Build & Validation
|
|
# =============================================================================
|
|
#
|
|
# Validates the StaffML Next.js interview-prep app before the preview/publish
|
|
# workflows touch it. Mirrors the build-side checks already done in
|
|
# staffml-preview-dev.yml so they fail fast in CI without paying the deploy
|
|
# cost. Adds a Tier 2 link-check pass.
|
|
#
|
|
# Flow:
|
|
# 1. TYPECHECK_AND_TEST — npm ci + tsc --noEmit + npm test
|
|
# 2. BUILD — Next.js static export (no deploy)
|
|
# 3. SMOKE — Vault integrity + corpus invariants
|
|
# 4. CHECK_LINKS — Lychee external-link reachability (non-blocking)
|
|
# 5. SUMMARY — Aggregate results
|
|
#
|
|
# Triggers:
|
|
# - push: dev branch, interviews/staffml/** paths
|
|
# - pull_request: interviews/staffml/** changes
|
|
# - workflow_dispatch: manual
|
|
#
|
|
# Deploys to: N/A (validate only)
|
|
#
|
|
# Related:
|
|
# - staffml-preview-dev.yml — Dev preview deploy
|
|
# - staffml-publish-live.yml — Production deploy
|
|
#
|
|
# =============================================================================
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
# Reusable: staffml-preview-dev.yml calls this via `uses:` so the deploy
|
|
# job can `needs:` a green validate. Standalone push/PR triggers below
|
|
# stay so the publish guard (infra-publish-guard.yml) and README badge
|
|
# still see direct runs on dev.
|
|
workflow_call:
|
|
pull_request:
|
|
paths:
|
|
- 'interviews/staffml/**'
|
|
- '.github/workflows/staffml-validate-dev.yml'
|
|
- '.github/workflows/staffml-preview-dev.yml'
|
|
push:
|
|
branches: [dev]
|
|
paths:
|
|
- 'interviews/staffml/**'
|
|
- '.github/workflows/staffml-validate-dev.yml'
|
|
- '.github/workflows/staffml-preview-dev.yml'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
# Group key uses a literal workflow-identifying string instead of
|
|
# ${{ github.workflow }} because the latter resolves to the CALLER's
|
|
# workflow name when this runs via `workflow_call`. Without the
|
|
# literal, both staffml-validate-dev and staffml-validate-vault — when
|
|
# called from the same staffml-preview-dev parent — collapse to the
|
|
# same group (parent-name + parent-run-id) and cancel each other.
|
|
# `head_ref || run_id` still preserves PR cancel-on-amend (head_ref
|
|
# set on PRs, stable across amends) and per-run uniqueness for push/
|
|
# dispatch (head_ref empty → run_id fallback) so a push to dev that
|
|
# triggers BOTH this workflow standalone AND Preview's `uses:` call
|
|
# into it doesn't share a group either.
|
|
group: staffml-validate-dev-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# Workflow-wide single source for paths and versions. See docs/CI-VARIABLES.md.
|
|
# The `paths:` trigger filter is intentionally NOT vars-ified — GitHub Actions
|
|
# evaluates trigger filters at workflow-load time, before vars are resolved.
|
|
STAFFML_ROOT: ${{ vars.STAFFML_ROOT || 'interviews/staffml' }}
|
|
VAULT_DIR: ${{ vars.VAULT_DIR || 'interviews/vault' }}
|
|
VAULT_CLI_DIR: ${{ vars.VAULT_CLI_DIR || 'interviews/vault-cli' }}
|
|
DEV_STAFFML_PATH: ${{ vars.DEV_STAFFML_PATH || 'staffml' }}
|
|
NODE_VERSION: ${{ vars.NODE_VERSION || '20' }}
|
|
PYTHON_VERSION: ${{ vars.PYTHON_VERSION || '3.12' }}
|
|
|
|
jobs:
|
|
# ===========================================================================
|
|
# Stage 1: Type-check + unit tests
|
|
# ===========================================================================
|
|
typecheck-and-test:
|
|
name: '🔍 Typecheck + Tests'
|
|
runs-on: ubuntu-latest
|
|
# vault build from YAMLs before tsc (corpus.json is gitignored)
|
|
timeout-minutes: 25
|
|
|
|
steps:
|
|
- name: 📥 Checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: 🐍 Setup Python (for vault-cli)
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ env.PYTHON_VERSION }}
|
|
|
|
- name: 🛠️ Install vault-cli
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -e "$VAULT_CLI_DIR/"
|
|
|
|
- name: 🔄 Regenerate staffml data bundle (corpus.json; not in git)
|
|
run: |
|
|
vault build --vault-dir "$VAULT_DIR" --release-id validate-dev-tsc --local-json
|
|
|
|
- name: 🔧 Setup Node.js
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
cache: 'npm'
|
|
cache-dependency-path: ${{ env.STAFFML_ROOT }}/package-lock.json
|
|
|
|
- name: 📦 Install dependencies
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
run: npm ci
|
|
|
|
- name: 🔍 Type check
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
run: npx tsc --noEmit
|
|
|
|
- name: 🧪 Run tests
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
run: npm test
|
|
|
|
# ===========================================================================
|
|
# Stage 2: Static build (no deploy)
|
|
# ===========================================================================
|
|
build:
|
|
name: '🔨 Build StaffML'
|
|
runs-on: ubuntu-latest
|
|
needs: typecheck-and-test
|
|
timeout-minutes: 25
|
|
|
|
steps:
|
|
- name: 📥 Checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: 🐍 Setup Python (for vault-cli)
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ env.PYTHON_VERSION }}
|
|
|
|
- name: 🛠️ Install vault-cli
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -e "$VAULT_CLI_DIR/"
|
|
|
|
- name: 🔄 Regenerate staffml data bundle (corpus.json; not in git)
|
|
run: |
|
|
vault build --vault-dir "$VAULT_DIR" --release-id validate-dev-build --local-json
|
|
|
|
- name: 🔧 Setup Node.js
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
cache: 'npm'
|
|
cache-dependency-path: ${{ env.STAFFML_ROOT }}/package-lock.json
|
|
|
|
- name: 📦 Install dependencies
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
run: npm ci
|
|
|
|
- name: 🔨 Build StaffML (static export)
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
env:
|
|
# Validation-only build; matches dev preview env so we exercise the
|
|
# same code paths but no deploy follows.
|
|
NEXT_PUBLIC_BASE_PATH: /cs249r_book_dev/${{ env.DEV_STAFFML_PATH }}
|
|
NEXT_PUBLIC_ECOSYSTEM_BASE: https://harvard-edge.github.io/cs249r_book_dev
|
|
run: npm run build
|
|
|
|
- name: 🔍 Validate build output
|
|
run: |
|
|
if [ ! -f "$STAFFML_ROOT/out/index.html" ]; then
|
|
echo "❌ CRITICAL: index.html missing from build output."
|
|
exit 1
|
|
fi
|
|
MISSING=0
|
|
# next.config.mjs sets `trailingSlash: true`, so /<page> emits
|
|
# out/<page>/index.html (not out/<page>.html). The 404 page is the
|
|
# one exception — Next.js still ships it flat as out/404.html so
|
|
# GitHub Pages can serve it as the not-found template.
|
|
for page in practice gauntlet progress about; do
|
|
if [ ! -f "$STAFFML_ROOT/out/${page}/index.html" ]; then
|
|
echo "❌ MISSING: ${page}/index.html"
|
|
MISSING=$((MISSING + 1))
|
|
fi
|
|
done
|
|
if [ ! -f "$STAFFML_ROOT/out/404.html" ]; then
|
|
echo "❌ MISSING: 404.html"
|
|
MISSING=$((MISSING + 1))
|
|
fi
|
|
if [ "$MISSING" -gt 0 ]; then
|
|
echo "❌ $MISSING critical pages missing."
|
|
exit 1
|
|
fi
|
|
echo "✅ StaffML built: $(find "$STAFFML_ROOT/out" -name '*.html' | wc -l) pages"
|
|
|
|
# ===========================================================================
|
|
# Stage 2.5: Headless-Chromium end-to-end smoke
|
|
# ===========================================================================
|
|
# Would have caught the hydration shape-mismatch bug from PR #1440
|
|
# before merge. Loads a handful of critical routes in a real browser
|
|
# and fails on uncaught page errors / console.error / missing content.
|
|
# See interviews/staffml/scripts/e2e-smoke.py for the allowlist +
|
|
# assertion catalog.
|
|
e2e-smoke:
|
|
name: '🎭 E2E Smoke (headless Chromium)'
|
|
runs-on: ubuntu-latest
|
|
needs: build
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: 📥 Checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: 🔧 Setup Node.js
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
cache: 'npm'
|
|
cache-dependency-path: ${{ env.STAFFML_ROOT }}/package-lock.json
|
|
|
|
- name: 📦 Install dependencies
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
run: npm ci
|
|
|
|
- name: 🐍 Setup Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ env.PYTHON_VERSION }}
|
|
|
|
- name: 🛠️ Install vault-cli
|
|
run: pip install -e "$VAULT_CLI_DIR/"
|
|
|
|
- name: 🔄 Regenerate corpus from YAMLs
|
|
# Ensures the E2E build sees current YAML state (same as preview/
|
|
# publish workflows do).
|
|
run: vault build --vault-dir "$VAULT_DIR" --release-id e2e-smoke --local-json
|
|
|
|
- name: 🔨 Build StaffML for E2E (no base-path)
|
|
working-directory: ${{ env.STAFFML_ROOT }}
|
|
# Builds at the root path so we can serve out/ locally without
|
|
# mirroring the /cs249r_book_dev/staffml/ prefix. Base-path
|
|
# fidelity is already validated by the main `build` job above.
|
|
env:
|
|
NEXT_PUBLIC_VAULT_API: https://staffml-vault.mlsysbook-ai-account.workers.dev
|
|
run: npm run build
|
|
|
|
- name: 🎭 Install Playwright + Chromium
|
|
run: |
|
|
pip install playwright
|
|
playwright install --with-deps chromium
|
|
|
|
- name: 🧪 Run E2E smoke
|
|
run: python3 "$STAFFML_ROOT/scripts/e2e"-smoke.py
|
|
|
|
# ===========================================================================
|
|
# Stage 3: Vault integrity + corpus smoke tests
|
|
# ===========================================================================
|
|
smoke:
|
|
name: '🧪 Vault + Corpus Smoke Tests'
|
|
runs-on: ubuntu-latest
|
|
# vault build from YAMLs can exceed 5m on cold runners
|
|
timeout-minutes: 20
|
|
|
|
steps:
|
|
- name: 📥 Checkout
|
|
uses: actions/checkout@v6
|
|
|
|
- name: 🐍 Setup Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ env.PYTHON_VERSION }}
|
|
|
|
- name: 🛠️ Install vault-cli
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -e "$VAULT_CLI_DIR/"
|
|
|
|
- name: 🔄 Regenerate staffml bundle (corpus.json is a build artifact)
|
|
# Same contract as staffml-preview-dev / publish-live: YAMLs are
|
|
# source of truth; --local-json emits staffml/src/data/corpus.json
|
|
# for validate-vault.py and inline corpus invariants (not in git).
|
|
run: |
|
|
vault build --vault-dir "$VAULT_DIR" --release-id validate-dev-smoke --local-json
|
|
|
|
- name: 🐍 Install PyYAML (for schema drift check)
|
|
run: pip install pyyaml
|
|
|
|
- name: 🔐 Validate vault integrity
|
|
run: python3 "$STAFFML_ROOT/scripts/validate"-vault.py
|
|
|
|
- name: 🧬 Vault schema drift check (enums.py ↔ LinkML)
|
|
run: python3 "$VAULT_CLI_DIR/scripts/check_schema_sync".py
|
|
|
|
- name: 🧪 Corpus invariants
|
|
run: |
|
|
python3 - <<'PYEOF'
|
|
import json, sys
|
|
|
|
with open('interviews/staffml/src/data/corpus.json') as f:
|
|
corpus = json.load(f)
|
|
assert len(corpus) >= 4000, f'Corpus too small: {len(corpus)} questions'
|
|
print(f'✅ Corpus: {len(corpus)} questions')
|
|
|
|
required = ['id', 'title', 'level', 'track', 'scenario',
|
|
'competency_area', 'topic', 'zone', 'details']
|
|
missing = []
|
|
for q in corpus:
|
|
for field in required:
|
|
if not q.get(field):
|
|
missing.append(f"{q.get('id', '???')} missing {field}")
|
|
if missing:
|
|
print(f'⚠️ {len(missing)} questions with missing fields (non-fatal)')
|
|
for m in missing[:5]:
|
|
print(f' {m}')
|
|
|
|
valid_levels = {'L1','L2','L3','L4','L5','L6','L6+'}
|
|
bad = [q['id'] for q in corpus if q.get('level') not in valid_levels]
|
|
assert not bad, f'{len(bad)} invalid levels: {bad[:3]}'
|
|
print('✅ All levels valid (L1-L6+)')
|
|
|
|
with open('interviews/staffml/src/data/vault-manifest.json') as f:
|
|
manifest = json.load(f)
|
|
assert 'releaseId' in manifest, 'Manifest missing releaseId'
|
|
assert 'releaseHash' in manifest and len(manifest['releaseHash']) >= 16, \
|
|
'Manifest missing or truncated releaseHash'
|
|
assert manifest.get('questionCount') == len(corpus), \
|
|
f"Manifest count mismatch: {manifest.get('questionCount')} vs {len(corpus)}"
|
|
print(f"✅ Manifest: v{manifest['releaseId']} ({manifest['questionCount']} Qs, hash {manifest['releaseHash'][:7]})")
|
|
PYEOF
|
|
|
|
# ===========================================================================
|
|
# Stage 4: Link integrity (Tier 2 — non-blocking baseline)
|
|
# ===========================================================================
|
|
# The StaffML app is a Next.js TSX codebase, so the markdown link checker
|
|
# only catches links inside .md/.mdx prose (READMEs, paper). External link
|
|
# validation through Lychee covers the rest. Flip fail_on_broken=true once
|
|
# baseline is clean.
|
|
check-links:
|
|
name: '🔗 Check Links'
|
|
uses: ./.github/workflows/infra-link-check.yml
|
|
with:
|
|
path_pattern: './interviews/**/*.{md,mdx,qmd}'
|
|
lycheeignore_path: 'shared/config/.lycheeignore'
|
|
fail_on_broken: false
|
|
max_concurrency: 8
|
|
|
|
# ===========================================================================
|
|
# Summary
|
|
# ===========================================================================
|
|
summary:
|
|
name: '📊 Summary'
|
|
runs-on: ubuntu-latest
|
|
needs: [typecheck-and-test, build, e2e-smoke, smoke, check-links]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: 📊 Generate Summary
|
|
run: |
|
|
echo "## 🎯 StaffML Validation Results" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Check | Status |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
|
|
echo "| 🔍 Typecheck + Tests | ${{ needs.typecheck-and-test.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| 🔨 Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| 🎭 E2E Smoke | ${{ needs.e2e-smoke.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| 🧪 Vault Smoke Tests | ${{ needs.smoke.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| 🔗 Link Check | ${{ needs.check-links.result }} (non-blocking) |" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: ❌ Check for failures
|
|
run: |
|
|
if [ "${{ needs.typecheck-and-test.result }}" = "failure" ] || \
|
|
[ "${{ needs.build.result }}" = "failure" ] || \
|
|
[ "${{ needs.e2e-smoke.result }}" = "failure" ] || \
|
|
[ "${{ needs.smoke.result }}" = "failure" ]; then
|
|
echo "❌ Validation failed"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.check-links.result }}" = "failure" ]; then
|
|
echo "⚠️ Link check found issues (non-blocking)"
|
|
fi
|
|
echo "✅ Core checks passed"
|