diff --git a/book/vscode-ext/package.json b/book/vscode-ext/package.json index 35ebf54a0..a8601a78d 100644 --- a/book/vscode-ext/package.json +++ b/book/vscode-ext/package.json @@ -80,6 +80,9 @@ { "command": "mlsysbook.refreshQmdDiagnostics", "title": "MLSysBook: Refresh QMD Diagnostics" }, { "command": "mlsysbook.setNavigatorFilterPreset", "title": "MLSysBook: Set Navigator Filter Preset" }, { "command": "mlsysbook.renameLabelReferences", "title": "MLSysBook: Rename Label and References" }, + { "command": "mlsysbook.addSectionIds", "title": "MLSysBook: Add Missing Section IDs" }, + { "command": "mlsysbook.verifySectionIds", "title": "MLSysBook: Verify Section IDs" }, + { "command": "mlsysbook.validateCrossReferences", "title": "MLSysBook: Validate Cross-References" }, { "command": "mlsysbook.historyRerunSession", "title": "MLSysBook: Rerun Run Session" }, { "command": "mlsysbook.historyRerunFailed", "title": "MLSysBook: Rerun Failed Chapters (Run Session)" }, { "command": "mlsysbook.historyOpenOutput", "title": "MLSysBook: Open Run Output" }, @@ -160,8 +163,8 @@ }, "mlsysbook.enableQmdDiagnostics": { "type": "boolean", - "default": false, - "description": "Enable MLSysBook inline QMD diagnostics (cross-reference and inline-python checks). Disable to avoid diagnostic squiggles while writing." + "default": true, + "description": "Enable MLSysBook QMD diagnostics on save (cross-reference validation across all files, inline-python checks). Disable to suppress diagnostic squiggles." }, "mlsysbook.qmdVisualPreset": { "type": "string", diff --git a/book/vscode-ext/src/extension.ts b/book/vscode-ext/src/extension.ts index ee73eae3e..8f3d69c09 100644 --- a/book/vscode-ext/src/extension.ts +++ b/book/vscode-ext/src/extension.ts @@ -10,7 +10,7 @@ import { ChapterNavigatorProvider } from './providers/chapterNavigatorProvider'; import { RunHistoryProvider } from './providers/runHistoryProvider'; import { QmdFoldingProvider } from './providers/qmdFoldingProvider'; import { QmdAutoFoldManager } from './providers/qmdAutoFoldManager'; -import { QmdDiagnosticsManager } from './validation/qmdDiagnostics'; +import { QmdDiagnosticsManager, WorkspaceLabelIndex } from './validation/qmdDiagnostics'; import { QmdChunkHighlighter } from './providers/qmdChunkHighlighter'; import { QmdPythonValueResolver } from './providers/qmdPythonValueResolver'; import { QmdPythonHoverProvider, QmdPythonGhostText, QmdPythonCodeLensProvider } from './providers/qmdInlinePythonProviders'; @@ -27,6 +27,7 @@ import { revealRunTerminal, showLastFailureDetails, setExecutionModeInteractively, + runInVisibleTerminal, } from './utils/terminal'; import { ChapterOrderSource } from './types'; @@ -64,8 +65,11 @@ export function activate(context: vscode.ExtensionContext): void { const runHistoryProvider = new RunHistoryProvider(); const qmdFoldingProvider = new QmdFoldingProvider(); const qmdAutoFoldManager = new QmdAutoFoldManager(); + const workspaceLabelIndex = new WorkspaceLabelIndex(); const diagnosticsManager = new QmdDiagnosticsManager(); + diagnosticsManager.setWorkspaceIndex(workspaceLabelIndex); const chunkHighlighter = new QmdChunkHighlighter(); + chunkHighlighter.setWorkspaceIndex(workspaceLabelIndex); // Inline Python value resolution (hover, ghost text, CodeLens) let pythonResolver: QmdPythonValueResolver | undefined; @@ -86,6 +90,7 @@ export function activate(context: vscode.ExtensionContext): void { const preset = NAVIGATOR_PRESETS.find(p => p.id === defaultPreset) ?? NAVIGATOR_PRESETS[0]; void config.update('navigatorVisibleEntryKinds', preset.kinds, vscode.ConfigurationTarget.Workspace); navigatorProvider.refreshFromEditor(vscode.window.activeTextEditor); + workspaceLabelIndex.start(); diagnosticsManager.start(); chunkHighlighter.start(); pythonGhostText?.start(); @@ -106,6 +111,7 @@ export function activate(context: vscode.ExtensionContext): void { ), runHistoryProvider, qmdAutoFoldManager, + workspaceLabelIndex, diagnosticsManager, chunkHighlighter, ); @@ -244,6 +250,36 @@ export function activate(context: vscode.ExtensionContext): void { await vscode.commands.executeCommand('workbench.action.openSettings', '@ext:mlsysbook.mlsysbook-workbench'); }), vscode.commands.registerCommand('mlsysbook.renameLabelReferences', () => void renameLabelAcrossWorkspace()), + + // Section ID management + vscode.commands.registerCommand('mlsysbook.addSectionIds', () => { + if (!root) { return; } + const editor = vscode.window.activeTextEditor; + const target = editor?.document.uri.fsPath.endsWith('.qmd') + ? `-f ${editor.document.uri.fsPath}` + : '-d book/quarto/contents/vol1/'; + runInVisibleTerminal( + `python3 book/tools/scripts/content/manage_section_ids.py ${target} --force`, + root, + 'Add Section IDs', + ); + }), + vscode.commands.registerCommand('mlsysbook.verifySectionIds', () => { + if (!root) { return; } + runInVisibleTerminal( + 'python3 book/tools/scripts/content/manage_section_ids.py -d book/quarto/contents/vol1/ --verify --force', + root, + 'Verify Section IDs', + ); + }), + vscode.commands.registerCommand('mlsysbook.validateCrossReferences', () => { + if (!root) { return; } + runInVisibleTerminal( + 'python3 book/tools/scripts/content/check_unreferenced_labels.py ./book/quarto/contents/vol1/', + root, + 'Validate Cross-References', + ); + }), ); // Register all command groups diff --git a/book/vscode-ext/src/providers/qmdChunkHighlighter.ts b/book/vscode-ext/src/providers/qmdChunkHighlighter.ts index fdbb53719..8928c3a86 100644 --- a/book/vscode-ext/src/providers/qmdChunkHighlighter.ts +++ b/book/vscode-ext/src/providers/qmdChunkHighlighter.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import { resolveHighlightStyle, type QmdColorOverrides, type VisualPreset } from './qmdHighlightPalette'; +import type { WorkspaceLabelIndex } from '../validation/qmdDiagnostics'; function isQmdEditor(editor: vscode.TextEditor | undefined): editor is vscode.TextEditor { return Boolean(editor && editor.document.uri.fsPath.endsWith('.qmd')); @@ -45,12 +46,28 @@ export class QmdChunkHighlighter implements vscode.Disposable { private divFenceMarkerDecoration: vscode.TextEditorDecorationType | undefined; private inlinePythonKeywordDecoration: vscode.TextEditorDecorationType | undefined; private inlinePythonVarDecoration: vscode.TextEditorDecorationType | undefined; + private brokenReferenceDecoration: vscode.TextEditorDecorationType | undefined; private refreshTimer: NodeJS.Timeout | undefined; + private workspaceIndex: WorkspaceLabelIndex | undefined; constructor() { this.recreateDecorations(); } + /** Inject the workspace label index for broken-reference highlighting. */ + setWorkspaceIndex(index: WorkspaceLabelIndex): void { + this.workspaceIndex = index; + // Re-apply decorations when the index updates + this.disposables.push( + index.onDidUpdate(() => { + const editor = vscode.window.activeTextEditor; + if (isQmdEditor(editor)) { + this.debouncedApply(editor); + } + }), + ); + } + start(): void { this.applyToEditor(vscode.window.activeTextEditor); this.disposables.push( @@ -247,6 +264,7 @@ export class QmdChunkHighlighter implements vscode.Disposable { this.divFenceMarkerDecoration?.dispose(); this.inlinePythonKeywordDecoration?.dispose(); this.inlinePythonVarDecoration?.dispose(); + this.brokenReferenceDecoration?.dispose(); const preset = this.getVisualPreset(); const style = resolveHighlightStyle(preset, this.getColorOverrides()); @@ -384,6 +402,11 @@ export class QmdChunkHighlighter implements vscode.Disposable { fontWeight: style.fontWeight === 'normal' ? '500' : '600', borderRadius: '0 3px 3px 0', }); + this.brokenReferenceDecoration = vscode.window.createTextEditorDecorationType({ + color: 'rgba(239, 68, 68, 0.9)', + textDecoration: 'underline wavy rgba(239, 68, 68, 0.6)', + fontWeight: style.fontWeight, + }); } private debouncedApply(editor: vscode.TextEditor): void { @@ -426,6 +449,21 @@ export class QmdChunkHighlighter implements vscode.Disposable { const tableRefRanges: vscode.Range[] = []; const listingRefRanges: vscode.Range[] = []; const equationRefRanges: vscode.Range[] = []; + const brokenRefRanges: vscode.Range[] = []; + + // Collect local label definitions for broken-ref checking + const localLabels = new Set(); + const fullText = document.getText(); + const labelCollectRegex = /\{#((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)\}/g; + const yamlLabelCollectRegex = /#\|\s*(?:label|fig-label|tbl-label|lst-label):\s*((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)/g; + let labelMatch: RegExpExecArray | null; + while ((labelMatch = labelCollectRegex.exec(fullText)) !== null) { + localLabels.add(labelMatch[1]); + } + while ((labelMatch = yamlLabelCollectRegex.exec(fullText)) !== null) { + localLabels.add(labelMatch[1]); + } + const hasIndex = this.workspaceIndex?.isReady() ?? false; const labelDefinitionRanges: vscode.Range[] = []; const sectionLabelDefinitionRanges: vscode.Range[] = []; const figureLabelDefinitionRanges: vscode.Range[] = []; @@ -746,45 +784,38 @@ export class QmdChunkHighlighter implements vscode.Disposable { } structuralRefRegex.lastIndex = 0; - while ((match = sectionRefRegex.exec(text)) !== null) { - sectionRefRanges.push(new vscode.Range( - new vscode.Position(line, match.index), - new vscode.Position(line, match.index + match[0].length), - )); - } - sectionRefRegex.lastIndex = 0; + // Helper: check if a reference label is valid (exists locally or in workspace) + const isRefValid = (label: string): boolean => { + if (localLabels.has(label)) { return true; } + if (hasIndex && this.workspaceIndex!.hasLabel(label)) { return true; } + return false; + }; - while ((match = figureRefRegex.exec(text)) !== null) { - figureRefRanges.push(new vscode.Range( - new vscode.Position(line, match.index), - new vscode.Position(line, match.index + match[0].length), - )); - } - figureRefRegex.lastIndex = 0; + // For each typed reference, check validity and route to broken or typed range + const typedRefConfigs: Array<{ regex: RegExp; ranges: vscode.Range[] }> = [ + { regex: sectionRefRegex, ranges: sectionRefRanges }, + { regex: figureRefRegex, ranges: figureRefRanges }, + { regex: tableRefRegex, ranges: tableRefRanges }, + { regex: listingRefRegex, ranges: listingRefRanges }, + { regex: equationRefRegex, ranges: equationRefRanges }, + ]; - while ((match = tableRefRegex.exec(text)) !== null) { - tableRefRanges.push(new vscode.Range( - new vscode.Position(line, match.index), - new vscode.Position(line, match.index + match[0].length), - )); + for (const { regex, ranges } of typedRefConfigs) { + while ((match = regex.exec(text)) !== null) { + const range = new vscode.Range( + new vscode.Position(line, match.index), + new vscode.Position(line, match.index + match[0].length), + ); + // Extract the label (strip leading @) + const label = match[0].startsWith('@') ? match[0].slice(1) : match[0]; + if (hasIndex && !isRefValid(label)) { + brokenRefRanges.push(range); + } else { + ranges.push(range); + } + } + regex.lastIndex = 0; } - tableRefRegex.lastIndex = 0; - - while ((match = listingRefRegex.exec(text)) !== null) { - listingRefRanges.push(new vscode.Range( - new vscode.Position(line, match.index), - new vscode.Position(line, match.index + match[0].length), - )); - } - listingRefRegex.lastIndex = 0; - - while ((match = equationRefRegex.exec(text)) !== null) { - equationRefRanges.push(new vscode.Range( - new vscode.Position(line, match.index), - new vscode.Position(line, match.index + match[0].length), - )); - } - equationRefRegex.lastIndex = 0; } if (highlightInlinePython && !inFence) { @@ -877,6 +908,9 @@ export class QmdChunkHighlighter implements vscode.Disposable { if (this.equationReferenceDecoration) { editor.setDecorations(this.equationReferenceDecoration, equationRefRanges); } + if (this.brokenReferenceDecoration) { + editor.setDecorations(this.brokenReferenceDecoration, brokenRefRanges); + } if (this.labelDefinitionDecoration) { editor.setDecorations(this.labelDefinitionDecoration, labelDefinitionRanges); } @@ -988,6 +1022,9 @@ export class QmdChunkHighlighter implements vscode.Disposable { if (this.inlinePythonVarDecoration) { editor.setDecorations(this.inlinePythonVarDecoration, []); } + if (this.brokenReferenceDecoration) { + editor.setDecorations(this.brokenReferenceDecoration, []); + } } dispose(): void { diff --git a/book/vscode-ext/src/validation/qmdDiagnostics.ts b/book/vscode-ext/src/validation/qmdDiagnostics.ts index c3a3c39e6..89c675e84 100644 --- a/book/vscode-ext/src/validation/qmdDiagnostics.ts +++ b/book/vscode-ext/src/validation/qmdDiagnostics.ts @@ -6,16 +6,52 @@ function isQmdDocument(document: vscode.TextDocument): boolean { return document.uri.fsPath.endsWith('.qmd'); } +// ─── Label collection patterns ────────────────────────────────────────────── + +/** Inline label definitions: {#sec-foo}, {#fig-bar}, etc. */ +const INLINE_LABEL_REGEX = /\{#((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)\}/g; + +/** YAML-style label definitions: #| label: fig-foo, #| fig-label: fig-foo, etc. */ +const YAML_LABEL_REGEX = /#\|\s*(?:label|fig-label|tbl-label|lst-label):\s*((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)/g; + +/** Cross-reference pattern: @sec-foo, @fig-bar, etc. */ +const CROSSREF_REGEX = /@((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)/g; + +/** + * Collect all label definitions from a document's text. + * Handles both inline {#label} and YAML #| label: formats. + */ function collectDefinedLabels(text: string): Set { const labels = new Set(); - const labelRegex = /\{#((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)\}/g; + let match: RegExpExecArray | null; - while ((match = labelRegex.exec(text)) !== null) { + + INLINE_LABEL_REGEX.lastIndex = 0; + while ((match = INLINE_LABEL_REGEX.exec(text)) !== null) { labels.add(match[1]); } + + YAML_LABEL_REGEX.lastIndex = 0; + while ((match = YAML_LABEL_REGEX.exec(text)) !== null) { + labels.add(match[1]); + } + return labels; } +/** + * Collect all cross-references from a document's text. + */ +function collectReferences(text: string): Array<{ ref: string; index: number }> { + const refs: Array<{ ref: string; index: number }> = []; + let match: RegExpExecArray | null; + CROSSREF_REGEX.lastIndex = 0; + while ((match = CROSSREF_REGEX.exec(text)) !== null) { + refs.push({ ref: match[1], index: match.index }); + } + return refs; +} + function collectDefinedPythonVariables(text: string): Set { const vars = new Set(); const fenceRegex = /```(?:\{python\}|python)\s*\n([\s\S]*?)```/g; @@ -37,27 +73,162 @@ function makeRangeForMatch(document: vscode.TextDocument, offset: number, length return new vscode.Range(start, end); } -function buildDiagnostics(document: vscode.TextDocument): vscode.Diagnostic[] { - const diagnostics: vscode.Diagnostic[] = []; - const text = document.getText(); - const definedLabels = collectDefinedLabels(text); - const definedPythonVars = collectDefinedPythonVariables(text); +// ─── Workspace Label Index ────────────────────────────────────────────────── - const crossrefRegex = /@((?:sec|fig|tbl|eq|lst)-[A-Za-z0-9:_-]+)/g; - let crossrefMatch: RegExpExecArray | null; - while ((crossrefMatch = crossrefRegex.exec(text)) !== null) { - const ref = crossrefMatch[1]; - if (!definedLabels.has(ref)) { - diagnostics.push( - new vscode.Diagnostic( - makeRangeForMatch(document, crossrefMatch.index + 1, ref.length), - `Unresolved cross-reference: @${ref}`, - vscode.DiagnosticSeverity.Warning, - ), - ); +/** + * Maintains a workspace-wide index of all label definitions across .qmd files. + * Updated incrementally when individual files are saved. + */ +export class WorkspaceLabelIndex implements vscode.Disposable { + /** Map from label string to the set of URIs where it's defined. */ + private readonly labelToUris = new Map>(); + /** Map from file URI string to the set of labels defined in that file. */ + private readonly uriToLabels = new Map>(); + private readonly disposables: vscode.Disposable[] = []; + private initialized = false; + + private readonly onDidUpdateEmitter = new vscode.EventEmitter(); + /** Fires when the index has been updated (for triggering re-validation). */ + readonly onDidUpdate = this.onDidUpdateEmitter.event; + + start(): void { + // Build initial index + this.buildFullIndex().then(() => { + this.initialized = true; + this.onDidUpdateEmitter.fire(); + }); + + // Update index when .qmd files are saved + this.disposables.push( + vscode.workspace.onDidSaveTextDocument(document => { + if (isQmdDocument(document)) { + this.updateFileLabels(document.uri.toString(), document.getText()); + this.onDidUpdateEmitter.fire(); + } + }), + // Handle file deletions + vscode.workspace.onDidDeleteFiles(event => { + let changed = false; + for (const uri of event.files) { + if (uri.fsPath.endsWith('.qmd')) { + this.removeFile(uri.toString()); + changed = true; + } + } + if (changed) { this.onDidUpdateEmitter.fire(); } + }), + ); + } + + /** Check if a label exists anywhere in the workspace. */ + hasLabel(label: string): boolean { + const uris = this.labelToUris.get(label); + return !!uris && uris.size > 0; + } + + /** Check if the index has been built at least once. */ + isReady(): boolean { + return this.initialized; + } + + /** Get the total number of indexed labels. */ + get size(): number { + return this.labelToUris.size; + } + + /** Get all known labels (for completions, etc.). */ + allLabels(): string[] { + return Array.from(this.labelToUris.keys()); + } + + private async buildFullIndex(): Promise { + const files = await vscode.workspace.findFiles('**/*.qmd', '**/node_modules/**'); + for (const uri of files) { + try { + const doc = await vscode.workspace.openTextDocument(uri); + this.updateFileLabels(uri.toString(), doc.getText()); + } catch { + // File may have been deleted between findFiles and openTextDocument + } } } + private updateFileLabels(uriStr: string, text: string): void { + // Remove old labels for this file + this.removeFile(uriStr); + + // Collect new labels + const labels = collectDefinedLabels(text); + this.uriToLabels.set(uriStr, labels); + + for (const label of labels) { + let uris = this.labelToUris.get(label); + if (!uris) { + uris = new Set(); + this.labelToUris.set(label, uris); + } + uris.add(uriStr); + } + } + + private removeFile(uriStr: string): void { + const oldLabels = this.uriToLabels.get(uriStr); + if (oldLabels) { + for (const label of oldLabels) { + const uris = this.labelToUris.get(label); + if (uris) { + uris.delete(uriStr); + if (uris.size === 0) { + this.labelToUris.delete(label); + } + } + } + this.uriToLabels.delete(uriStr); + } + } + + dispose(): void { + this.onDidUpdateEmitter.dispose(); + this.disposables.forEach(d => d.dispose()); + this.labelToUris.clear(); + this.uriToLabels.clear(); + } +} + +// ─── Diagnostics Builder ──────────────────────────────────────────────────── + +function buildDiagnostics( + document: vscode.TextDocument, + workspaceIndex?: WorkspaceLabelIndex, +): vscode.Diagnostic[] { + const diagnostics: vscode.Diagnostic[] = []; + const text = document.getText(); + const localLabels = collectDefinedLabels(text); + const definedPythonVars = collectDefinedPythonVariables(text); + + // Cross-reference validation + const refs = collectReferences(text); + for (const { ref, index } of refs) { + // Check local labels first (fast path), then workspace index + const isLocal = localLabels.has(ref); + const isWorkspace = workspaceIndex?.hasLabel(ref) ?? false; + + if (!isLocal && !isWorkspace) { + const diagnostic = new vscode.Diagnostic( + makeRangeForMatch(document, index + 1, ref.length), + workspaceIndex?.isReady() + ? `Unresolved cross-reference: @${ref} (not found in any .qmd file)` + : `Unresolved cross-reference: @${ref} (workspace index loading...)`, + workspaceIndex?.isReady() + ? vscode.DiagnosticSeverity.Warning + : vscode.DiagnosticSeverity.Information, + ); + diagnostic.source = DIAGNOSTIC_SOURCE; + diagnostics.push(diagnostic); + } + } + + // Inline Python validation const inlinePythonRegex = /`\{python\}\s+([^`]+)`/g; let inlineMatch: RegExpExecArray | null; while ((inlineMatch = inlinePythonRegex.exec(text)) !== null) { @@ -113,28 +284,56 @@ function buildDiagnostics(document: vscode.TextDocument): vscode.Diagnostic[] { return diagnostics; } +// ─── Diagnostics Manager ──────────────────────────────────────────────────── + export class QmdDiagnosticsManager implements vscode.Disposable { private readonly collection: vscode.DiagnosticCollection; private readonly disposables: vscode.Disposable[] = []; private refreshTimer: NodeJS.Timeout | undefined; + private workspaceIndex: WorkspaceLabelIndex | undefined; constructor() { this.collection = vscode.languages.createDiagnosticCollection(DIAGNOSTIC_SOURCE); this.disposables.push(this.collection); } + /** + * Inject the workspace label index for cross-file reference validation. + * Must be called before start(). + */ + setWorkspaceIndex(index: WorkspaceLabelIndex): void { + this.workspaceIndex = index; + } + start(): void { - this.refreshActiveEditorDiagnostics(); + // Only validate on save and editor switch — NOT on every keystroke this.disposables.push( vscode.window.onDidChangeActiveTextEditor(() => this.refreshActiveEditorDiagnostics()), vscode.workspace.onDidSaveTextDocument(document => this.refreshDocumentDiagnostics(document)), - vscode.workspace.onDidChangeTextDocument(event => this.debouncedRefresh(event.document)), vscode.workspace.onDidChangeConfiguration(event => { if (event.affectsConfiguration('mlsysbook.enableQmdDiagnostics')) { - this.refreshActiveEditorDiagnostics(); + if (this.isEnabled()) { + this.refreshActiveEditorDiagnostics(); + } else { + this.collection.clear(); + } } }), ); + + // Re-validate active editor when workspace index updates + if (this.workspaceIndex) { + this.disposables.push( + this.workspaceIndex.onDidUpdate(() => { + if (this.isEnabled()) { + this.refreshActiveEditorDiagnostics(); + } + }), + ); + } + + // Initial validation + this.refreshActiveEditorDiagnostics(); } refreshActiveEditorDiagnostics(): void { @@ -148,24 +347,14 @@ export class QmdDiagnosticsManager implements vscode.Disposable { this.collection.delete(document.uri); return; } - const diagnostics = buildDiagnostics(document); + const diagnostics = buildDiagnostics(document, this.workspaceIndex); this.collection.set(document.uri, diagnostics); } private isEnabled(): boolean { return vscode.workspace .getConfiguration('mlsysbook') - .get('enableQmdDiagnostics', false); - } - - private debouncedRefresh(document: vscode.TextDocument): void { - if (!isQmdDocument(document)) { return; } - if (this.refreshTimer) { - clearTimeout(this.refreshTimer); - } - this.refreshTimer = setTimeout(() => { - this.refreshDocumentDiagnostics(document); - }, 300); + .get('enableQmdDiagnostics', true); } dispose(): void {