diff --git a/src/kohaku-board-ui/src/components.d.ts b/src/kohaku-board-ui/src/components.d.ts index 36068bf..efb833d 100644 --- a/src/kohaku-board-ui/src/components.d.ts +++ b/src/kohaku-board-ui/src/components.d.ts @@ -14,6 +14,8 @@ declare module 'vue' { ConfigurableChartCard: typeof import('./components/ConfigurableChartCard.vue')['default'] DataTable: typeof import('./components/DataTable.vue')['default'] ElAlert: typeof import('element-plus/es')['ElAlert'] + ElBreadcrumb: typeof import('element-plus/es')['ElBreadcrumb'] + ElBreadcrumbItem: typeof import('element-plus/es')['ElBreadcrumbItem'] ElButton: typeof import('element-plus/es')['ElButton'] ElCard: typeof import('element-plus/es')['ElCard'] ElCol: typeof import('element-plus/es')['ElCol'] diff --git a/src/kohaku-board-ui/src/components/ConfigurableChartCard.vue b/src/kohaku-board-ui/src/components/ConfigurableChartCard.vue index 6e9a895..b3a9890 100644 --- a/src/kohaku-board-ui/src/components/ConfigurableChartCard.vue +++ b/src/kohaku-board-ui/src/components/ConfigurableChartCard.vue @@ -126,10 +126,14 @@ const hasDataError = computed(() => { const xData = props.sparseData[config.xMetric]; if (!xData || xData.length === 0) return true; - // Check if all y metrics are empty or missing + // Check if all y metrics have no valid data (including NaN/inf as valid!) const allEmpty = config.yMetrics.every((yMetric) => { const yData = props.sparseData[yMetric]; - return !yData || yData.length === 0; + if (!yData || yData.length === 0) return true; + + // Check if there's at least ONE non-null value (including NaN/inf) + const hasAnyData = yData.some((val) => val !== null); + return !hasAnyData; }); return allEmpty; @@ -156,6 +160,9 @@ const processedChartData = computed(() => { const x = []; const y = []; let lastXValue = null; + let nanCount = 0; + let infCount = 0; + let negInfCount = 0; for (let i = 0; i < xData.length; i++) { let xVal = xData[i]; @@ -171,12 +178,38 @@ const processedChartData = computed(() => { } if (xVal !== null) lastXValue = xVal; + + // Include NaN/inf values (they are not null!) + // Only skip if yVal is actually null (missing data) if (yVal !== null && lastXValue !== null) { x.push(lastXValue); y.push(yVal); + + // Count special values for logging + if (isNaN(yVal)) { + nanCount++; + } else if (yVal === Infinity) { + infCount++; + } else if (yVal === -Infinity) { + negInfCount++; + } } } + const specialValuesLog = []; + if (nanCount > 0) specialValuesLog.push(`NaN=${nanCount}`); + if (infCount > 0) specialValuesLog.push(`+inf=${infCount}`); + if (negInfCount > 0) specialValuesLog.push(`-inf=${negInfCount}`); + + console.log( + `[${props.cardId}] ${yMetric}: collected ${y.length} points` + + (specialValuesLog.length > 0 + ? ` (${specialValuesLog.join(", ")})` + : ""), + ); + + // Return series even if all values are NaN/inf (x/y might be empty but we still need the series) + // The LinePlot component will handle special values return { name: yMetric, x, y }; }) .filter((d) => d !== null); diff --git a/src/kohaku-board-ui/src/components/LinePlot.vue b/src/kohaku-board-ui/src/components/LinePlot.vue index 25b7bba..2ddf3cd 100644 --- a/src/kohaku-board-ui/src/components/LinePlot.vue +++ b/src/kohaku-board-ui/src/components/LinePlot.vue @@ -282,24 +282,95 @@ const processedData = computed(() => { const rate = config.downsampleRate; return props.data.map((series) => { - const smoothedY = applySmoothing(series.y, mode, value); - const downsampledSmoothed = downsampleData(series.x, smoothedY, rate); + console.log( + `[LinePlot] Processing series: ${series.name}, data length: ${series.y.length}`, + ); + + // Build line segments separated by NaN/inf breaks + // If we have: [1, 2, NaN, 3, 4], we create 2 segments: [[1,2], [3,4]] + const segments = []; // Each segment: {x: [], y: []} + const nanX = []; + const infX = []; + const negInfX = []; + + let currentSegmentX = []; + let currentSegmentY = []; + + for (let i = 0; i < series.y.length; i++) { + const yVal = series.y[i]; + const xVal = series.x[i]; + + if (Number.isNaN(yVal)) { + // NaN breaks the line - finish current segment + if (currentSegmentX.length > 0) { + segments.push({ x: currentSegmentX, y: currentSegmentY }); + currentSegmentX = []; + currentSegmentY = []; + } + nanX.push(xVal); + console.log( + `[LinePlot] ${series.name}[${i}]: NaN at x=${xVal} - breaking line`, + ); + } else if (yVal === Infinity) { + // +inf breaks the line + if (currentSegmentX.length > 0) { + segments.push({ x: currentSegmentX, y: currentSegmentY }); + currentSegmentX = []; + currentSegmentY = []; + } + infX.push(xVal); + console.log( + `[LinePlot] ${series.name}[${i}]: +inf at x=${xVal} - breaking line`, + ); + } else if (yVal === -Infinity) { + // -inf breaks the line + if (currentSegmentX.length > 0) { + segments.push({ x: currentSegmentX, y: currentSegmentY }); + currentSegmentX = []; + currentSegmentY = []; + } + negInfX.push(xVal); + console.log( + `[LinePlot] ${series.name}[${i}]: -inf at x=${xVal} - breaking line`, + ); + } else { + // Normal finite value - add to current segment + currentSegmentX.push(xVal); + currentSegmentY.push(yVal); + } + } + + // Don't forget the last segment + if (currentSegmentX.length > 0) { + segments.push({ x: currentSegmentX, y: currentSegmentY }); + } + + console.log( + `[LinePlot] ${series.name}: ${segments.length} segments, NaN=${nanX.length}, +inf=${infX.length}, -inf=${negInfX.length}`, + ); + + // Process each segment separately (smoothing + downsampling) + const processedSegments = segments.map((seg) => { + const smoothedY = applySmoothing(seg.y, mode, value); + return downsampleData(seg.x, smoothedY, rate); + }); const result = { name: series.name, - x: downsampledSmoothed.x, - y: downsampledSmoothed.y, + segments: processedSegments, // Multiple line segments original: null, - originalY: null, + originalSegments: null, + // Store special value x-positions for marker rendering + nanX, + infX, + negInfX, }; if (mode !== "disabled" && config.showOriginal) { - const downsampledOriginal = downsampleData(series.x, series.y, rate); - result.original = { - x: downsampledOriginal.x, - y: downsampledOriginal.y, - }; - result.originalY = downsampledOriginal.y; + const originalSegments = segments.map((seg) => + downsampleData(seg.x, seg.y, rate), + ); + result.originalSegments = originalSegments; } return result; @@ -341,9 +412,14 @@ function createPlot() { const traces = []; data.forEach((series, index) => { + console.log( + `[LinePlot] createPlot - processing series ${series.name}: ${series.segments?.length || 0} segments, nanX=${series.nanX?.length || 0}, infX=${series.infX?.length || 0}, negInfX=${series.negInfX?.length || 0}`, + ); + const baseColor = plotlyColors[index % plotlyColors.length]; - if (series.original) { + // Render original (smoothed) segments if enabled + if (series.originalSegments && series.originalSegments.length > 0) { // Convert to dimmer color (not opacity) - blend with gray let dimmerColor = baseColor; if (baseColor.startsWith("#")) { @@ -358,68 +434,155 @@ function createPlot() { dimmerColor = `rgb(${dimR}, ${dimG}, ${dimB})`; } - traces.push({ - type: "scattergl", - mode: "lines", - name: `${series.name} (original)`, - x: series.original.x, - y: series.original.y, - line: { - width: config.lineWidth * 0.5, - color: dimmerColor, - }, - showlegend: false, - legendgroup: series.name, - hoverinfo: "skip", + // Add each original segment as a separate trace + series.originalSegments.forEach((seg, segIdx) => { + traces.push({ + type: "scattergl", + mode: "lines", + name: `${series.name} (original)`, + x: seg.x, + y: seg.y, + line: { + width: config.lineWidth * 0.5, + color: dimmerColor, + }, + showlegend: segIdx === 0, // Only first segment shows in legend + legendgroup: series.name + "_original", + hoverinfo: "skip", + }); }); } - const trace = { - type: "scattergl", - mode: config.showMarkers ? "lines+markers" : "lines", - name: series.name, - x: series.x, - y: series.y, - line: { - width: config.lineWidth, - color: baseColor, - }, - marker: { - size: 3, - color: baseColor, - }, - legendgroup: series.name, - }; + // Render main line segments + if (series.segments && series.segments.length > 0) { + series.segments.forEach((seg, segIdx) => { + const trace = { + type: "scattergl", + mode: config.showMarkers ? "lines+markers" : "lines", + name: series.name, + x: seg.x, + y: seg.y, + line: { + width: config.lineWidth, + color: baseColor, + }, + marker: { + size: 3, + color: baseColor, + }, + showlegend: segIdx === 0, // Only first segment shows in legend + legendgroup: series.name, + }; - // Custom hover template for relative_walltime - if (props.xaxis === "relative_walltime") { - trace.customdata = series.x.map((xVal, idx) => ({ - duration: formatDuration(xVal), - original: series.originalY ? series.originalY[idx] : null, - })); - if (series.originalY) { - trace.hovertemplate = - "%{fullData.name}
" + - "Time: %{customdata.duration}
" + - "Value: %{y:.4f} (%{customdata.original:.4f})"; - } else { - trace.hovertemplate = - "%{fullData.name}
" + - "Time: %{customdata.duration}
" + - "Value: %{y:.4f}"; - } - } else if (series.originalY) { - trace.customdata = series.originalY.map((origY, idx) => ({ - original: origY, - smoothed: series.y[idx], - })); - trace.hovertemplate = - "%{fullData.name}: %{y:.4f} (%{customdata.original:.4f})"; - } else { - trace.hovertemplate = "%{fullData.name}: %{y:.4f}"; + // Custom hover template for relative_walltime + if (props.xaxis === "relative_walltime") { + trace.customdata = seg.x.map((xVal, idx) => ({ + duration: formatDuration(xVal), + })); + trace.hovertemplate = + "%{fullData.name}
" + + "Time: %{customdata.duration}
" + + "Value: %{y:.4f}"; + } else { + trace.hovertemplate = + "%{fullData.name}: %{y:.4f}"; + } + + traces.push(trace); + }); } - traces.push(trace); + // Add special marker traces for NaN/inf values + // These render as dots at top/bottom without affecting axis range + const hasSpecialValues = + series.nanX.length > 0 || + series.infX.length > 0 || + series.negInfX.length > 0; + + console.log( + `[LinePlot] ${series.name} special values: NaN=${series.nanX.length}, +inf=${series.infX.length}, -inf=${series.negInfX.length}`, + ); + + if (hasSpecialValues) { + // NaN markers (circle at top of chart) + if (series.nanX.length > 0) { + console.log( + `[LinePlot] Adding NaN markers for ${series.name}:`, + series.nanX, + ); + traces.push({ + type: "scattergl", + mode: "markers", + name: `${series.name} (NaN)`, + x: series.nanX, + y: series.nanX.map(() => 1), // Will be positioned at top via yaxis2 + marker: { + symbol: "circle", + size: 8, + color: baseColor, + line: { color: "white", width: 1.5 }, + }, + yaxis: "y2", // Use secondary y-axis fixed at [0, 1] + showlegend: false, + legendgroup: series.name, + hoverinfo: "skip", // CRITICAL: Don't show in unified hover tooltip + // Individual hover still works when directly hovering the marker + text: series.nanX.map(() => `${series.name}: NaN`), + }); + } + + // +Infinity markers (triangle-up at top of chart) + if (series.infX.length > 0) { + console.log( + `[LinePlot] Adding +inf markers for ${series.name}:`, + series.infX, + ); + traces.push({ + type: "scattergl", + mode: "markers", + name: `${series.name} (+inf)`, + x: series.infX, + y: series.infX.map(() => 0.95), // Slightly below top + marker: { + symbol: "triangle-up", + size: 10, + color: baseColor, + line: { color: "white", width: 1.5 }, + }, + yaxis: "y2", + showlegend: false, + legendgroup: series.name, + hoverinfo: "skip", // CRITICAL: Don't show in unified hover tooltip + text: series.infX.map(() => `${series.name}: +inf`), + }); + } + + // -Infinity markers (triangle-down at bottom of chart) + if (series.negInfX.length > 0) { + console.log( + `[LinePlot] Adding -inf markers for ${series.name}:`, + series.negInfX, + ); + traces.push({ + type: "scattergl", + mode: "markers", + name: `${series.name} (-inf)`, + x: series.negInfX, + y: series.negInfX.map(() => 0.05), // Slightly above bottom + marker: { + symbol: "triangle-down", + size: 10, + color: baseColor, + line: { color: "white", width: 1.5 }, + }, + yaxis: "y2", + showlegend: false, + legendgroup: series.name, + hoverinfo: "skip", // CRITICAL: Don't show in unified hover tooltip + text: series.negInfX.map(() => `${series.name}: -inf`), + }); + } + } }); const xAxisConfig = config.xRange.auto @@ -457,6 +620,16 @@ function createPlot() { showline: false, ...yAxisConfig, }, + // Secondary y-axis for NaN/inf markers (fixed [0,1] range, overlaid) + yaxis2: { + overlaying: "y", + side: "right", + showgrid: false, + showticklabels: false, + showline: false, + range: [0, 1], + fixedrange: true, + }, dragmode: "zoom", height: props.height, paper_bgcolor: "rgba(0,0,0,0)", diff --git a/src/kohaku-board-ui/src/components/layout/TheHeader.vue b/src/kohaku-board-ui/src/components/layout/TheHeader.vue index 6573b09..a5e530a 100644 --- a/src/kohaku-board-ui/src/components/layout/TheHeader.vue +++ b/src/kohaku-board-ui/src/components/layout/TheHeader.vue @@ -16,14 +16,21 @@ KohakuBoard -
- + + Projects - Projects - -
+ + {{ currentProject }} + + + {{ currentRun }} + +
@@ -99,8 +106,13 @@ import { ElMessage } from "element-plus"; const { animationsEnabled, toggleAnimations } = useAnimationPreference(); const authStore = useAuthStore(); const router = useRouter(); +const route = useRoute(); const systemInfo = ref(null); +// Extract current project and run from route for breadcrumb +const currentProject = computed(() => route.params.project || null); +const currentRun = computed(() => route.params.id || null); + const props = defineProps({ darkMode: { type: Boolean, diff --git a/src/kohaku-board-ui/src/composables/useHoverSync.js b/src/kohaku-board-ui/src/composables/useHoverSync.js index f885897..7d14181 100644 --- a/src/kohaku-board-ui/src/composables/useHoverSync.js +++ b/src/kohaku-board-ui/src/composables/useHoverSync.js @@ -296,7 +296,7 @@ export function useHoverSync() { const xRange = xaxis.range || [0, 1]; const yRange = yaxis.range || [0, 1]; - // Create vertical spike line shape at xValue + // Create vertical spike line (x-axis spike only) const spikeShape = { type: "line", x0: xValue, @@ -327,7 +327,7 @@ export function useHoverSync() { }); console.log( - `${LOG_PREFIX} Drew spike line at x=${xValue} for chart:`, + `${LOG_PREFIX} Drew x-axis spike line at x=${xValue} for chart:`, chartInfo.id, ); } else { diff --git a/src/kohaku-board-ui/src/pages/projects/[project]/[id].vue b/src/kohaku-board-ui/src/pages/projects/[project]/[id].vue index 8d0a1bc..532cb6b 100644 --- a/src/kohaku-board-ui/src/pages/projects/[project]/[id].vue +++ b/src/kohaku-board-ui/src/pages/projects/[project]/[id].vue @@ -18,6 +18,7 @@ const isUpdating = ref(false); const showAddTabDialog = ref(false); const newTabName = ref(""); const showGlobalSettings = ref(false); +const isInitializing = ref(true); // Prevent watch triggers during init // Pagination for WebGL context limit const currentPage = ref(0); @@ -51,11 +52,19 @@ const storageKey = computed( () => `experiment-layout-${route.params.project}-${route.params.id}`, ); -onMounted(async () => { +// Extracted initialization logic for reuse +async function initializeExperiment() { try { + isInitializing.value = true; + const projectName = route.params.project; const runId = route.params.id; + console.log(`[Init] Loading experiment: ${projectName}/${runId}`); + + // Clear previous data + metricDataCache.value = {}; + // Fetch summary using runs API const summaryResponse = await fetch( `/api/projects/${projectName}/runs/${runId}/summary`, @@ -298,9 +307,18 @@ onMounted(async () => { // Determine default x-axis (prefer global_step if it's used, otherwise step) await determineDefaultXAxis(); + + // Initialization complete - allow watch to fire on user tab changes + isInitializing.value = false; + console.log("[Init] Initialization complete"); } catch (error) { console.error("Failed to load experiment:", error); + isInitializing.value = false; } +} + +onMounted(() => { + initializeExperiment(); }); async function determineDefaultXAxis() { @@ -313,9 +331,10 @@ async function determineDefaultXAxis() { ); const result = await response.json(); + // New columnar format: {values: []} // Check if any global_step value is non-zero - const hasNonZeroGlobalStep = result.data.some( - (item) => item.value !== 0 && item.value !== null, + const hasNonZeroGlobalStep = result.values.some( + (value) => value !== 0 && value !== null, ); // Update all cards to use global_step if it's being used @@ -338,8 +357,14 @@ async function determineDefaultXAxis() { async function fetchMetricsForTab() { try { + console.log( + `[fetchMetricsForTab] Starting fetch for tab: ${activeTab.value}`, + ); const tab = tabs.value.find((t) => t.name === activeTab.value); - if (!tab) return; + if (!tab) { + console.warn(`[fetchMetricsForTab] Tab not found: ${activeTab.value}`); + return; + } // Metrics that are computed on frontend (don't fetch from API) const computedMetrics = new Set(["walltime", "relative_walltime"]); @@ -368,35 +393,65 @@ async function fetchMetricsForTab() { for (const metric of neededMetrics) { if (!metricDataCache.value[metric]) { try { + console.log(`[fetchMetricsForTab] Fetching metric: ${metric}`); // Don't URL-encode - FastAPI :path parameter handles it const response = await fetch( `/api/projects/${projectName}/runs/${runId}/scalars/${metric}`, ); if (!response.ok) { - console.warn(`Failed to fetch ${metric}: ${response.status}`); + console.warn( + `[fetchMetricsForTab] Failed to fetch ${metric}: ${response.status}`, + ); // Set empty array so card can still render (just shows "no data") metricDataCache.value[metric] = []; continue; } const result = await response.json(); + console.log( + `[fetchMetricsForTab] Fetched ${metric}: ${result.values?.length || 0} values`, + ); - // Convert step-value pairs to sparse array - const sparseArray = new Array( - result.data[result.data.length - 1]?.step + 1 || 0, - ).fill(null); - for (const item of result.data) { - sparseArray[item.step] = item.value; + // New columnar format: {steps: [], global_steps: [], timestamps: [], values: []} + // Convert to sparse array (indexed by step number) + const maxStep = Math.max(...result.steps); + const sparseArray = new Array(maxStep + 1).fill(null); + + for (let i = 0; i < result.steps.length; i++) { + const step = result.steps[i]; + let value = result.values[i]; + + // Convert special string markers back to numeric NaN/inf + // Backend sends these as strings because JSON doesn't support Infinity + if (value === "NaN") { + value = NaN; + } else if (value === "Infinity") { + value = Infinity; + } else if (value === "-Infinity") { + value = -Infinity; + } + + sparseArray[step] = value; } metricDataCache.value[metric] = sparseArray; + console.log( + `[fetchMetricsForTab] Stored ${metric} in cache: ${sparseArray.length} slots, ${sparseArray.filter((v) => v !== null).length} non-null values`, + ); - // Also store timestamp data if available - if (result.data[0]?.timestamp) { + // Also store timestamp data if available (as Unix seconds) + if (result.timestamps && result.timestamps.length > 0) { const timestampArray = new Array(sparseArray.length).fill(null); - for (const item of result.data) { - timestampArray[item.step] = item.timestamp; + for (let i = 0; i < result.steps.length; i++) { + const step = result.steps[i]; + const unixSeconds = result.timestamps[i]; + if (unixSeconds !== null) { + // Convert Unix seconds to ISO string for consistency + timestampArray[step] = new Date( + unixSeconds * 1000, + ).toISOString(); + } } metricDataCache.value[`${metric}_timestamp`] = timestampArray; } @@ -435,8 +490,13 @@ async function fetchMetricsForTab() { metricDataCache.value.walltime = walltime; metricDataCache.value.relative_walltime = relativeWalltime; } + + console.log( + `[fetchMetricsForTab] Completed. Cache keys:`, + Object.keys(metricDataCache.value), + ); } catch (error) { - console.error("Error in fetchMetricsForTab:", error); + console.error("[fetchMetricsForTab] Error:", error); // Don't throw - allow page to render with partial data } } @@ -446,6 +506,7 @@ const sparseData = computed(() => { for (const [key, values] of Object.entries(metricDataCache.value)) { data[key] = values; } + console.log(`[sparseData] Computed with ${Object.keys(data).length} metrics`); return data; }); @@ -509,11 +570,30 @@ const visibleCards = computed(() => { }); watch(activeTab, () => { + // Don't fetch during initialization (prevents race condition) + if (isInitializing.value) { + console.log("[Watch] activeTab changed during init, skipping fetch"); + return; + } + + console.log("[Watch] activeTab changed to:", activeTab.value); currentPage.value = 0; fetchMetricsForTab(); saveLayout(); }); +// Watch route params to reinitialize when navigating between runs +watch( + () => [route.params.project, route.params.id], + ([newProject, newId], [oldProject, oldId]) => { + if (oldId && (newId !== oldId || newProject !== oldProject)) { + console.log("[Watch] Route changed - reinitializing"); + currentPage.value = 0; + initializeExperiment(); + } + }, +); + function saveLayout() { const layout = { tabs: tabs.value, @@ -833,54 +913,19 @@ function onDragEnd(evt) {