From 1da325b2544ca5dba34e304ad242638f0472516c Mon Sep 17 00:00:00 2001 From: Navneeth Dhamotharan Date: Sun, 31 May 2026 18:21:40 -0700 Subject: [PATCH] feat: show run failure, cancel, and truncation reasons in UI Consume bench-api status_reason, status_detail, and episode terminal_info across recent runs, live monitor, dev dashboard, and test bench results. --- cspell.json | 1 + src/components/BenchResultPanel.tsx | 25 ++++-- src/components/DeveloperDashboard.tsx | 21 +++++- src/components/ModelSelector.tsx | 39 +++++----- src/components/RecentRuns.tsx | 52 ++++++++----- src/components/RunStatusReason.tsx | 105 ++++++++++++++++++++++++++ src/lib/api.ts | 48 ++++++++++++ src/lib/runStatus.ts | 96 +++++++++++++++++++++++ 8 files changed, 339 insertions(+), 48 deletions(-) create mode 100644 src/components/RunStatusReason.tsx create mode 100644 src/lib/runStatus.ts diff --git a/cspell.json b/cspell.json index 0b8d003..30cb388 100644 --- a/cspell.json +++ b/cspell.json @@ -29,6 +29,7 @@ "pydantic", "relitigate", "sandboxed", + "scoreable", "sigils", "Streamable", "Syms", diff --git a/src/components/BenchResultPanel.tsx b/src/components/BenchResultPanel.tsx index d2b5f93..4b28a46 100644 --- a/src/components/BenchResultPanel.tsx +++ b/src/components/BenchResultPanel.tsx @@ -2,6 +2,7 @@ import { BenchJob, Episode, SUPPORTED_MODELS } from "@/lib/api"; import ExpandableErrorText from "@/components/ExpandableErrorText"; +import { episodeStatusDetail } from "@/lib/runStatus"; const MODEL_LABELS: Record = Object.fromEntries( SUPPORTED_MODELS.map(({ id, label }) => [id, label]) @@ -13,6 +14,8 @@ const STATUS_STYLE: Record = { running: "text-leaf-deep", queued: "text-ink-3", timeout: "text-bad", + truncated: "text-warn", + cancelled: "text-warn", }; function Metric({ label, value }: { label: string; value: string | number | null }) { @@ -44,13 +47,23 @@ export function TestBenchResult({ episode }: TestBenchResultProps) { - {episode.status === "failed" && episode.terminal_info?.error != null && ( + {(episode.status === "failed" || + episode.status === "timeout" || + episode.status === "truncated" || + episode.status === "cancelled") && (
-

error

- + {episodeStatusDetail(episode) && ( +

{episodeStatusDetail(episode)}

+ )} + {episode.terminal_info?.error != null && ( + <> +

error

+ + + )}
)} diff --git a/src/components/DeveloperDashboard.tsx b/src/components/DeveloperDashboard.tsx index 70ab9da..74ead21 100644 --- a/src/components/DeveloperDashboard.tsx +++ b/src/components/DeveloperDashboard.tsx @@ -24,6 +24,7 @@ import { testBench, SUPPORTED_MODELS, } from "@/lib/api"; +import { reasonLabel } from "@/lib/runStatus"; import BindingVowChip from "@/components/BindingVowChip"; import { Btn } from "@/components/ds/Btn"; import { FullBenchResult, TestBenchResult } from "@/components/BenchResultPanel"; @@ -610,9 +611,16 @@ function EnvironmentCard({ {run.config.agent_config.model ?? "unknown"} - - {run.status} - +
+ + {run.status} + + {run.status_reason && ( + + {reasonLabel(run.status_reason)} + + )} +
))} @@ -777,9 +785,14 @@ function DeveloperRunsPanel({ Domain → )} - + {run.status} + {run.status_reason && ( + + {reasonLabel(run.status_reason)} + + )} ))} diff --git a/src/components/ModelSelector.tsx b/src/components/ModelSelector.tsx index a249990..4c927e2 100644 --- a/src/components/ModelSelector.tsx +++ b/src/components/ModelSelector.tsx @@ -16,6 +16,8 @@ import { } from "@/lib/api"; import { BenchAccessGate } from "@/components/BenchAccessGate"; import { Btn } from "@/components/ds/Btn"; +import { RunStatusReason } from "@/components/RunStatusReason"; +import { summarizeEpisodes } from "@/lib/runStatus"; import { useActiveTeam } from "@/hooks/useActiveTeam"; import { useBenchAuth } from "@/hooks/useBenchAuth"; import { getActiveTeamId } from "@/lib/benchAuth"; @@ -33,31 +35,12 @@ interface Props { type RunView = { run: Run; episodes: Episode[] }; -interface EpSummary { - total: number; - completed: number; - running: number; - pending: number; - failed: number; - timeout: number; -} - -function summarizeEpisodes(episodes: Episode[]): EpSummary { - return { - total: episodes.length, - completed: episodes.filter((e) => e.status === "completed").length, - running: episodes.filter((e) => e.status === "running").length, - pending: episodes.filter((e) => e.status === "pending").length, - failed: episodes.filter((e) => e.status === "failed").length, - timeout: episodes.filter((e) => e.status === "timeout").length, - }; -} - const RUN_TONE: Record = { completed: "text-ok", failed: "text-bad", timeout: "text-bad", cancelled: "text-warn", + truncated: "text-warn", running: "text-leaf-deep", pending: "text-ink-3", }; @@ -125,7 +108,7 @@ export default function ModelSelector({ domain, envId }: Props) { listRunEpisodes(runId), ]); return [runId, { run, episodes }] as const; - }) + }), ); if (!alive) return; const next: Record = {}; @@ -379,8 +362,22 @@ export default function ModelSelector({ domain, envId }: Props) { {s.timeout} )} + {s.truncated > 0 && ( + + truncated + {s.truncated} + + )} + {s.cancelled > 0 && ( + + cancelled + {s.cancelled} + + )} + + {run.status === "completed" && Object.keys(run.scores ?? {}).length > 0 && (
diff --git a/src/components/RecentRuns.tsx b/src/components/RecentRuns.tsx index 154211e..337a318 100644 --- a/src/components/RecentRuns.tsx +++ b/src/components/RecentRuns.tsx @@ -16,6 +16,7 @@ import { useBenchAuth } from "@/hooks/useBenchAuth"; import { useActiveTeam } from "@/hooks/useActiveTeam"; import { ScopePill } from "@/components/ScopePill"; import ExpandableErrorText from "@/components/ExpandableErrorText"; +import { EpisodeTerminalReason, RunStatusReason } from "@/components/RunStatusReason"; import { RunVisibilityButton } from "@/components/RunVisibilityButton"; import { benchAuthDisabled } from "@/lib/env"; @@ -42,6 +43,7 @@ const RUN_TONE: Record = { failed: "text-bad", timeout: "text-bad", cancelled: "text-warn", + truncated: "text-warn", running: "text-leaf-deep", pending: "text-ink-3", }; @@ -51,6 +53,7 @@ const EP_DOT: Record = { failed: "bg-bad", timeout: "bg-bad", cancelled: "bg-warn", + truncated: "bg-warn", running: "bg-leaf-deep animate-pulse", pending: "bg-ink-3", }; @@ -285,7 +288,10 @@ export default function RecentRuns({ const episodes = episodesByRunId[run.id]; const episodesLoading = loadingEpisodes.has(run.id); const completedEps = episodes?.filter((e) => e.status === "completed") ?? []; - const failedEps = episodes?.filter((e) => e.status === "failed") ?? []; + const failedEps = + episodes?.filter((e) => + ["failed", "timeout", "cancelled", "truncated"].includes(e.status), + ) ?? []; const totalReward = completedEps.reduce((s, e) => s + e.total_reward, 0); const avgReward = completedEps.length > 0 ? totalReward / completedEps.length : 0; @@ -357,6 +363,10 @@ export default function RecentRuns({
+ {(run.status === "failed" || run.status === "cancelled") && ( + + )} + {run.status === "completed" && Object.keys(run.scores ?? {}).length > 0 && (
@@ -415,13 +425,16 @@ export default function RecentRuns({ key={ep.id} className="flex items-center justify-between text-xs bg-paper-2 border border-line rounded-[2px] px-3 py-2" > -
+
- - {ep.id.slice(0, 8)} - +
+ + {ep.id.slice(0, 8)} + + +
@@ -460,17 +473,22 @@ export default function RecentRuns({ {!episodesLoading && episodes && failedEps.length > 0 && - Boolean(failedEps[0].terminal_info?.error) && ( -
-
- - error details - - -
+ failedEps.some((ep) => ep.terminal_info?.error) && ( +
+ {failedEps + .filter((ep) => ep.terminal_info?.error) + .slice(0, 3) + .map((ep) => ( +
+ + {ep.status} · {ep.id.slice(0, 8)} + + +
+ ))}
)}
diff --git a/src/components/RunStatusReason.tsx b/src/components/RunStatusReason.tsx new file mode 100644 index 0000000..d6a976e --- /dev/null +++ b/src/components/RunStatusReason.tsx @@ -0,0 +1,105 @@ +"use client"; + +import type { Episode, Run } from "@/lib/api"; +import { + episodeStatusDetail, + formatEpisodeOutcomes, + formatReasonList, + reasonLabel, + runNeedsStatusDetail, +} from "@/lib/runStatus"; + +interface Props { + run: Run; + className?: string; +} + +/** Run-level terminal explanation from bench-api `status_reason` / `status_detail`. */ +export function RunStatusReason({ run, className = "" }: Props) { + if (!runNeedsStatusDetail(run.status) && !run.status_reason) return null; + + const detail = run.status_detail; + const outcomes = + formatEpisodeOutcomes(detail) ?? + (run.completed_count != null || run.truncated_count != null + ? [ + run.completed_count ? `completed ${run.completed_count}` : null, + run.truncated_count ? `truncated ${run.truncated_count}` : null, + run.failed_count ? `failed ${run.failed_count}` : null, + run.cancelled_count ? `cancelled ${run.cancelled_count}` : null, + ] + .filter(Boolean) + .join(", ") + : null); + + const failureReasons = + formatReasonList(detail, "failure_reasons") ?? + (run.failure_reasons?.length + ? run.failure_reasons.map((r) => reasonLabel(r)).join(", ") + : null); + const truncationReasons = + formatReasonList(detail, "truncation_reasons") ?? + (run.truncation_reasons?.length + ? run.truncation_reasons.map((r) => reasonLabel(r)).join(", ") + : null); + + if (!run.status_reason && !outcomes && !failureReasons && !truncationReasons) { + return null; + } + + return ( +
+ {run.status_reason && ( +

+ + reason + + + {reasonLabel(run.status_reason)} + +

+ )} + {outcomes && ( +

+ + episodes + + {outcomes} +

+ )} + {truncationReasons && ( +

+ + truncated + + {truncationReasons} +

+ )} + {failureReasons && ( +

+ + failures + + {failureReasons} +

+ )} + {detail?.detail && ( +

{String(detail.detail)}

+ )} +
+ ); +} + +interface EpisodeReasonProps { + episode: Episode; + className?: string; +} + +export function EpisodeTerminalReason({ episode, className = "" }: EpisodeReasonProps) { + if (episode.status === "completed" || episode.status === "pending") return null; + const text = episodeStatusDetail(episode); + if (!text) return null; + return ( +

{text}

+ ); +} diff --git a/src/lib/api.ts b/src/lib/api.ts index 2a22120..2bf0174 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -25,10 +25,39 @@ export type EpStatus = | "pending" | "running" | "completed" + | "truncated" | "failed" | "timeout" | "cancelled"; +export interface RunStatusDetail { + detail?: string; + episode_outcomes?: Partial< + Record<"completed" | "failed" | "truncated" | "cancelled" | "timeout", number> + >; + failure_reasons?: string[]; + truncation_reasons?: string[]; + cancellation_reasons?: string[]; + primary_episode_reason?: string; + num_episodes?: number; + scoreable?: number; + required_ratio?: number; +} + +export interface RunStatusBatchItem { + id: string; + status: RunStatus; + scores: Record; + completed_at?: string | null; + status_reason?: string | null; + status_detail?: RunStatusDetail; + truncated_count?: number; + failed_count?: number; + cancelled_count?: number; + failure_reasons?: string[]; + truncation_reasons?: string[]; +} + export function isActiveRunStatus(status: RunStatus | string): boolean { return status === "pending" || status === "running"; } @@ -298,6 +327,16 @@ export interface Run { team_id?: string | null; env_id?: string | null; visibility?: RunVisibility | null; + status_reason?: string | null; + status_detail?: RunStatusDetail; + completed_count?: number; + failed_count?: number; + truncated_count?: number; + cancelled_count?: number; + timeout_count?: number; + failure_reasons?: string[]; + truncation_reasons?: string[]; + cancellation_reasons?: string[]; } export interface Episode { @@ -481,6 +520,15 @@ export async function getRun(runId: string): Promise { return getJson(`/v1/runs/${encodeURIComponent(runId)}`); } +export async function batchRunStatus(runIds: string[]): Promise> { + if (runIds.length === 0) return {}; + const ids = runIds.map(encodeURIComponent).join(","); + const body = await getJson<{ runs: Record }>( + `/v1/runs/status?ids=${ids}`, + ); + return body.runs ?? {}; +} + export async function listRunEpisodes(runId: string): Promise { return getJson(`/v1/runs/${encodeURIComponent(runId)}/episodes`); } diff --git a/src/lib/runStatus.ts b/src/lib/runStatus.ts new file mode 100644 index 0000000..bdac522 --- /dev/null +++ b/src/lib/runStatus.ts @@ -0,0 +1,96 @@ +import type { Episode, Run, RunStatusDetail } from "@/lib/api"; + +/** Stable codes from bench-api `status_reason` / episode `terminal_info.reason`. */ +export const REASON_LABELS: Record = { + user_cancelled: "Cancelled by user", + task_cancelled: "Run task cancelled", + domain_missing: "Domain no longer exists", + no_episodes: "No episodes recorded", + insufficient_scoreable_episodes: "Too few episodes finished successfully", + execute_run_error: "Run execution error", + stale_run_timeout: "Run timed out (stale)", + bench_api_restart: "Interrupted by platform restart", + manual_reap: "Marked failed by operator", + stale_orphan: "Stranded run cleaned up", + unknown_failure: "Run failed (reason unknown)", + run_cancelled: "Cancelled while running", + episode_error: "Episode error", + no_env_url: "No environment URL configured", + step_limit: "Step limit reached", + wall_time_limit: "Wall-clock time limit reached", + token_budget_exceeded: "Token budget exceeded", +}; + +export function reasonLabel(code: string | null | undefined): string { + if (!code) return "Unknown"; + return REASON_LABELS[code] ?? code.replace(/_/g, " "); +} + +export function episodeTerminalReason(episode: Episode): string | null { + const info = episode.terminal_info ?? {}; + const reason = info.reason; + if (typeof reason === "string" && reason) return reason; + if (episode.status === "failed" && info.error) return "episode_error"; + return null; +} + +export function episodeStatusDetail(episode: Episode): string | null { + const info = episode.terminal_info ?? {}; + const reason = episodeTerminalReason(episode); + const parts: string[] = []; + if (reason) parts.push(reasonLabel(reason)); + if (typeof info.error === "string" && info.error) parts.push(info.error); + if (reason === "step_limit" && info.max_steps != null) { + parts.push(`max_steps=${String(info.max_steps)}`); + } + if (info.detail != null && info.detail !== "") { + parts.push(String(info.detail)); + } + return parts.length > 0 ? parts.join(" · ") : null; +} + +export function runNeedsStatusDetail(status: Run["status"]): boolean { + return status === "failed" || status === "cancelled"; +} + +export function formatEpisodeOutcomes(detail: RunStatusDetail | undefined): string | null { + const outcomes = detail?.episode_outcomes; + if (!outcomes) return null; + const parts = Object.entries(outcomes) + .filter(([, n]) => typeof n === "number" && n > 0) + .map(([k, n]) => `${k} ${n}`); + return parts.length > 0 ? parts.join(", ") : null; +} + +export function formatReasonList( + detail: RunStatusDetail | undefined, + key: "failure_reasons" | "truncation_reasons" | "cancellation_reasons", +): string | null { + const vals = detail?.[key]; + if (!Array.isArray(vals) || vals.length === 0) return null; + return vals.map((v) => reasonLabel(String(v))).join(", "); +} + +export interface EpisodeCounts { + total: number; + completed: number; + running: number; + pending: number; + failed: number; + timeout: number; + truncated: number; + cancelled: number; +} + +export function summarizeEpisodes(episodes: Episode[]): EpisodeCounts { + return { + total: episodes.length, + completed: episodes.filter((e) => e.status === "completed").length, + running: episodes.filter((e) => e.status === "running").length, + pending: episodes.filter((e) => e.status === "pending").length, + failed: episodes.filter((e) => e.status === "failed").length, + timeout: episodes.filter((e) => e.status === "timeout").length, + truncated: episodes.filter((e) => e.status === "truncated").length, + cancelled: episodes.filter((e) => e.status === "cancelled").length, + }; +}