diff --git a/web/packages/studio/src/components/Breadcrumbs/index.test.tsx b/web/packages/studio/src/components/Breadcrumbs/index.test.tsx new file mode 100644 index 0000000000..0e1e8beeb5 --- /dev/null +++ b/web/packages/studio/src/components/Breadcrumbs/index.test.tsx @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Breadcrumbs } from '@studio/components/Breadcrumbs'; +import { BreadcrumbsProvider } from '@studio/providers/breadcrumbs/BreadcrumbsProvider'; +import { + useBreadcrumbs, + type BreadcrumbsItemProps, +} from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { renderRoute, screen } from '@studio/tests/util/render'; +import type { FC } from 'react'; + +const SetCrumbs: FC<{ items: BreadcrumbsItemProps[] }> = ({ items }) => { + useBreadcrumbs({ items }); + return null; +}; + +const renderCrumbs = (items: BreadcrumbsItemProps[]) => + renderRoute( + + + + , + { history: '/workspaces/default/agents/optimizations/sweep-3' } + ); + +describe('Breadcrumbs', () => { + it('strips the query string by default, so an "up" link does not leak detail state', () => { + renderCrumbs([{ slotLabel: 'Agents', href: '/workspaces/default/agents?tab=optimizations' }]); + + expect(screen.getByRole('link', { name: 'Agents' })).toHaveAttribute( + 'href', + '/workspaces/default/agents' + ); + }); + + it('keeps the query string when the item opts in with preserveQuery', () => { + renderCrumbs([ + { + slotLabel: 'Optimizations', + href: '/workspaces/default/agents/email-analyzer?tab=optimizations', + preserveQuery: true, + }, + ]); + + // Without this the parent route falls back to its default tab — the bug this flag exists for. + expect(screen.getByRole('link', { name: 'Optimizations' })).toHaveAttribute( + 'href', + '/workspaces/default/agents/email-analyzer?tab=optimizations' + ); + }); +}); diff --git a/web/packages/studio/src/components/Breadcrumbs/index.tsx b/web/packages/studio/src/components/Breadcrumbs/index.tsx index 165becff03..1a73e0cb2a 100644 --- a/web/packages/studio/src/components/Breadcrumbs/index.tsx +++ b/web/packages/studio/src/components/Breadcrumbs/index.tsx @@ -20,8 +20,8 @@ export const Breadcrumbs: FC = () => { allItems.push(WORKSPACE_BREADCRUMB_ITEM); } return allItems.concat( - breadcrumbs.map(({ href = '#', slotLabel }) => ({ - children: {slotLabel}, + breadcrumbs.map(({ href = '#', slotLabel, preserveQuery }) => ({ + children: {slotLabel}, })) ); }, [breadcrumbs, workspace]); diff --git a/web/packages/studio/src/components/dataViews/OptimizationJobsDataView/index.tsx b/web/packages/studio/src/components/dataViews/OptimizationJobsDataView/index.tsx new file mode 100644 index 0000000000..a1d058f8ea --- /dev/null +++ b/web/packages/studio/src/components/dataViews/OptimizationJobsDataView/index.tsx @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; +import { StatusBadge, type StatusConfigEntry } from '@nemo/common/src/components/StatusBadge'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { formatDurationMs } from '@nemo/common/src/utils/date'; +import { Badge, Button, Text } from '@nvidia/foundations-react-core'; +import type { + StudyResults, + Trial, +} from '@studio/routes/agents/AgentOptimizationDetailRoute/studyResults'; +import { Ban, CircleCheck, CircleX, RefreshCw } from 'lucide-react'; +import { type ComponentProps, type FC, useMemo } from 'react'; + +const EM_DASH = '—'; + +/** Sort id prefix for the one-column-per-objective metric columns. */ +const METRIC_SORT_PREFIX = 'metric:'; + +/** Sort id prefix for the one-column-per-parameter columns. */ +const PARAM_SORT_PREFIX = 'param:'; + +const formatMetric = (value: number | null): string => { + if (value === null) return EM_DASH; + const fixed = value.toLocaleString(undefined, { maximumFractionDigits: 4 }); + if (value !== 0 && Number(fixed.replace(/,/g, '')) === 0) { + return value.toLocaleString(undefined, { maximumSignificantDigits: 4 }); + } + return fixed; +}; + +/** `COMPLETE` → `Complete`, matching the sentence-case status text in the design. */ +const formatState = (state: string): string => + state ? state.charAt(0) + state.slice(1).toLowerCase() : EM_DASH; + +/** + * Optuna's `TrialState` names, mapped onto the shared badge vocabulary. + */ +const TRIAL_STATUS_CONFIG: Record = { + COMPLETE: { label: 'Complete', color: 'green', icon: CircleCheck }, + FAIL: { label: 'Failed', color: 'red', icon: CircleX }, + PRUNED: { label: 'Pruned', color: 'yellow', icon: Ban }, + RUNNING: { label: 'Running', color: 'blue', icon: RefreshCw }, + WAITING: { label: 'Waiting', color: 'gray', icon: RefreshCw }, +}; + +const formatParamValue = (value: string): string => { + const parsed = Number(value); + if (value.trim() === '' || Number.isNaN(parsed) || Number.isInteger(parsed)) return value; + const rounded = Number(parsed.toFixed(3)); + return rounded === 0 ? value : String(rounded); +}; + +const paramsText = (trial: Trial): string => + trial.params.map((param) => `${param.name} ${formatParamValue(param.value)}`).join(' · '); + +const paramValue = (trial: Trial, name: string): string | undefined => + trial.params.find((param) => param.name === name)?.value; + +/** + * Every parameter the study tuned, in the order the trials list them (which follows the + * `params_*` column order of the source CSV). A trial omits parameters it did not record, + * so the union across trials is what defines the column set. + */ +const collectParamNames = (trials: Trial[]): string[] => { + const names = new Set(); + for (const trial of trials) { + for (const param of trial.params) names.add(param.name); + } + return [...names]; +}; + +/** + * Parameters whose every recorded value parses as a number, so the column can sort numerically + * instead of lexicographically (`10` after `9`, not before it). + */ +const collectNumericParams = (trials: Trial[], names: string[]): Set => + new Set( + names.filter((name) => + trials.every((trial) => { + const value = paramValue(trial, name); + return value === undefined || value === '' || Number.isFinite(Number(value)); + }) + ) + ); + +/** What the search bar matches against: the trial id plus every parameter name and value. */ +const searchableText = (trial: Trial): string => + `trial ${trial.number} ${paramsText(trial)}`.toLowerCase(); + +/** Missing values sort last in both directions, so an empty metric never tops the table. */ +const compareNullable = ( + a: number | string | null, + b: number | string | null, + desc: boolean +): number => { + if (a === null && b === null) return 0; + if (a === null) return 1; + if (b === null) return -1; + const order = + typeof a === 'string' || typeof b === 'string' ? String(a).localeCompare(String(b)) : a - b; + return desc ? -order : order; +}; + +const sortValue = ( + trial: Trial, + sortId: string, + numericParams: ReadonlySet +): number | string | null => { + if (sortId === 'number') return trial.number; + if (sortId === 'duration') return trial.durationSeconds; + if (sortId === 'frontier') return trial.paretoOptimal ? 1 : 0; + if (sortId === 'state') return trial.state || null; + if (sortId.startsWith(METRIC_SORT_PREFIX)) { + const name = sortId.slice(METRIC_SORT_PREFIX.length); + return trial.metrics.find((metric) => metric.name === name)?.value ?? null; + } + if (sortId.startsWith(PARAM_SORT_PREFIX)) { + const name = sortId.slice(PARAM_SORT_PREFIX.length); + const value = paramValue(trial, name); + if (value === undefined || value === '') return null; + return numericParams.has(name) ? Number(value) : value; + } + return null; +}; + +export interface TrialsDataViewProps { + results: StudyResults; +} + +export const TrialsDataView: FC = ({ results }) => { + const { trials, metricNames } = results; + const primaryMetric = metricNames[0]; + + const paramNames = useMemo(() => collectParamNames(trials), [trials]); + const numericParams = useMemo( + () => collectNumericParams(trials, paramNames), + [trials, paramNames] + ); + + const dataViewState = useStudioDataViewState({ + defaultPageSize: 25, + defaultSort: [ + primaryMetric + ? { id: `${METRIC_SORT_PREFIX}${primaryMetric}`, desc: true } + : { id: 'number', desc: false }, + ], + }); + + const { debouncedSearchBar } = dataViewState; + const sorting = dataViewState.sorting.state; + const { pageIndex, pageSize } = dataViewState.pagination.state; + + const processedTrials = useMemo(() => { + const search = debouncedSearchBar.trim().toLowerCase(); + const filtered = search + ? trials.filter((trial) => searchableText(trial).includes(search)) + : trials; + + const [sort] = sorting; + if (!sort) return filtered; + return [...filtered].sort((a, b) => + compareNullable( + sortValue(a, sort.id, numericParams), + sortValue(b, sort.id, numericParams), + !!sort.desc + ) + ); + }, [trials, debouncedSearchBar, sorting, numericParams]); + + const lastPageIndex = Math.max(0, Math.ceil(processedTrials.length / pageSize) - 1); + const safePageIndex = Math.min(pageIndex, lastPageIndex); + const pageTrials = useMemo( + () => processedTrials.slice(safePageIndex * pageSize, safePageIndex * pageSize + pageSize), + [processedTrials, safePageIndex, pageSize] + ); + + const makeColumns: ComponentProps>['makeColumns'] = ({ + accessor, + display, + }) => [ + accessor('number', { + id: 'number', + header: 'Trial', + size: 110, + enableSorting: true, + cell: ({ row }) => Trial {row.original.number}, + }), + ...metricNames.map((name) => + accessor((row: Trial) => row.metrics.find((metric) => metric.name === name)?.value ?? null, { + id: `${METRIC_SORT_PREFIX}${name}`, + header: name, + size: 140, + enableSorting: true, + cell: ({ row }) => ( + + {formatMetric( + row.original.metrics.find((metric) => metric.name === name)?.value ?? null + )} + + ), + }) + ), + ...paramNames.map((name) => + accessor((row: Trial) => paramValue(row, name) ?? '', { + id: `${PARAM_SORT_PREFIX}${name}`, + header: name, + size: 150, + enableSorting: true, + cell: ({ row }) => { + const value = paramValue(row.original, name); + return ( + + {value === undefined || value === '' ? EM_DASH : formatParamValue(value)} + + ); + }, + }) + ), + accessor('durationSeconds', { + id: 'duration', + header: 'Duration', + size: 120, + enableSorting: true, + cell: ({ row }) => ( + + {row.original.durationSeconds === null + ? EM_DASH + : formatDurationMs(row.original.durationSeconds * 1_000)} + + ), + }), + accessor('state', { + id: 'state', + header: 'Status', + size: 140, + enableSorting: true, + cell: ({ row }) => + row.original.state ? ( + + ) : ( + + {EM_DASH} + + ), + }), + accessor('paretoOptimal', { + id: 'frontier', + header: 'Frontier', + size: 130, + enableSorting: true, + cell: ({ row }) => + row.original.paretoOptimal ? ( + + On frontier + + ) : ( + + {EM_DASH} + + ), + }), + display({ + id: 'promote', + header: 'Promote', + size: 130, + cell: () => ( + + ), + }), + ]; + + return ( + + dataViewState={dataViewState} + searchField="params" + makeColumns={makeColumns} + attributes={{ + DataViewSearchBar: { placeholder: 'Search parameters or trial ID...' }, + DataViewRoot: { + data: pageTrials, + totalCount: processedTrials.length, + }, + DataViewTableContent: { + renderEmptyState: () => ( + + ), + }, + }} + /> + ); +}; diff --git a/web/packages/studio/src/constants/routes.ts b/web/packages/studio/src/constants/routes.ts index e2fff03fd1..6fc295f1cf 100644 --- a/web/packages/studio/src/constants/routes.ts +++ b/web/packages/studio/src/constants/routes.ts @@ -35,6 +35,7 @@ export const ROUTE_PARAMS = { agentName: 'agentName', agentDeploymentName: 'agentDeploymentName', agentEvalJobName: 'agentEvalJobName', + optimizeJobName: 'optimizeJobName', jobName: 'jobName', /** Benchmark entity name segment under evaluation/benchmarks/:name */ benchmarkName: 'benchmarkName', @@ -145,6 +146,7 @@ export const ROUTES = { agentDeploymentDetail: `/workspaces/:${P.workspace}/agent-deployments/:${P.agentDeploymentName}`, /** Detail view for a single agent-evaluation job. */ agentEvaluationDetail: `/workspaces/:${P.workspace}/agents/evaluations/:${P.agentEvalJobName}`, + agentOptimizationDetail: `/workspaces/:${P.workspace}/agents/optimizations/:${P.optimizeJobName}`, modelCompare: `/workspaces/:${P.workspace}/playground`, agentMonitor: `/workspaces/:${P.workspace}/agents/monitor`, /** Plugin-owned page; the plugin's internal router owns sub-paths via a `/*` suffix. */ diff --git a/web/packages/studio/src/providers/breadcrumbs/useBreadcrumbs.ts b/web/packages/studio/src/providers/breadcrumbs/useBreadcrumbs.ts index cfb8b1aec5..77090eca70 100644 --- a/web/packages/studio/src/providers/breadcrumbs/useBreadcrumbs.ts +++ b/web/packages/studio/src/providers/breadcrumbs/useBreadcrumbs.ts @@ -9,6 +9,7 @@ export const BreadcrumbsContext = createContext( export type BreadcrumbsItemProps = { href?: string; slotLabel: ReactNode; + preserveQuery?: boolean; }; export type BreadCrumbItemsProps = { diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/optimizations/OptimizeJobsTable.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/optimizations/OptimizeJobsTable.tsx index 04e6ea1991..172dd79904 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/optimizations/OptimizeJobsTable.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/optimizations/OptimizeJobsTable.tsx @@ -12,7 +12,7 @@ import { useAgentsListOptimizeJobs } from '@nemo/sdk/generated/agents/agents'; import type { OptimizeJob, OptimizeJobsListFilter } from '@nemo/sdk/generated/agents/schema'; import { Banner, Text } from '@nvidia/foundations-react-core'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getWorkspaceJobDetailRoute } from '@studio/routes/utils'; +import { getAgentOptimizationDetailRoute } from '@studio/routes/utils'; import { keepPreviousData } from '@tanstack/react-query'; import { type ComponentProps, type FC, useCallback } from 'react'; import { useNavigate } from 'react-router'; @@ -133,7 +133,7 @@ export const OptimizeJobsTable: FC = ({ agentName }) => dataViewState={dataViewState} searchField="name" makeColumns={makeColumns} - onRowClick={(row) => navigate(getWorkspaceJobDetailRoute(workspace, row.name))} + onRowClick={(row) => navigate(getAgentOptimizationDetailRoute(workspace, row.name))} attributes={{ DataViewSearchBar: { placeholder: 'Search by name...' }, DataViewRoot: { diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx index 50b1a6229a..38106b37e8 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx @@ -91,7 +91,11 @@ export const AgentEvaluationDetailRoute: FC = () => { setBreadcrumbs([ { slotLabel: 'Agents', href: getAgentsListRoute(workspace) }, agentName - ? { slotLabel: 'Evaluations', href: getAgentEvaluationsTabRoute(workspace, agentName) } + ? { + slotLabel: 'Evaluations', + href: getAgentEvaluationsTabRoute(workspace, agentName), + preserveQuery: true, + } : { slotLabel: 'Evaluations' }, { slotLabel: jobName }, ]); diff --git a/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/StudyStatTiles.tsx b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/StudyStatTiles.tsx new file mode 100644 index 0000000000..6e835c5527 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/StudyStatTiles.tsx @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StatTile } from '@nemo/common/src/components/StatTile'; +import { formatDurationMs } from '@nemo/common/src/utils/date'; +import { Grid } from '@nvidia/foundations-react-core'; +import type { StudyResults } from '@studio/routes/agents/AgentOptimizationDetailRoute/studyResults'; +import type { FC } from 'react'; + +const EM_DASH = '—'; + +/** Optuna's `TrialState.COMPLETE`, as written to the `state` column. */ +const COMPLETE = 'COMPLETE'; + +const formatScore = (value: number | null): string => + value === null ? EM_DASH : value.toLocaleString(undefined, { maximumFractionDigits: 4 }); + +export interface StudyStatTilesProps { + results: StudyResults; +} + +export const StudyStatTiles: FC = ({ results }) => { + const { summary, trials, metricNames } = results; + + const totalTrials = summary?.nTrials ?? trials.length; + const frontierCount = trials.filter((trial) => trial.paretoOptimal).length; + + const primaryMetric = metricNames[0]; + const bestValue = summary?.bestValues[0] ?? null; + + const timedTrials = trials.filter( + (trial) => trial.state === COMPLETE && trial.durationSeconds !== null + ); + const averageDurationMs = timedTrials.length + ? (timedTrials.reduce((total, trial) => total + (trial.durationSeconds ?? 0), 0) / + timedTrials.length) * + 1_000 + : null; + + return ( + + + + + + + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/index.tsx b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/index.tsx new file mode 100644 index 0000000000..2934bdd9dc --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/index.tsx @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AccessibleTitle } from '@nemo/common/src/components/AccessibleTitle'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { ErrorPanel } from '@nemo/common/src/components/ErrorPanel'; +import { LogViewer } from '@nemo/common/src/components/LogViewer'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; +import { JOB_POLLING_INTERVAL_MS } from '@nemo/common/src/constants'; +import { useJobLogs } from '@nemo/common/src/hooks/useJobLogs'; +import { useAgentsGetOptimizeJob } from '@nemo/sdk/generated/agents/agents'; +import type { PlatformJobStatus } from '@nemo/sdk/generated/platform/schema'; +import { Flex, PageHeader, Panel, Spinner, Stack, Text } from '@nvidia/foundations-react-core'; +import { TrialsDataView } from '@studio/components/dataViews/OptimizationJobsDataView'; +import { ROUTE_PARAMS } from '@studio/constants/routes'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { fetchStudyResults } from '@studio/routes/agents/AgentOptimizationDetailRoute/studyResults'; +import { StudyStatTiles } from '@studio/routes/agents/AgentOptimizationDetailRoute/StudyStatTiles'; +import { getAgentOptimizationsTabRoute, getAgentsListRoute } from '@studio/routes/utils'; +import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; +import { useQuery } from '@tanstack/react-query'; +import { ScrollText } from 'lucide-react'; +import { type FC, useEffect } from 'react'; + +/** Statuses that will not change again, so polling can stop. */ +const TERMINAL_STATUSES = new Set(['completed', 'error', 'cancelled']); +const FAILED_STATUSES = new Set(['error', 'cancelled']); + +export const AgentOptimizationDetailRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + const { optimizeJobName: jobName } = useRequiredPathParams([ROUTE_PARAMS.optimizeJobName]); + + const { + data: job, + isLoading: isLoadingJob, + error: jobError, + } = useAgentsGetOptimizeJob(workspace, jobName, { + query: { + enabled: !!workspace && !!jobName, + refetchInterval: (query) => + query.state.data?.status && TERMINAL_STATUSES.has(query.state.data.status) + ? false + : JOB_POLLING_INTERVAL_MS, + }, + }); + + const status = job?.status ?? undefined; + const isTerminal = status ? TERMINAL_STATUSES.has(status) : false; + const hasFailed = status ? FAILED_STATUSES.has(status) : false; + const agentName = job?.spec?.agent?.split('/').pop() ?? undefined; + + const { setBreadcrumbs } = useBreadcrumbs(); + useEffect(() => { + setBreadcrumbs([ + { slotLabel: 'Agents', href: getAgentsListRoute(workspace) }, + agentName + ? { + slotLabel: 'Optimizations', + href: getAgentOptimizationsTabRoute(workspace, agentName), + preserveQuery: true, + } + : { slotLabel: 'Optimizations' }, + { slotLabel: jobName }, + ]); + return () => setBreadcrumbs([]); + }, [setBreadcrumbs, workspace, agentName, jobName]); + + const { + data: results, + isLoading: isLoadingResults, + isError: isResultsError, + error: resultsError, + } = useQuery({ + queryKey: ['optimize-study-results', workspace, jobName] as const, + queryFn: ({ signal }) => fetchStudyResults(workspace, jobName, signal), + enabled: !!workspace && !!jobName && isTerminal && !hasFailed, + refetchInterval: (query) => (query.state.data === undefined ? JOB_POLLING_INTERVAL_MS : false), + }); + + const { + data: logs, + isLoading: isLoadingLogs, + loadProgress, + } = useJobLogs({ + workspace, + name: jobName, + jobStatus: status, + enabled: hasFailed, + }); + + if (isLoadingJob && !job) { + return ( + + + + ); + } + + if (!job && jobError && jobError.response?.status !== 404) { + return ( + + + + ); + } + + if (!job) { + return ( + + + + ); + } + + const errorMessage = + typeof job.error_details?.message === 'string' ? job.error_details.message : undefined; + + return ( + + + + {jobName} + + + {job.updated_at && isTerminal && ( + + + + )} + + + } + /> + + {hasFailed ? ( + <> + + } elevation="high" density="compact"> + + + + ) : !isTerminal ? ( + + Trials appear once the study finishes. + + ) : isResultsError ? ( + + ) : isLoadingResults ? ( + + + + ) : !results ? ( + + This study did not register any trial results. + + ) : ( + <> + + + + )} + + + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.test.ts b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.test.ts new file mode 100644 index 0000000000..7bda80cd4f --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.test.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { agentsListOptimizeJobResults } from '@nemo/sdk/generated/agents/agents'; +import { filesDownloadFile, filesListFilesetFiles } from '@nemo/sdk/generated/platform/files'; +import { + fetchStudyResults, + parseDurationSeconds, +} from '@studio/routes/agents/AgentOptimizationDetailRoute/studyResults'; + +vi.mock('@nemo/sdk/generated/agents/agents', () => ({ + agentsListOptimizeJobResults: vi.fn(), +})); +vi.mock('@nemo/sdk/generated/platform/files', () => ({ + filesDownloadFile: vi.fn(), + filesListFilesetFiles: vi.fn(), +})); + +const workspace = 'default'; +const jobName = 'brevity-sweep-3'; +const resultDir = 'results/attempt-1/optimizer_results'; + +const SUMMARY = JSON.stringify({ + status: 'completed', + n_trials: 3, + best_trial: 6, + best_params: { temperature: 0.2 }, + best_values: [4.19, 742], + metric_names: ['avg_score', 'avg_tokens'], +}); + +const TRIALS_CSV = [ + 'number,state,datetime_start,datetime_complete,duration,values_avg_score,values_avg_tokens,params_prompt,params_temperature,rep_scores,pareto_optimal', + '6,COMPLETE,2026-09-11T10:00:00,2026-09-11T10:00:12,0:00:12.500000,4.19,742,v3-concise,0.2,"[4.1, 4.2, 4.27]",True', + '3,COMPLETE,2026-09-11T10:00:12,2026-09-11T10:00:21,0:00:09,4.14,610,v2-structured,0.2,"[4.1, 4.18]",True', + '4,FAIL,2026-09-11T10:00:21,,,,,v2-structured,0.9,null,False', +].join('\n'); + +const mockResults = (artifactUrl: string) => + vi.mocked(agentsListOptimizeJobResults).mockResolvedValue({ + data: [ + { + name: 'optimizer_results', + job: jobName, + workspace, + artifact_url: artifactUrl, + artifact_storage_type: 'fileset', + }, + ], + } as Awaited>); + +const mockFiles = (paths: string[]) => + vi.mocked(filesListFilesetFiles).mockResolvedValue({ + data: paths.map((path) => ({ path, size: 1, file_ref: path })), + } as Awaited>); + +const mockDownloads = (byPath: Record) => + vi + .mocked(filesDownloadFile) + .mockImplementation(async (_workspace, _fileset, path) => new Blob([byPath[path] ?? ''])); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('parseDurationSeconds', () => { + it.each([ + ['0:00:12.500000', 12.5], + ['0:01:09', 69], + ['1:00:00', 3600], + ['2 days, 0:00:30', 172_830], + ])('parses %s', (input, expected) => { + expect(parseDurationSeconds(input)).toBe(expected); + }); + + it('returns null for the empty duration of a trial that never completed', () => { + expect(parseDurationSeconds('')).toBeNull(); + expect(parseDurationSeconds(undefined)).toBeNull(); + }); +}); + +describe('fetchStudyResults', () => { + it('parses the summary and every trial row', async () => { + mockResults(`fileset://${workspace}/study-artifacts#${resultDir}`); + mockFiles([`${resultDir}/study_summary.json`, `${resultDir}/trials_dataframe_params.csv`]); + mockDownloads({ + [`${resultDir}/study_summary.json`]: SUMMARY, + [`${resultDir}/trials_dataframe_params.csv`]: TRIALS_CSV, + }); + + const results = await fetchStudyResults(workspace, jobName); + + expect(results?.summary).toEqual({ + nTrials: 3, + bestTrial: 6, + metricNames: ['avg_score', 'avg_tokens'], + bestValues: [4.19, 742], + }); + expect(results?.metricNames).toEqual(['avg_score', 'avg_tokens']); + expect(results?.trials).toHaveLength(3); + + const [best] = results?.trials ?? []; + expect(best).toEqual({ + number: 6, + state: 'COMPLETE', + durationSeconds: 12.5, + paretoOptimal: true, + metrics: [ + { name: 'avg_score', value: 4.19 }, + { name: 'avg_tokens', value: 742 }, + ], + // rep_scores and the datetime columns are not parameters, so they stay out of params. + params: [ + { name: 'prompt', value: 'v3-concise' }, + { name: 'temperature', value: '0.2' }, + ], + }); + }); + + it('keeps a failed trial, with null metrics and no duration', async () => { + mockResults(`fileset://${workspace}/study-artifacts#${resultDir}`); + mockFiles([`${resultDir}/study_summary.json`, `${resultDir}/trials_dataframe_params.csv`]); + mockDownloads({ + [`${resultDir}/study_summary.json`]: SUMMARY, + [`${resultDir}/trials_dataframe_params.csv`]: TRIALS_CSV, + }); + + const results = await fetchStudyResults(workspace, jobName); + const failed = results?.trials.find((trial) => trial.number === 4); + + expect(failed?.state).toBe('FAIL'); + expect(failed?.durationSeconds).toBeNull(); + expect(failed?.paretoOptimal).toBe(false); + expect(failed?.metrics).toEqual([ + { name: 'avg_score', value: null }, + { name: 'avg_tokens', value: null }, + ]); + }); + + it('returns null when the job has published no study artifacts', async () => { + mockResults(`fileset://${workspace}/study-artifacts#${resultDir}`); + mockFiles([`${resultDir}/optimized_config.yml`]); + + await expect(fetchStudyResults(workspace, jobName)).resolves.toBeNull(); + expect(filesDownloadFile).not.toHaveBeenCalled(); + }); + + it('downloads from the workspace the artifact URL names, not the job workspace', async () => { + mockResults(`fileset://shared-artifacts/study-artifacts#${resultDir}`); + mockFiles([`${resultDir}/trials_dataframe_params.csv`]); + mockDownloads({ [`${resultDir}/trials_dataframe_params.csv`]: TRIALS_CSV }); + + await fetchStudyResults(workspace, jobName); + + expect(filesListFilesetFiles).toHaveBeenCalledWith( + 'shared-artifacts', + 'study-artifacts', + expect.anything(), + undefined + ); + expect(filesDownloadFile).toHaveBeenCalledWith( + 'shared-artifacts', + 'study-artifacts', + `${resultDir}/trials_dataframe_params.csv`, + undefined + ); + }); + + it('finds sibling study files when the artifact URL points at a specific file', async () => { + mockResults(`fileset://${workspace}/study-artifacts#${resultDir}/study_summary.json`); + mockFiles([`${resultDir}/study_summary.json`, `${resultDir}/trials_dataframe_params.csv`]); + mockDownloads({ + [`${resultDir}/study_summary.json`]: SUMMARY, + [`${resultDir}/trials_dataframe_params.csv`]: TRIALS_CSV, + }); + + const results = await fetchStudyResults(workspace, jobName); + + expect(filesListFilesetFiles).toHaveBeenCalledWith( + workspace, + 'study-artifacts', + { path: resultDir }, + undefined + ); + expect(results?.summary).not.toBeNull(); + expect(results?.trials).toHaveLength(3); + }); + + it('reads trials even when the summary is missing, falling back to CSV metric order', async () => { + mockResults(`fileset://${workspace}/study-artifacts#${resultDir}`); + mockFiles([`${resultDir}/trials_dataframe_params.csv`]); + mockDownloads({ [`${resultDir}/trials_dataframe_params.csv`]: TRIALS_CSV }); + + const results = await fetchStudyResults(workspace, jobName); + + expect(results?.summary).toBeNull(); + expect(results?.metricNames).toEqual(['avg_score', 'avg_tokens']); + expect(results?.trials).toHaveLength(3); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.ts b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.ts new file mode 100644 index 0000000000..30fe053093 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentOptimizationDetailRoute/studyResults.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSelect/parseFilesetLocation'; +import { agentsListOptimizeJobResults } from '@nemo/sdk/generated/agents/agents'; +import { filesDownloadFile, filesListFilesetFiles } from '@nemo/sdk/generated/platform/files'; +import { FileStorageType } from '@nemo/sdk/generated/platform/schema'; +import Papa from 'papaparse'; + +const SUMMARY_FILE = 'study_summary.json'; +const TRIALS_FILE = 'trials_dataframe_params.csv'; + +/** One row of {@link TRIALS_FILE}, before the `values_`/`params_` columns are split out. */ +type TrialCsvRow = Record; + +export interface StudySummary { + nTrials: number | null; + bestTrial: number | null; + metricNames: string[]; + bestValues: (number | null)[]; +} + +export interface TrialMetric { + name: string; + value: number | null; +} + +export interface TrialParam { + name: string; + value: string; +} + +export interface Trial { + number: number; + state: string; + durationSeconds: number | null; + paretoOptimal: boolean; + metrics: TrialMetric[]; + params: TrialParam[]; +} + +export interface StudyResults { + summary: StudySummary | null; + trials: Trial[]; + metricNames: string[]; +} + +const toNumber = (value: string | number | null | undefined): number | null => { + if (value === null || value === undefined || value === '') return null; + const parsed = typeof value === 'number' ? value : Number(value); + return Number.isFinite(parsed) ? parsed : null; +}; + +/** + * Parse `str(timedelta)` — `H:MM:SS[.ffffff]`, with an optional leading `N day(s), `. + * Returns null for the empty string the writer emits for a trial that never completed. + */ +export const parseDurationSeconds = (value: string | undefined): number | null => { + if (!value) return null; + const match = /^(?:(\d+)\s+days?,\s*)?(\d+):(\d{2}):(\d{2}(?:\.\d+)?)$/.exec(value.trim()); + if (!match) return null; + const [, days, hours, minutes, seconds] = match; + return ( + Number(days ?? 0) * 86_400 + Number(hours) * 3_600 + Number(minutes) * 60 + Number(seconds) + ); +}; + +/** Python's `csv` writer emits bare `True`/`False` for booleans. */ +const parseBoolean = (value: string | undefined): boolean => (value ?? '').toLowerCase() === 'true'; + +const parseSummary = (text: string): StudySummary => { + const raw = JSON.parse(text) as Record; + const metricNames = Array.isArray(raw.metric_names) ? raw.metric_names.map(String) : []; + const bestValues = Array.isArray(raw.best_values) + ? raw.best_values.map((v) => toNumber(v as number)) + : []; + return { + nTrials: toNumber(raw.n_trials as number), + bestTrial: toNumber(raw.best_trial as number), + metricNames, + bestValues, + }; +}; + +/** + * Split each CSV row into its fixed columns, its `values_` metrics and its + * `params_` parameters. Metric order follows `metricNames` when the summary supplied it, + * so the table's columns match the objective order the study was configured with; otherwise it + * falls back to header order. + */ +const parseTrials = ( + text: string, + metricNames: string[] +): { trials: Trial[]; metrics: string[] } => { + const parsed = Papa.parse(text, { header: true, skipEmptyLines: true }); + const headers = parsed.meta.fields ?? []; + + const headerMetrics = headers + .filter((h) => h.startsWith('values_')) + .map((h) => h.slice('values_'.length)); + const metrics = metricNames.length + ? metricNames.filter((name) => headerMetrics.includes(name)) + : headerMetrics; + const paramNames = headers + .filter((h) => h.startsWith('params_')) + .map((h) => h.slice('params_'.length)); + + const trials = parsed.data.flatMap((row) => { + const number = toNumber(row.number); + if (number === null) return []; + return [ + { + number, + state: row.state ?? '', + durationSeconds: parseDurationSeconds(row.duration), + paretoOptimal: parseBoolean(row.pareto_optimal), + metrics: metrics.map((name) => ({ name, value: toNumber(row[`values_${name}`]) })), + params: paramNames + .map((name) => ({ name, value: row[`params_${name}`] ?? '' })) + .filter((param) => param.value !== ''), + }, + ]; + }); + + return { trials, metrics }; +}; + +/** + * Locate the study artifacts inside the job's registered results. + */ +const locateStudyFiles = async ( + workspace: string, + jobName: string, + signal?: AbortSignal +): Promise<{ + workspace: string; + fileset: string; + summaryPath?: string; + trialsPath?: string; +} | null> => { + const { data: results } = await agentsListOptimizeJobResults(workspace, jobName, signal); + + for (const result of results) { + if (result.artifact_storage_type !== FileStorageType.fileset) continue; + const parsed = parseFilesetLocation(result.artifact_url, workspace); + if (!parsed) continue; + + const { data: files } = await filesListFilesetFiles( + parsed.workspace, + parsed.name, + { path: parsed.filesListPathPrefix }, + signal + ); + const at = (fileName: string) => + files.find((file) => file.path === fileName || file.path.endsWith(`/${fileName}`))?.path; + + const summaryPath = at(SUMMARY_FILE); + const trialsPath = at(TRIALS_FILE); + if (summaryPath ?? trialsPath) { + return { workspace: parsed.workspace, fileset: parsed.name, summaryPath, trialsPath }; + } + } + + return null; +}; + +const downloadText = async ( + workspace: string, + fileset: string, + path: string, + signal?: AbortSignal +): Promise => { + const blob = await filesDownloadFile(workspace, fileset, path, signal); + return blob ? blob.text() : null; +}; + +/** + * Read the study summary and the per-trial table for one optimize job. + */ +export const fetchStudyResults = async ( + workspace: string, + jobName: string, + signal?: AbortSignal +): Promise => { + const located = await locateStudyFiles(workspace, jobName, signal); + if (!located) return null; + + const { workspace: artifactWorkspace, fileset, summaryPath, trialsPath } = located; + const [summaryText, trialsText] = await Promise.all([ + summaryPath ? downloadText(artifactWorkspace, fileset, summaryPath, signal) : null, + trialsPath ? downloadText(artifactWorkspace, fileset, trialsPath, signal) : null, + ]); + + const summary = summaryText ? parseSummary(summaryText) : null; + const { trials, metrics } = trialsText + ? parseTrials(trialsText, summary?.metricNames ?? []) + : { trials: [], metrics: summary?.metricNames ?? [] }; + + return { summary, trials, metricNames: metrics }; +}; diff --git a/web/packages/studio/src/routes/groups/agentRoutes.tsx b/web/packages/studio/src/routes/groups/agentRoutes.tsx index 7b3b0f4e8e..ad72666c13 100644 --- a/web/packages/studio/src/routes/groups/agentRoutes.tsx +++ b/web/packages/studio/src/routes/groups/agentRoutes.tsx @@ -3,7 +3,11 @@ import { RouteErrorPanel } from '@nemo/common/src/components/ErrorPanel'; import { ENTITY_ICONS } from '@nemo/common/src/constants/entityIcons'; -import { AGENTS_ENABLED, MONITOR_ENABLED } from '@studio/constants/environment'; +import { + AGENT_OPTIMIZATIONS_ENABLED, + AGENTS_ENABLED, + MONITOR_ENABLED, +} from '@studio/constants/environment'; import { ROUTES } from '@studio/constants/routes'; import { iconColorClass } from '@studio/routes/constants'; import { agentsRoutes, getAgentMonitorRoute } from '@studio/routes/utils'; @@ -36,6 +40,14 @@ const AgentEvaluationDetailRoute = default: m.AgentEvaluationDetailRoute, })) ); +const AgentOptimizationDetailRoute = + AGENTS_ENABLED && + AGENT_OPTIMIZATIONS_ENABLED && + lazy(() => + import('@studio/routes/agents/AgentOptimizationDetailRoute/index').then((m) => ({ + default: m.AgentOptimizationDetailRoute, + })) + ); export const agentRoutes: RouteObject[] = agentsRoutes([ { @@ -57,6 +69,15 @@ export const agentRoutes: RouteObject[] = agentsRoutes([ element: AgentEvaluationDetailRoute ? : null, errorElement: , }, + ...(AgentOptimizationDetailRoute + ? [ + { + path: ROUTES.workspace.agentOptimizationDetail, + element: , + errorElement: , + }, + ] + : []), { path: ROUTES.workspace.agentDetail, element: AgentDetailRoute ? : null, diff --git a/web/packages/studio/src/routes/utils.ts b/web/packages/studio/src/routes/utils.ts index 2c888bb2dd..f8d695586b 100644 --- a/web/packages/studio/src/routes/utils.ts +++ b/web/packages/studio/src/routes/utils.ts @@ -650,6 +650,17 @@ export const getAgentEvaluationsTabRoute = (workspace: string, agentName: string return `${getAgentDetailRoute(workspace, agentName)}?tab=evaluations`; }; +export const getAgentOptimizationsTabRoute = (workspace: string, agentName: string) => { + return `${getAgentDetailRoute(workspace, agentName)}?tab=optimizations`; +}; + +export const getAgentOptimizationDetailRoute = (workspace: string, optimizeJobName: string) => { + return generatePath(ROUTES.workspace.agentOptimizationDetail, { + workspace, + optimizeJobName, + }); +}; + export const getAgentDeploymentsListRoute = (workspace: string) => { return generatePath(ROUTES.workspace.agentDeploymentsList, { workspace }); }; diff --git a/web/packages/studio/src/tests/title-change.test.tsx b/web/packages/studio/src/tests/title-change.test.tsx index a2bcfd0fd7..7a9b19f638 100644 --- a/web/packages/studio/src/tests/title-change.test.tsx +++ b/web/packages/studio/src/tests/title-change.test.tsx @@ -33,6 +33,7 @@ const pathParams = { [RP.agentName]: '', [RP.agentDeploymentName]: '', [RP.agentEvalJobName]: 'test-agent-eval-job', + [RP.optimizeJobName]: 'test-optimize-job', [RP.jobName]: 'test-job', [RP.benchmarkName]: 'test-benchmark', [RP.experimentName]: 'test-experiment-group',