From 9f2e1689ca5512799d1565076da5787558a777a7 Mon Sep 17 00:00:00 2001 From: Jeff Puzzo Date: Tue, 14 May 2024 14:07:26 -0400 Subject: [PATCH] [RHOAIENG-2989] artifact node drawer details section --- .../mlmd/useArtifactsFromMlmdContext.ts | 32 +++++++ .../mlmd/useGetEventsByExecutionId.ts | 14 +-- .../pipelineRun/PipelineRunDetails.tsx | 55 +++++++----- .../PipelineRunDrawerRightContent.tsx | 37 +++++--- .../artifacts/ArtifactNodeDetails.tsx | 88 +++++++++++++++++++ .../artifacts/ArtifactNodeDrawerContent.tsx | 78 ++++++++++++++++ .../pipelineRun/artifacts/index.ts | 3 + .../artifacts/usePipelineRunArtifacts.tsx | 17 ++++ .../pipelines/topology/pipelineTaskTypes.ts | 2 + .../topology/usePipelineTaskTopology.tsx | 37 ++++++-- .../artifacts/useGetArtifactsList.ts | 4 +- 11 files changed, 318 insertions(+), 49 deletions(-) create mode 100644 frontend/src/concepts/pipelines/apiHooks/mlmd/useArtifactsFromMlmdContext.ts create mode 100644 frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx create mode 100644 frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx create mode 100644 frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/index.ts create mode 100644 frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/usePipelineRunArtifacts.tsx diff --git a/frontend/src/concepts/pipelines/apiHooks/mlmd/useArtifactsFromMlmdContext.ts b/frontend/src/concepts/pipelines/apiHooks/mlmd/useArtifactsFromMlmdContext.ts new file mode 100644 index 0000000000..a8cdedf4ec --- /dev/null +++ b/frontend/src/concepts/pipelines/apiHooks/mlmd/useArtifactsFromMlmdContext.ts @@ -0,0 +1,32 @@ +import React from 'react'; +import { MlmdContext } from '~/concepts/pipelines/apiHooks/mlmd/types'; +import { usePipelinesAPI } from '~/concepts/pipelines/context'; +import { Artifact } from '~/third_party/mlmd'; +import { GetArtifactsByContextRequest } from '~/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb'; +import useFetchState, { + FetchState, + FetchStateCallbackPromise, + NotReadyError, +} from '~/utilities/useFetchState'; + +export const useArtifactsFromMlmdContext = ( + context: MlmdContext | null, + refreshRate?: number, +): FetchState => { + const { metadataStoreServiceClient } = usePipelinesAPI(); + + const getArtifactsList = React.useCallback>(async () => { + if (!context) { + return Promise.reject(new NotReadyError('No context')); + } + + const request = new GetArtifactsByContextRequest(); + request.setContextId(context.getId()); + const res = await metadataStoreServiceClient.getArtifactsByContext(request); + return res.getArtifactsList(); + }, [metadataStoreServiceClient, context]); + + return useFetchState(getArtifactsList, [], { + refreshRate, + }); +}; diff --git a/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts b/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts index 30da97f0b8..810ad059e8 100644 --- a/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts +++ b/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts @@ -8,25 +8,25 @@ import useFetchState, { FetchState, FetchStateCallbackPromise } from '~/utilitie export const useGetEventsByExecutionId = ( executionId?: string, +): FetchState => + useGetEventsByExecutionIds([Number(executionId)]); + +export const useGetEventsByExecutionIds = ( + executionIds: number[], ): FetchState => { const { metadataStoreServiceClient } = usePipelinesAPI(); const call = React.useCallback< FetchStateCallbackPromise >(async () => { - const numberId = Number(executionId); const request = new GetEventsByExecutionIDsRequest(); - if (!executionId || Number.isNaN(numberId)) { - return null; - } - - request.setExecutionIdsList([numberId]); + request.setExecutionIdsList(executionIds); const response = await metadataStoreServiceClient.getEventsByExecutionIDs(request); return response; - }, [executionId, metadataStoreServiceClient]); + }, [executionIds, metadataStoreServiceClient]); return useFetchState(call, null); }; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx index 231d291403..fc6ff9ce51 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx @@ -37,25 +37,40 @@ import { PipelineRunType } from '~/pages/pipelines/global/runs/types'; import { routePipelineRunsNamespace } from '~/routes'; import PipelineJobReferenceName from '~/concepts/pipelines/content/PipelineJobReferenceName'; import useExecutionsForPipelineRun from '~/concepts/pipelines/content/pipelinesDetails/pipelineRun/useExecutionsForPipelineRun'; +import { useGetEventsByExecutionIds } from '~/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId'; +import { parseEventsByType } from '~/pages/pipelines/global/experiments/executions/utils'; +import { Event } from '~/third_party/mlmd'; +import { usePipelineRunArtifacts } from './artifacts'; const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, contextPath }) => { const { runId } = useParams(); const navigate = useNavigate(); const { namespace } = usePipelinesAPI(); - const [runResource, runLoaded, runError] = usePipelineRunById(runId, true); + const [run, runLoaded, runError] = usePipelineRunById(runId, true); const [version, versionLoaded, versionError] = usePipelineVersionById( - runResource?.pipeline_version_reference?.pipeline_id, - runResource?.pipeline_version_reference?.pipeline_version_id, + run?.pipeline_version_reference?.pipeline_id, + run?.pipeline_version_reference?.pipeline_version_id, ); - const pipelineSpec = version?.pipeline_spec ?? runResource?.pipeline_spec; + const pipelineSpec = version?.pipeline_spec ?? run?.pipeline_spec; const [deleting, setDeleting] = React.useState(false); const [detailsTab, setDetailsTab] = React.useState( RunDetailsTabs.DETAILS, ); const [selectedId, setSelectedId] = React.useState(null); - const [executions, executionsLoaded, executionsError] = useExecutionsForPipelineRun(runResource); - const nodes = usePipelineTaskTopology(pipelineSpec, runResource?.run_details, executions); + const [executions, executionsLoaded, executionsError] = useExecutionsForPipelineRun(run); + const [artifacts] = usePipelineRunArtifacts(run); + const [eventsResponse] = useGetEventsByExecutionIds( + React.useMemo(() => executions.map((execution) => execution.getId()), [executions]), + ); + const events = parseEventsByType(eventsResponse); + const nodes = usePipelineTaskTopology( + pipelineSpec, + run?.run_details, + executions, + events[Event.Type.OUTPUT], + artifacts, + ); const selectedNode = React.useMemo( () => nodes.find((n) => n.id === selectedId), @@ -64,7 +79,7 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, const getFirstNode = (firstId: string) => nodes.find((n) => n.id === firstId); - const loaded = runLoaded && (versionLoaded || !!runResource?.pipeline_spec); + const loaded = runLoaded && (versionLoaded || !!run?.pipeline_spec); const error = versionError || runError; if (error) { @@ -95,6 +110,7 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, panelContent={ setSelectedId(null)} executions={executions} /> @@ -110,29 +126,29 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, setDetailsTab(selection); setSelectedId(null); }} - pipelineRunDetails={runResource && pipelineSpec ? runResource : undefined} + pipelineRunDetails={run && pipelineSpec ? run : undefined} /> } > + run ? ( + ) : ( 'Error loading run' ) } subtext={ - runResource && ( + run && ( ) } description={ - runResource?.description ? ( - + run?.description ? ( + ) : ( '' ) @@ -143,15 +159,12 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, {breadcrumbPath} - + } headerAction={ - setDeleting(true)} - /> + setDeleting(true)} /> } empty={false} > @@ -180,7 +193,7 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, { if (deleteComplete) { navigate(contextPath ?? routePipelineRunsNamespace(namespace)); diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent.tsx index 398af65de5..cc9b916457 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent.tsx @@ -12,16 +12,19 @@ import PipelineRunDrawerRightTabs from '~/concepts/pipelines/content/pipelinesDe import './PipelineRunDrawer.scss'; import { PipelineTask } from '~/concepts/pipelines/topology'; import { Execution } from '~/third_party/mlmd'; +import { ArtifactNodeDrawerContent } from './artifacts'; type PipelineRunDrawerRightContentProps = { task?: PipelineTask; executions: Execution[]; + upstreamTaskName?: string; onClose: () => void; }; const PipelineRunDrawerRightContent: React.FC = ({ task, executions, + upstreamTaskName, onClose, }) => { if (!task) { @@ -35,18 +38,28 @@ const PipelineRunDrawerRightContent: React.FC - - - {task.name} {task.type === 'artifact' ? 'Artifact details' : ''} - - {task.status?.podName && {task.status.podName}} - - - - - - - + {task.type === 'artifact' ? ( + + ) : ( + <> + + + {task.name} + + {task.status?.podName && {task.status.podName}} + + + + + + + + + )} ); }; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx new file mode 100644 index 0000000000..293fe531bd --- /dev/null +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx @@ -0,0 +1,88 @@ +import React from 'react'; +import { Link } from 'react-router-dom'; + +import { + Title, + Flex, + FlexItem, + Stack, + DescriptionList, + DescriptionListGroup, + DescriptionListTerm, + DescriptionListDescription, +} from '@patternfly/react-core'; + +import { Artifact } from '~/third_party/mlmd'; +import { artifactsDetailsRoute } from '~/routes'; +import { usePipelinesAPI } from '~/concepts/pipelines/context'; +import { getArtifactName } from '~/pages/pipelines/global/experiments/artifacts/utils'; +import PipelinesTableRowTime from '~/concepts/pipelines/content/tables/PipelinesTableRowTime'; +import PipelineRunDrawerRightContent from '~/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent'; + +type ArtifactNodeDetailsProps = Pick< + React.ComponentProps, + 'upstreamTaskName' +> & { + artifact: Artifact.AsObject; +}; + +export const ArtifactNodeDetails: React.FC = ({ + artifact, + upstreamTaskName, +}) => { + const { namespace } = usePipelinesAPI(); + const artifactName = getArtifactName(artifact); + + return ( + + + + Artifact details + + + Upstream task + {upstreamTaskName} + + Artifact name + + {artifactName} + + + Artifact type + {artifact.type} + + Created at + + + + + + + + + + + Artifact URI + + + {artifactName} + {artifact.uri} + + + + + + ); +}; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx new file mode 100644 index 0000000000..2751386f67 --- /dev/null +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx @@ -0,0 +1,78 @@ +import React from 'react'; + +import { + DrawerHead, + Title, + Text, + DrawerActions, + DrawerCloseButton, + DrawerPanelBody, + Tabs, + Tab, + TabTitleText, + EmptyState, + EmptyStateHeader, + EmptyStateVariant, +} from '@patternfly/react-core'; + +import PipelineRunDrawerRightContent from '~/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent'; +import { ArtifactNodeDetails } from './ArtifactNodeDetails'; + +type ArtifactNodeDrawerContentProps = Omit< + React.ComponentProps, + 'executions' +>; + +enum ArtifactNodeDrawerTab { + Details = 'details', + Visualization = 'visualization', +} + +export const ArtifactNodeDrawerContent: React.FC = ({ + task, + upstreamTaskName, + onClose, +}) => { + const [activeTab, setActiveTab] = React.useState(ArtifactNodeDrawerTab.Details); + const artifact = task?.metadata?.toObject(); + + return task ? ( + <> + + + {task.name} + + {task.status?.podName && {task.status.podName}} + + + + + + {artifact ? ( + setActiveTab(tabName)} + > + Artifact details} + aria-label="Overview" + > + + + Visualization} + aria-label="Visualization" + /> + + ) : ( + + + + )} + + + ) : null; +}; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/index.ts b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/index.ts new file mode 100644 index 0000000000..9dfebc53ba --- /dev/null +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/index.ts @@ -0,0 +1,3 @@ +export * from './ArtifactNodeDetails'; +export * from './ArtifactNodeDrawerContent'; +export * from './usePipelineRunArtifacts'; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/usePipelineRunArtifacts.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/usePipelineRunArtifacts.tsx new file mode 100644 index 0000000000..575bb4f170 --- /dev/null +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/usePipelineRunArtifacts.tsx @@ -0,0 +1,17 @@ +import { useArtifactsFromMlmdContext } from '~/concepts/pipelines/apiHooks/mlmd/useArtifactsFromMlmdContext'; +import { usePipelineRunMlmdContext } from '~/concepts/pipelines/apiHooks/mlmd/usePipelineRunMlmdContext'; +import { isPipelineRunFinished } from '~/concepts/pipelines/apiHooks/usePipelineRunById'; +import { PipelineRunKFv2 } from '~/concepts/pipelines/kfTypes'; +import { Artifact } from '~/third_party/mlmd'; +import { FAST_POLL_INTERVAL } from '~/utilities/const'; + +export const usePipelineRunArtifacts = ( + run: PipelineRunKFv2 | null, +): [artifacts: Artifact[], loaded: boolean, error?: Error] => { + const isFinished = isPipelineRunFinished(run); + const refreshRate = isFinished ? 0 : FAST_POLL_INTERVAL; + const [context, , contextError] = usePipelineRunMlmdContext(run?.run_id, refreshRate); + const [artifacts, artifactsLoaded] = useArtifactsFromMlmdContext(context, refreshRate); + + return [artifacts, artifactsLoaded, contextError]; +}; diff --git a/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts b/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts index a43eb61885..bb94eb7717 100644 --- a/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts +++ b/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts @@ -5,6 +5,7 @@ import { InputDefinitionParameterType, RuntimeStateKF, } from '~/concepts/pipelines/kfTypes'; +import { Artifact } from '~/third_party/mlmd'; import { VolumeMount } from '~/types'; export type PipelineTaskParam = { @@ -46,6 +47,7 @@ export type PipelineTask = { steps?: PipelineTaskStep[]; inputs?: PipelineTaskInputOutput; outputs?: PipelineTaskInputOutput; + metadata?: Artifact | undefined; /** Run Status */ status?: PipelineTaskRunStatus; /** Volume Mounts */ diff --git a/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx b/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx index 99a7133363..43d91cfb0c 100644 --- a/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx +++ b/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx @@ -12,7 +12,7 @@ import { import { createNode } from '~/concepts/topology'; import { PipelineNodeModelExpanded } from '~/concepts/topology/types'; import { createArtifactNode, createGroupNode } from '~/concepts/topology/utils'; -import { Execution } from '~/third_party/mlmd'; +import { Artifact, Execution, Event } from '~/third_party/mlmd'; import { ComponentArtifactMap, composeArtifactType, @@ -31,9 +31,11 @@ import { PipelineTask, PipelineTaskRunStatus } from './pipelineTaskTypes'; const getArtifactPipelineTask = ( name: string, artifactType: InputOutputArtifactType, + artifactData: Artifact | undefined, ): PipelineTask => ({ type: 'artifact', name, + metadata: artifactData, inputs: { artifacts: [{ label: name, type: composeArtifactType(artifactType) }], }, @@ -46,19 +48,20 @@ const getInputArtifacts = ( groupId: string | undefined, status: PipelineTaskRunStatus | undefined, inputArtifacts: InputOutputDefinitionArtifacts | undefined, + artifactData: Artifact | undefined, ) => { if (!inputArtifacts) { return []; } - return Object.entries(inputArtifacts).map(([artifactKey, data]) => + return Object.entries(inputArtifacts).map(([artifactKey, { artifactType }]) => createArtifactNode( idForTaskArtifact(groupId, '', artifactKey), artifactKey, - getArtifactPipelineTask(artifactKey, data.artifactType), + getArtifactPipelineTask(artifactKey, artifactType, artifactData), undefined, translateStatusForNode(status?.state), - data.artifactType.schemaTitle, + artifactType.schemaTitle, ), ); }; @@ -69,17 +72,19 @@ const getTaskArtifacts = ( status: PipelineTaskRunStatus | undefined, componentRef: string, componentArtifactMap: ComponentArtifactMap, + artifactData: Artifact | undefined, ): PipelineNodeModelExpanded[] => { const artifactsInComponent = componentArtifactMap[componentRef]; if (!artifactsInComponent) { return []; } + return Object.entries(artifactsInComponent).map(([artifactKey, data]) => createArtifactNode( idForTaskArtifact(groupId, taskId, artifactKey), artifactKey, - getArtifactPipelineTask(artifactKey, data), + getArtifactPipelineTask(artifactKey, data, artifactData), [taskId], translateStatusForNode(status?.state), data.schemaTitle, @@ -98,6 +103,8 @@ const getNodesForTasks = ( runDetails?: RunDetailsKF, executions?: Execution[] | null, inputArtifacts?: InputOutputDefinitionArtifacts, + events?: Event[], + artifacts?: Artifact[] | undefined, ): [nestedNodes: PipelineNodeModelExpanded[], children: string[]] => { const nodes: PipelineNodeModelExpanded[] = []; const children: string[] = []; @@ -132,6 +139,17 @@ const getNodesForTasks = ( volumeMounts: parseVolumeMounts(spec.platform_spec, executorLabel), }; + const artifactData = artifacts?.find((artifact) => { + const artifactEvent = events?.find((event) => event.getArtifactId() === artifact.getId()); + const artifactExecution = executions?.find( + (execution) => execution.getId() === artifactEvent?.getExecutionId(), + ); + + return ( + artifactExecution?.getCustomPropertiesMap().get('task_name')?.getStringValue() === taskId + ); + }); + // Build artifact nodes with inputs from task nodes const artifactNodes = getTaskArtifacts( groupId, @@ -139,6 +157,7 @@ const getNodesForTasks = ( status, componentRef, componentArtifactMap, + artifactData, ); if (artifactNodes.length) { nodes.push(...artifactNodes); @@ -146,7 +165,7 @@ const getNodesForTasks = ( } // Build artifact nodes without inputs - const inputArtifactNodes = getInputArtifacts(groupId, status, inputArtifacts); + const inputArtifactNodes = getInputArtifacts(groupId, status, inputArtifacts, artifactData); if (inputArtifactNodes.length) { nodes.push(...inputArtifactNodes); children.push(...inputArtifactNodes.map((n) => n.id)); @@ -195,6 +214,8 @@ export const usePipelineTaskTopology = ( spec?: PipelineSpecVariable, runDetails?: RunDetailsKF, executions?: Execution[] | null, + events?: Event[], + artifacts?: Artifact[] | undefined, ): PipelineNodeModelExpanded[] => React.useMemo(() => { if (!spec) { @@ -227,7 +248,9 @@ export const usePipelineTaskTopology = ( runDetails, executions, inputDefinitions?.artifacts, + events, + artifacts, )[0], (node) => node.id, ); - }, [executions, runDetails, spec]); + }, [artifacts, events, executions, runDetails, spec]); diff --git a/frontend/src/pages/pipelines/global/experiments/artifacts/useGetArtifactsList.ts b/frontend/src/pages/pipelines/global/experiments/artifacts/useGetArtifactsList.ts index db9f88d5c2..7ef83a5e84 100644 --- a/frontend/src/pages/pipelines/global/experiments/artifacts/useGetArtifactsList.ts +++ b/frontend/src/pages/pipelines/global/experiments/artifacts/useGetArtifactsList.ts @@ -12,7 +12,7 @@ export interface ArtifactsListResponse { export const useGetArtifactsList = ( refreshRate?: number, -): FetchState => { +): FetchState => { const { pageToken, maxResultSize, filterQuery } = useMlmdListContext(); const { metadataStoreServiceClient } = usePipelinesAPI(); @@ -39,7 +39,7 @@ export const useGetArtifactsList = ( }; }, [filterQuery, pageToken, maxResultSize, metadataStoreServiceClient]); - return useFetchState(fetchArtifactsList, null, { + return useFetchState(fetchArtifactsList, undefined, { refreshRate, }); };