From 53cf25498539148e67386712bb747ee75d1f8290 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Mon, 1 Jul 2024 07:53:33 -0500 Subject: [PATCH] add compare runs tests --- .../cypress/pages/pipelines/compareRuns.ts | 165 +++++++++++++++++- .../cypress/cypress/support/commands/odh.ts | 16 ++ .../tests/mocked/pipelines/compareRuns.cy.ts | 146 ++++++++++++++-- .../content/artifacts/charts/ROCCurve.tsx | 2 +- .../confusionMatrix/ConfusionMatrix.tsx | 4 +- .../PipelineRunArtifactSelect.tsx | 10 +- .../ConfusionMatrixCompare.tsx | 9 +- .../markdown/MarkdownCompare.tsx | 8 +- .../metricsSection/roc/RocCurveTable.tsx | 1 + .../concepts/pipelines/topology/parseUtils.ts | 2 + .../compareRuns/CompareRunsMetricsSection.tsx | 13 +- 11 files changed, 345 insertions(+), 31 deletions(-) diff --git a/frontend/src/__tests__/cypress/cypress/pages/pipelines/compareRuns.ts b/frontend/src/__tests__/cypress/cypress/pages/pipelines/compareRuns.ts index 0b6674e8bd..8383adc86c 100644 --- a/frontend/src/__tests__/cypress/cypress/pages/pipelines/compareRuns.ts +++ b/frontend/src/__tests__/cypress/cypress/pages/pipelines/compareRuns.ts @@ -1,4 +1,5 @@ import { TableRow } from '~/__tests__/cypress/cypress/pages/components/table'; +import { Contextual } from '~/__tests__/cypress/cypress/pages/components/Contextual'; class CompareRunsGlobal { visit(projectName: string, experimentId: string, runIds: string[] = []) { @@ -61,8 +62,79 @@ class CompareMetricsContent { return cy.findByTestId('compare-runs-metrics-content'); } + findScalarMetricsTab() { + return this.find().findByTestId('compare-runs-scalar-metrics-tab'); + } + + findScalarMetricsTabContent() { + return new CompareRunsScalarMetrics(() => + this.find().findByTestId('compare-runs-scalar-metrics-tab-content').parent(), + ); + } + + findConfusionMatrixTab() { + return this.find().findByTestId('compare-runs-confusion-matrix-tab'); + } + + findConfusionMatrixTabContent() { + return new CompareRunsConfusionMatrix(() => + this.find().findByTestId('compare-runs-confusion-matrix-tab-content').parent(), + ); + } + + findRocCurveTab() { + return this.find().findByTestId('compare-runs-roc-curve-tab'); + } + + findRocCurveTabContent() { + return new CompareRunsRocCurve(() => + this.find().findByTestId('compare-runs-roc-curve-tab-content').parent(), + ); + } + + findMarkdownTab() { + return this.find().findByTestId('compare-runs-markdown-tab'); + } + + findMarkdownTabContent() { + return new CompareRunsMarkdown(() => + this.find().findByTestId('compare-runs-markdown-tab-content').parent(), + ); + } +} + +class RocCurveFilterTableRow extends TableRow { + findRunName() { + return this.find().find(`[data-label="Run name"]`); + } +} + +class CompareRunsRocCurve extends Contextual { + findRocCurveEmptyState() { + return this.find().findByTestId('no-result-found-title'); + } + + getRocCurveRowByName(name: string) { + return new RocCurveFilterTableRow(() => + this.find() + .find(`[data-label="Execution name > Artifact name"]`) + .contains(name) + .parents('tr'), + ); + } + + findRocCruveSearchBar() { + return this.find().findByTestId('roc-curve-search'); + } + + findRocCurveGraph() { + return this.find().findByTestId('roc-curve-graph'); + } +} + +class CompareRunsScalarMetrics extends Contextual { findScalarMetricsTable() { - return cy.findByTestId('compare-runs-scalar-metrics-table'); + return this.find().findByTestId('compare-runs-scalar-metrics-table'); } findScalarMetricsColumnByName(name: string) { @@ -78,7 +150,96 @@ class CompareMetricsContent { } findScalarMetricsEmptyState() { - return cy.findByTestId('compare-runs-scalar-metrics-empty-state'); + return this.find().findByTestId('compare-runs-scalar-metrics-empty-state'); + } +} + +class CompareRunsArtifactSelect extends Contextual { + findSelectOption(name: string) { + return this.find().findByTestId('pipeline-run-artifact-select').findSelectOption(name); + } + + findExpandButton() { + return this.find().findByTestId('pipeline-run-artifact-expand-button'); + } + + findArtifactContent(index = 0) { + return this.find().findByTestId(`pipeline-run-artifact-content-${index}`); + } +} + +class ConfusionMatrixArtifactSelect extends CompareRunsArtifactSelect { + findConfusionMatrixGraph(index = 0) { + return new ConfusionMatrixGraph(() => this.findArtifactContent(index)); + } +} + +class CompareRunsMarkdown extends Contextual { + findMarkdownEmptyState() { + return this.find().findByTestId('compare-runs-markdown-empty-state'); + } + + findExpandedMarkdown() { + return this.find().findByTestId('compare-runs-markdown-expanded-graph'); + } + + findMarkdownSelect(runId: string) { + return new CompareRunsArtifactSelect(() => + this.find().findByTestId(`compare-runs-markdown-${runId}`), + ); + } +} + +class CompareRunsConfusionMatrix extends Contextual { + findConfusionMatrixEmptyState() { + return this.find().findByTestId('compare-runs-confusion-matrix-empty-state'); + } + + findExpandedConfusionMatrix() { + return new ConfusionMatrixGraph(() => + this.find().findByTestId('compare-runs-confusion-matrix-expanded-graph'), + ); + } + + findConfusionMatrixSelect(runId: string) { + return new ConfusionMatrixArtifactSelect(() => + this.find().findByTestId(`compare-runs-confusion-matrix-${runId}`), + ); + } +} + +class ConfusionMatrixGraph extends Contextual { + findLabelY(index: number) { + return this.find().findByTestId(`confusion-matrix-label-y${index}`); + } + + findLabelX(index: number) { + return this.find().findByTestId(`confusion-matrix-label-x${index}`); + } + + findCell(rowIndex: number, colIndex: number) { + return this.find().findByTestId(`confusion-matrix-cell-${rowIndex}-${colIndex}`); + } + + checkLabels(labels: string[]) { + // Check the labels on the left side (true labels) + labels.forEach((label, index) => { + this.findLabelY(index).should('contain.text', label); + }); + + // Check the labels at the bottom (predicted labels) + labels.forEach((label, index) => { + this.findLabelX(index).should('contain.text', label); + }); + } + + checkCells(data: number[][]) { + // Check the data in the cells + data.forEach((row, rowIndex) => { + row.forEach((cell, cellIndex) => { + this.findCell(rowIndex, cellIndex).should('contain.text', cell.toString()); + }); + }); } } diff --git a/frontend/src/__tests__/cypress/cypress/support/commands/odh.ts b/frontend/src/__tests__/cypress/cypress/support/commands/odh.ts index 371b1513fe..24e813f8b4 100644 --- a/frontend/src/__tests__/cypress/cypress/support/commands/odh.ts +++ b/frontend/src/__tests__/cypress/cypress/support/commands/odh.ts @@ -500,6 +500,22 @@ declare global { path: { username: string }; }, response: OdhResponse<{ notebook: NotebookKind; isRunning: boolean }>, + ) => Cypress.Chainable) & + (( + type: 'GET /api/storage/:namespace', + options: { + query: { key: string; peek?: number }; + path: { namespace: string }; + }, + response: OdhResponse, + ) => Cypress.Chainable) & + (( + type: 'GET /api/storage/:namespace/size', + options: { + query: { key: string }; + path: { namespace: string }; + }, + response: OdhResponse, ) => Cypress.Chainable); } } diff --git a/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/compareRuns.cy.ts b/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/compareRuns.cy.ts index b3dd860f9d..8bd387085f 100644 --- a/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/compareRuns.cy.ts +++ b/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/compareRuns.cy.ts @@ -209,43 +209,135 @@ describe('Compare runs', () => { it('shows empty state when the Runs list has no selections', () => { compareRunsListTable.findSelectAllCheckbox().click(); // Uncheck all - compareRunsMetricsContent.findScalarMetricsEmptyState().should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricsEmptyState() + .should('exist'); }); it('displays scalar metrics table data based on selections from Run list', () => { - compareRunsMetricsContent.findScalarMetricsTable().should('exist'); - compareRunsMetricsContent.findScalarMetricsColumnByName('Run name').should('exist'); - compareRunsMetricsContent.findScalarMetricsColumnByName('Run 1').should('exist'); - compareRunsMetricsContent.findScalarMetricsColumnByName('Run 2').should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricsTable() + .should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricsColumnByName('Run name') + .should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricsColumnByName('Run 1') + .should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricsColumnByName('Run 2') + .should('exist'); compareRunsMetricsContent + .findScalarMetricsTabContent() .findScalarMetricsColumnByName('Execution name > Artifact name') .should('exist'); compareRunsMetricsContent + .findScalarMetricsTabContent() .findScalarMetricsColumnByName('digit-classification > metrics') .should('exist'); - compareRunsMetricsContent.findScalarMetricName('accuracy').should('exist'); - compareRunsMetricsContent.findScalarMetricCell('accuracy', 1).should('contain.text', '92'); - compareRunsMetricsContent.findScalarMetricCell('accuracy', 2).should('contain.text', '92'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricName('accuracy') + .should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricCell('accuracy', 1) + .should('contain.text', '92'); + compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricCell('accuracy', 2) + .should('contain.text', '92'); - compareRunsMetricsContent.findScalarMetricName('displayName').should('exist'); compareRunsMetricsContent + .findScalarMetricsTabContent() + .findScalarMetricName('displayName') + .should('exist'); + compareRunsMetricsContent + .findScalarMetricsTabContent() .findScalarMetricCell('displayName', 1) .should('contain.text', '"metrics"'); compareRunsMetricsContent + .findScalarMetricsTabContent() .findScalarMetricCell('displayName', 2) .should('contain.text', '"metrics"'); }); - // TODO tests for Confusion matrix tab - // TODO tests for ROC curve tab - // TODO tests for Markdown tab + it('displays confusion matrix data based on selections from Run list', () => { + compareRunsMetricsContent.findConfusionMatrixTab().click(); + + const confusionMatrixCompare = compareRunsMetricsContent + .findConfusionMatrixTabContent() + .findConfusionMatrixSelect(mockRun.run_id); + + // check graph data + const graph = confusionMatrixCompare.findConfusionMatrixGraph(); + graph.checkLabels(['Setosa', 'Versicolour', 'Virginica']); + graph.checkCells([ + [38, 0, 0], + [2, 19, 9], + [1, 17, 19], + ]); + + // check expanded graph + confusionMatrixCompare.findExpandButton().click(); + compareRunsMetricsContent + .findConfusionMatrixTabContent() + .findExpandedConfusionMatrix() + .find() + .should('exist'); + }); + + it('displays ROC curve filter table with correct artifacts', () => { + compareRunsMetricsContent.findRocCurveTab().click(); + const content = compareRunsMetricsContent.findRocCurveTabContent(); + + const row = content.getRocCurveRowByName('wine-classification > metrics'); + row.findRunName().should('contain.text', 'Run 1'); + content.findRocCurveGraph().should('contain.text', 'Series #1'); + + row.findRowCheckbox().uncheck(); + content.findRocCurveGraph().should('not.contain.text', 'Series #1'); + }); + + it('displays ROC curve empty state when no artifacts are found', () => { + compareRunsMetricsContent.findRocCurveTab().click(); + const content = compareRunsMetricsContent.findRocCurveTabContent(); + content.findRocCruveSearchBar().type('invalid'); + content.findRocCurveEmptyState().should('exist'); + }); + + it('displays markdown fetched from S3 based on selected runs', () => { + cy.wait('@s3Loaded', { + timeout: 15000, + }); + + compareRunsMetricsContent.findMarkdownTab().click(); + const markdownCompare = compareRunsMetricsContent + .findMarkdownTabContent() + .findMarkdownSelect(mockRun.run_id); + + // check markdown content + markdownCompare.findArtifactContent().should('contain.text', 'This is a markdown file'); + + // check expanded graph + markdownCompare.findExpandButton().click(); + compareRunsMetricsContent.findMarkdownTabContent().findExpandedMarkdown().should('exist'); + }); }); }); const initIntercepts = () => { - cy.interceptOdh('GET /api/config', mockDashboardConfig({ disablePipelineExperiments: false })); + cy.interceptOdh( + 'GET /api/config', + mockDashboardConfig({ disablePipelineExperiments: false, disableS3Endpoint: false }), + ); cy.interceptK8sList( DataSciencePipelineApplicationModel, mockK8sResourceList([ @@ -305,4 +397,32 @@ const initIntercepts = () => { ); initMlmdIntercepts(projectName); + + cy.interceptOdh( + 'GET /api/storage/:namespace', + { + path: { + namespace: projectName, + }, + query: { + key: 'metrics-visualization-pipeline/16dbff18-a3d5-4684-90ac-4e6198a9da0f/markdown-visualization/markdown_artifact', + }, + }, + { body: 'This is a markdown file' }, + ).as('s3Loaded'); + + cy.interceptOdh( + 'GET /api/storage/:namespace/size', + { + path: { + namespace: projectName, + }, + query: { + key: 'metrics-visualization-pipeline/16dbff18-a3d5-4684-90ac-4e6198a9da0f/markdown-visualization/markdown_artifact', + }, + }, + { + body: 100, + }, + ); }; diff --git a/frontend/src/concepts/pipelines/content/artifacts/charts/ROCCurve.tsx b/frontend/src/concepts/pipelines/content/artifacts/charts/ROCCurve.tsx index c58beb2959..0278780ba2 100644 --- a/frontend/src/concepts/pipelines/content/artifacts/charts/ROCCurve.tsx +++ b/frontend/src/concepts/pipelines/content/artifacts/charts/ROCCurve.tsx @@ -53,7 +53,7 @@ const ROCCurve: React.FC = ({ configs, maxContainerWidth, maxDime const baseLineData = Array.from(Array(100).keys()).map((x) => ({ x: x / 100, y: x / 100 })); return ( -
+
= ({ {data.map((row, rowIndex) => ( = ({ {row.map((cell, cellIndex) => ( = ({ }} /> {labels.map((label, i) => ( - +
({ {!expandedGraph && run && (