Skip to content

Commit

Permalink
add compare runs tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@Gkrumbach07
Gkrumbach07 committed Jul 2, 2024
1 parent fce1bf5 commit 53cf254
Show file tree
Hide file tree
Showing 11 changed files with 345 additions and 31 deletions.
165 changes: 163 additions & 2 deletions frontend/src/__tests__/cypress/cypress/pages/pipelines/compareRuns.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { TableRow } from '~/__tests__/cypress/cypress/pages/components/table';
import { Contextual } from '~/__tests__/cypress/cypress/pages/components/Contextual';

class CompareRunsGlobal {
visit(projectName: string, experimentId: string, runIds: string[] = []) {
Expand Down Expand Up @@ -61,8 +62,79 @@ class CompareMetricsContent {
return cy.findByTestId('compare-runs-metrics-content');
}

findScalarMetricsTab() {
return this.find().findByTestId('compare-runs-scalar-metrics-tab');
}

findScalarMetricsTabContent() {
return new CompareRunsScalarMetrics(() =>
this.find().findByTestId('compare-runs-scalar-metrics-tab-content').parent(),
);
}

findConfusionMatrixTab() {
return this.find().findByTestId('compare-runs-confusion-matrix-tab');
}

findConfusionMatrixTabContent() {
return new CompareRunsConfusionMatrix(() =>
this.find().findByTestId('compare-runs-confusion-matrix-tab-content').parent(),
);
}

findRocCurveTab() {
return this.find().findByTestId('compare-runs-roc-curve-tab');
}

findRocCurveTabContent() {
return new CompareRunsRocCurve(() =>
this.find().findByTestId('compare-runs-roc-curve-tab-content').parent(),
);
}

findMarkdownTab() {
return this.find().findByTestId('compare-runs-markdown-tab');
}

findMarkdownTabContent() {
return new CompareRunsMarkdown(() =>
this.find().findByTestId('compare-runs-markdown-tab-content').parent(),
);
}
}

class RocCurveFilterTableRow extends TableRow {
findRunName() {
return this.find().find(`[data-label="Run name"]`);
}
}

class CompareRunsRocCurve extends Contextual<HTMLElement> {
findRocCurveEmptyState() {
return this.find().findByTestId('no-result-found-title');
}

getRocCurveRowByName(name: string) {
return new RocCurveFilterTableRow(() =>
this.find()
.find(`[data-label="Execution name > Artifact name"]`)
.contains(name)
.parents('tr'),
);
}

findRocCruveSearchBar() {
return this.find().findByTestId('roc-curve-search');
}

findRocCurveGraph() {
return this.find().findByTestId('roc-curve-graph');
}
}

class CompareRunsScalarMetrics extends Contextual<HTMLDivElement> {
findScalarMetricsTable() {
return cy.findByTestId('compare-runs-scalar-metrics-table');
return this.find().findByTestId('compare-runs-scalar-metrics-table');
}

findScalarMetricsColumnByName(name: string) {
Expand All @@ -78,7 +150,96 @@ class CompareMetricsContent {
}

findScalarMetricsEmptyState() {
return cy.findByTestId('compare-runs-scalar-metrics-empty-state');
return this.find().findByTestId('compare-runs-scalar-metrics-empty-state');
}
}

class CompareRunsArtifactSelect extends Contextual<HTMLElement> {
findSelectOption(name: string) {
return this.find().findByTestId('pipeline-run-artifact-select').findSelectOption(name);
}

findExpandButton() {
return this.find().findByTestId('pipeline-run-artifact-expand-button');
}

findArtifactContent(index = 0) {
return this.find().findByTestId(`pipeline-run-artifact-content-${index}`);
}
}

class ConfusionMatrixArtifactSelect extends CompareRunsArtifactSelect {
findConfusionMatrixGraph(index = 0) {
return new ConfusionMatrixGraph(() => this.findArtifactContent(index));
}
}

class CompareRunsMarkdown extends Contextual<HTMLElement> {
findMarkdownEmptyState() {
return this.find().findByTestId('compare-runs-markdown-empty-state');
}

findExpandedMarkdown() {
return this.find().findByTestId('compare-runs-markdown-expanded-graph');
}

findMarkdownSelect(runId: string) {
return new CompareRunsArtifactSelect(() =>
this.find().findByTestId(`compare-runs-markdown-${runId}`),
);
}
}

class CompareRunsConfusionMatrix extends Contextual<HTMLElement> {
findConfusionMatrixEmptyState() {
return this.find().findByTestId('compare-runs-confusion-matrix-empty-state');
}

findExpandedConfusionMatrix() {
return new ConfusionMatrixGraph(() =>
this.find().findByTestId('compare-runs-confusion-matrix-expanded-graph'),
);
}

findConfusionMatrixSelect(runId: string) {
return new ConfusionMatrixArtifactSelect(() =>
this.find().findByTestId(`compare-runs-confusion-matrix-${runId}`),
);
}
}

class ConfusionMatrixGraph extends Contextual<HTMLElement> {
findLabelY(index: number) {
return this.find().findByTestId(`confusion-matrix-label-y${index}`);
}

findLabelX(index: number) {
return this.find().findByTestId(`confusion-matrix-label-x${index}`);
}

findCell(rowIndex: number, colIndex: number) {
return this.find().findByTestId(`confusion-matrix-cell-${rowIndex}-${colIndex}`);
}

checkLabels(labels: string[]) {
// Check the labels on the left side (true labels)
labels.forEach((label, index) => {
this.findLabelY(index).should('contain.text', label);
});

// Check the labels at the bottom (predicted labels)
labels.forEach((label, index) => {
this.findLabelX(index).should('contain.text', label);
});
}

checkCells(data: number[][]) {
// Check the data in the cells
data.forEach((row, rowIndex) => {
row.forEach((cell, cellIndex) => {
this.findCell(rowIndex, cellIndex).should('contain.text', cell.toString());
});
});
}
}

Expand Down
16 changes: 16 additions & 0 deletions frontend/src/__tests__/cypress/cypress/support/commands/odh.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,22 @@ declare global {
path: { username: string };
},
response: OdhResponse<{ notebook: NotebookKind; isRunning: boolean }>,
) => Cypress.Chainable<null>) &
((
type: 'GET /api/storage/:namespace',
options: {
query: { key: string; peek?: number };
path: { namespace: string };
},
response: OdhResponse<string>,
) => Cypress.Chainable<null>) &
((
type: 'GET /api/storage/:namespace/size',
options: {
query: { key: string };
path: { namespace: string };
},
response: OdhResponse<number>,
) => Cypress.Chainable<null>);
}
}
Expand Down
146 changes: 133 additions & 13 deletions frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/compareRuns.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,43 +209,135 @@ describe('Compare runs', () => {

it('shows empty state when the Runs list has no selections', () => {
compareRunsListTable.findSelectAllCheckbox().click(); // Uncheck all
compareRunsMetricsContent.findScalarMetricsEmptyState().should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsEmptyState()
.should('exist');
});

it('displays scalar metrics table data based on selections from Run list', () => {
compareRunsMetricsContent.findScalarMetricsTable().should('exist');
compareRunsMetricsContent.findScalarMetricsColumnByName('Run name').should('exist');
compareRunsMetricsContent.findScalarMetricsColumnByName('Run 1').should('exist');
compareRunsMetricsContent.findScalarMetricsColumnByName('Run 2').should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsTable()
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsColumnByName('Run name')
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsColumnByName('Run 1')
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsColumnByName('Run 2')
.should('exist');

compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsColumnByName('Execution name > Artifact name')
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricsColumnByName('digit-classification > metrics')
.should('exist');

compareRunsMetricsContent.findScalarMetricName('accuracy').should('exist');
compareRunsMetricsContent.findScalarMetricCell('accuracy', 1).should('contain.text', '92');
compareRunsMetricsContent.findScalarMetricCell('accuracy', 2).should('contain.text', '92');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricName('accuracy')
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricCell('accuracy', 1)
.should('contain.text', '92');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricCell('accuracy', 2)
.should('contain.text', '92');

compareRunsMetricsContent.findScalarMetricName('displayName').should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricName('displayName')
.should('exist');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricCell('displayName', 1)
.should('contain.text', '"metrics"');
compareRunsMetricsContent
.findScalarMetricsTabContent()
.findScalarMetricCell('displayName', 2)
.should('contain.text', '"metrics"');
});

// TODO tests for Confusion matrix tab
// TODO tests for ROC curve tab
// TODO tests for Markdown tab
it('displays confusion matrix data based on selections from Run list', () => {
compareRunsMetricsContent.findConfusionMatrixTab().click();

const confusionMatrixCompare = compareRunsMetricsContent
.findConfusionMatrixTabContent()
.findConfusionMatrixSelect(mockRun.run_id);

// check graph data
const graph = confusionMatrixCompare.findConfusionMatrixGraph();
graph.checkLabels(['Setosa', 'Versicolour', 'Virginica']);
graph.checkCells([
[38, 0, 0],
[2, 19, 9],
[1, 17, 19],
]);

// check expanded graph
confusionMatrixCompare.findExpandButton().click();
compareRunsMetricsContent
.findConfusionMatrixTabContent()
.findExpandedConfusionMatrix()
.find()
.should('exist');
});

it('displays ROC curve filter table with correct artifacts', () => {
compareRunsMetricsContent.findRocCurveTab().click();
const content = compareRunsMetricsContent.findRocCurveTabContent();

const row = content.getRocCurveRowByName('wine-classification > metrics');
row.findRunName().should('contain.text', 'Run 1');
content.findRocCurveGraph().should('contain.text', 'Series #1');

row.findRowCheckbox().uncheck();
content.findRocCurveGraph().should('not.contain.text', 'Series #1');
});

it('displays ROC curve empty state when no artifacts are found', () => {
compareRunsMetricsContent.findRocCurveTab().click();
const content = compareRunsMetricsContent.findRocCurveTabContent();
content.findRocCruveSearchBar().type('invalid');
content.findRocCurveEmptyState().should('exist');
});

it('displays markdown fetched from S3 based on selected runs', () => {
cy.wait('@s3Loaded', {
timeout: 15000,
});

compareRunsMetricsContent.findMarkdownTab().click();
const markdownCompare = compareRunsMetricsContent
.findMarkdownTabContent()
.findMarkdownSelect(mockRun.run_id);

// check markdown content
markdownCompare.findArtifactContent().should('contain.text', 'This is a markdown file');

// check expanded graph
markdownCompare.findExpandButton().click();
compareRunsMetricsContent.findMarkdownTabContent().findExpandedMarkdown().should('exist');
});
});
});

const initIntercepts = () => {
cy.interceptOdh('GET /api/config', mockDashboardConfig({ disablePipelineExperiments: false }));
cy.interceptOdh(
'GET /api/config',
mockDashboardConfig({ disablePipelineExperiments: false, disableS3Endpoint: false }),
);
cy.interceptK8sList(
DataSciencePipelineApplicationModel,
mockK8sResourceList([
Expand Down Expand Up @@ -305,4 +397,32 @@ const initIntercepts = () => {
);

initMlmdIntercepts(projectName);

cy.interceptOdh(
'GET /api/storage/:namespace',
{
path: {
namespace: projectName,
},
query: {
key: 'metrics-visualization-pipeline/16dbff18-a3d5-4684-90ac-4e6198a9da0f/markdown-visualization/markdown_artifact',
},
},
{ body: 'This is a markdown file' },
).as('s3Loaded');

cy.interceptOdh(
'GET /api/storage/:namespace/size',
{
path: {
namespace: projectName,
},
query: {
key: 'metrics-visualization-pipeline/16dbff18-a3d5-4684-90ac-4e6198a9da0f/markdown-visualization/markdown_artifact',
},
},
{
body: 100,
},
);
};
2 changes: 1 addition & 1 deletion frontend/src/concepts/pipelines/content/artifacts/charts/ROCCurve.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ const ROCCurve: React.FC<ROCCurveProps> = ({ configs, maxContainerWidth, maxDime
const baseLineData = Array.from(Array(100).keys()).map((x) => ({ x: x / 100, y: x / 100 }));

return (
<div style={{ width: maxContainerWidth || maxDimension }}>
<div style={{ width: maxContainerWidth || maxDimension }} data-testid="roc-curve-graph">
<Chart
ariaDesc="ROC Curve"
ariaTitle="ROC Curve"
Expand Down
Loading

0 comments on commit 53cf254

Please sign in to comment.