Skip to content

Commit

Permalink
JUnit report validation, all issues without flattening (#183)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dfrankland
dfrankland authored Nov 27, 2024
1 parent 3e54fd9 commit c21215e
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 51 deletions.
42 changes: 23 additions & 19 deletions cli/src/validate.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ use context::junit::{
parser::{JunitParseError, JunitParser},
validator::{
validate as validate_report, JunitReportValidation, JunitReportValidationFlatIssue,
JunitReportValidationIssueSubOptimal, JunitValidationLevel,
JunitReportValidationIssueSubOptimal, JunitValidationIssue, JunitValidationIssueType,
JunitValidationLevel,
},
};

Expand Down Expand Up @@ -212,12 +213,12 @@ fn print_validation_errors(report_validations: &JunitFileToValidation) -> (usize
match report_validation.1 {
Ok(validation) => {
num_test_suites = validation.test_suites().len();
num_test_cases = validation.test_cases_flat().len();
num_test_cases = validation.test_cases().len();

num_validation_errors = validation.num_invalid_issues();
num_validation_warnings = validation.num_suboptimal_issues();

all_issues = validation.all_issues_owned();
all_issues = validation.all_issues_flat();
}
Err(e) => {
report_parse_error = Some(e);
Expand Down Expand Up @@ -294,22 +295,25 @@ fn print_codeowners_validation(
print_validation_level(JunitValidationLevel::Valid)
);
println!(" Path: {:?}", owners.path);
for report_validation in report_validations {
match report_validation.1 {
Ok(validation) => {
let issues = validation.all_issues_owned();
if issues.iter().any(
|i|
i.error_message == JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing.to_string()
) {
println!(
" {} - CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.",
print_validation_level(JunitValidationLevel::SubOptimal)
);
}
}
Err(_) => {}
}

let has_test_cases_without_matching_codeowners_paths = report_validations
.iter()
.filter_map(|(_, report_validation)| report_validation.as_ref().ok())
.flat_map(|report_validation| report_validation.all_issues())
.any(|issue| {
matches!(
issue,
JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(
JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing
))
)
});

if has_test_cases_without_matching_codeowners_paths {
println!(
" {} - CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.",
print_validation_level(JunitValidationLevel::SubOptimal)
);
}
}
None => println!(
Expand Down
2 changes: 1 addition & 1 deletion context-js/tests/parse_and_validate.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ describe("context-js", () => {
expect(junitReportValidation.num_suboptimal_issues()).toBe(1);
expect(
junitReportValidation
.all_issues_owned()
.all_issues_flat()
.filter((issue) => issue.error_type === JunitValidationType.Report),
).toHaveLength(1);
});
Expand Down
2 changes: 1 addition & 1 deletion context-py/tests/test_parse_and_validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ def test_junit_validate_suboptimal():
len(
[
x
for x in junit_report_validation.all_issues_owned()
for x in junit_report_validation.all_issues_flat()
if x.error_type == JunitValidationType.Report
]
)
Expand Down
95 changes: 65 additions & 30 deletions context/src/junit/validator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,15 +208,52 @@ pub fn validate(report: &Report) -> JunitReportValidation {
report_validation
}

#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(eq, get_all))]
#[cfg_attr(feature = "wasm", wasm_bindgen(getter_with_clone))]
#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(eq))]
#[cfg_attr(feature = "wasm", wasm_bindgen)]
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct JunitReportValidation {
all_issues: Vec<JunitReportValidationFlatIssue>,
all_issues: Vec<JunitValidationIssueType>,
level: JunitValidationLevel,
test_suites: Vec<JunitTestSuiteValidation>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum JunitValidationIssueType {
Report(JunitReportValidationIssue),
TestSuite(JunitTestSuiteValidationIssue),
TestCase(JunitTestCaseValidationIssue),
}

impl ToString for JunitValidationIssueType {
fn to_string(&self) -> String {
match self {
JunitValidationIssueType::Report(i) => i.to_string(),
JunitValidationIssueType::TestSuite(i) => i.to_string(),
JunitValidationIssueType::TestCase(i) => i.to_string(),
}
}
}

impl From<&JunitValidationIssueType> for JunitValidationType {
fn from(value: &JunitValidationIssueType) -> Self {
match value {
JunitValidationIssueType::Report(..) => JunitValidationType::Report,
JunitValidationIssueType::TestSuite(..) => JunitValidationType::TestSuite,
JunitValidationIssueType::TestCase(..) => JunitValidationType::TestCase,
}
}
}

impl From<&JunitValidationIssueType> for JunitValidationLevel {
fn from(value: &JunitValidationIssueType) -> Self {
match value {
JunitValidationIssueType::Report(i) => JunitValidationLevel::from(i),
JunitValidationIssueType::TestSuite(i) => JunitValidationLevel::from(i),
JunitValidationIssueType::TestCase(i) => JunitValidationLevel::from(i),
}
}
}

#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))]
#[cfg_attr(feature = "wasm", wasm_bindgen(getter_with_clone))]
#[derive(Debug, Clone, Default, PartialEq, Eq)]
Expand All @@ -229,8 +266,15 @@ pub struct JunitReportValidationFlatIssue {
#[cfg_attr(feature = "pyo3", gen_stub_pymethods, pymethods)]
#[cfg_attr(feature = "wasm", wasm_bindgen)]
impl JunitReportValidation {
pub fn all_issues_owned(&self) -> Vec<JunitReportValidationFlatIssue> {
self.all_issues.clone()
pub fn all_issues_flat(&self) -> Vec<JunitReportValidationFlatIssue> {
self.all_issues
.iter()
.map(|i| JunitReportValidationFlatIssue {
level: JunitValidationLevel::from(i),
error_type: JunitValidationType::from(i),
error_message: i.to_string(),
})
.collect()
}

pub fn test_suites_owned(&self) -> Vec<JunitTestSuiteValidation> {
Expand All @@ -255,28 +299,28 @@ impl JunitReportValidation {
pub fn num_invalid_issues(&self) -> usize {
self.all_issues
.iter()
.filter(|issue| issue.level == JunitValidationLevel::Invalid)
.filter(|issue| JunitValidationLevel::from(*issue) == JunitValidationLevel::Invalid)
.count()
}

pub fn num_suboptimal_issues(&self) -> usize {
self.all_issues
.iter()
.filter(|issue| issue.level == JunitValidationLevel::SubOptimal)
.filter(|issue| JunitValidationLevel::from(*issue) == JunitValidationLevel::SubOptimal)
.count()
}
}

impl JunitReportValidation {
pub fn all_issues(&self) -> &[JunitReportValidationFlatIssue] {
pub fn all_issues(&self) -> &[JunitValidationIssueType] {
&self.all_issues
}

pub fn test_suites(&self) -> &[JunitTestSuiteValidation] {
&self.test_suites
}

pub fn test_cases_flat(&self) -> Vec<&JunitTestCaseValidation> {
pub fn test_cases(&self) -> Vec<&JunitTestCaseValidation> {
self.test_suites
.iter()
.flat_map(|test_suite| test_suite.test_cases())
Expand All @@ -285,15 +329,11 @@ impl JunitReportValidation {

fn derive_all_issues(&mut self) {
let mut report_level_issues: HashSet<JunitReportValidationIssue> = HashSet::new();
let mut other_issues: Vec<JunitReportValidationFlatIssue> = Vec::new();
let mut other_issues: Vec<JunitValidationIssueType> = Vec::new();

for test_suite in &self.test_suites {
for issue in &test_suite.issues {
other_issues.push(JunitReportValidationFlatIssue {
level: JunitValidationLevel::from(issue),
error_type: JunitValidationType::TestSuite,
error_message: issue.to_string(),
});
other_issues.push(JunitValidationIssueType::TestSuite(issue.clone()));
}

for test_case in &test_suite.test_cases {
Expand Down Expand Up @@ -330,11 +370,7 @@ impl JunitReportValidation {
} {
report_level_issues.insert(report_level_issue);
} else {
other_issues.push(JunitReportValidationFlatIssue {
level: JunitValidationLevel::from(issue),
error_type: JunitValidationType::TestCase,
error_message: issue.to_string(),
});
other_issues.push(JunitValidationIssueType::TestCase(issue.clone()));
}
}
}
Expand All @@ -349,18 +385,17 @@ impl JunitReportValidation {
other_issues.extend(
report_level_issues
.iter()
.map(|report_level_issue| JunitReportValidationFlatIssue {
level: JunitValidationLevel::from(report_level_issue),
error_type: JunitValidationType::Report,
error_message: report_level_issue.to_string(),
})
.collect::<Vec<JunitReportValidationFlatIssue>>(),
.map(|issue| JunitValidationIssueType::Report(issue.clone())),
);

other_issues.sort_by(|a, b| match (a.level, b.level) {
(JunitValidationLevel::Invalid, JunitValidationLevel::SubOptimal) => Ordering::Less,
(JunitValidationLevel::SubOptimal, JunitValidationLevel::Invalid) => Ordering::Greater,
_ => a.error_message.cmp(&b.error_message),
other_issues.sort_by(|a, b| {
match (JunitValidationLevel::from(a), JunitValidationLevel::from(b)) {
(JunitValidationLevel::Invalid, JunitValidationLevel::SubOptimal) => Ordering::Less,
(JunitValidationLevel::SubOptimal, JunitValidationLevel::Invalid) => {
Ordering::Greater
}
_ => a.to_string().cmp(&b.to_string()),
}
});

self.all_issues = other_issues;
Expand Down

0 comments on commit c21215e

Please sign in to comment.