diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index e7c37a1e792..bee3907e2db 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -117,10 +117,23 @@ def do_post_build(self, thebuilder): os.path.basename(test)) file_match_pattern = test + ".*." + arch + ".result.xml" xml_results_list = glob.glob(file_match_pattern) + # MU_CHANGE [BEGIN] - Check for invalid tests + if len(xml_results_list) == 0: + logging.error(f'{os.path.basename(test)} did not generate any test suites.') + logging.error('Review Code to ensure Test suites are created and tests are registered!') + failure_count += 1 + # MU_CHANGE [END] - Check for invalid tests for xml_result_file in xml_results_list: root = xml.etree.ElementTree.parse( xml_result_file).getroot() for suite in root: + # MU_CHANGE [BEGIN] - Check for invalid tests + if len(suite) == 0: + logging.error(f'TestSuite [{suite.attrib["name"]}] for test {test} did ' + 'not contain any test case.') + logging.error('Review Code to ensure test cases are registered to the suite!') + failure_count += 1 + # MU_CHANGE [END] - Check for invalid tests for case in suite: for result in case: if result.tag == 'failure': @@ -130,7 +143,7 @@ def do_post_build(self, thebuilder): " %s - %s" % (case.attrib['name'], result.text)) failure_count += 1 - if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE": + if thebuilder.env.GetValue("CODE_COVERAGE", "FALSE") == "TRUE": # MU_CHANGE if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": ret = self.gen_code_coverage_gcc(thebuilder) if ret != 0: diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index dd497617a93..4e05f442233 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1491,6 +1491,18 @@ GTEST_OUTPUT=xml: This mode is used by the test running plugin to aggregate the results for CI test status reporting in the web view. +### XML Reporting Test Consolidation + +There exists multiple tools for consolidating and generating consolidated test results from the +test xml files that are generated. The arguably most convenient tool available is the +`xunit-viewer` node package, installed via `npm install -g xunit-viewer`. This tool can +consolidate all generated xml reports and create an html or cli summary of test results. + +The following command will generate a consolidated report at `.html` and +also print summary overview of the test results to the command line: + +`xunit-viewer --results Build --output .html --console` + ### Code Coverage Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report @@ -1498,10 +1510,17 @@ per package tested, and combined cobertura report for all packages tested. The p present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present at `Build/coverage.xml` -Code coverage generation has two config knobs: +Code coverage generation has three config knobs. Each can be turned on/off by setting it to TRUE +or FALSE e.g. `CC_REORGANIZE=TRUE`: + +1. `CC_REORGANIZE`: Controls if code coverage results are re-formatted into a "by-inf" folder + structure rather than the default "by-test" folder structure. Default: `TRUE` +1. `CC_FULL`: Generates zero'd out coverage data for untested source files in the package. + Default: `FALSE` +2. `CC_FLATTEN`: Groups all source files together, rather than by INF. Default: `FALSE` -1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package. -2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF. +** NOTE: `CC_FULL` and `CC_FLATTEN` values only matter if `CC_REORGANIZE=TRUE`, as they only +effect how the coverage report is reorganized. **TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files that are consumed by multiple INFs.