diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index 12c9b6ebb7..7d4e8eaba7 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -99,6 +99,7 @@ def do_post_build(self, thebuilder): """).strip()) return 0 + error_messages = [] # MU_CHANGE- Check for invalid tests for test in testList: # Configure output name if test uses cmocka. shell_env.set_shell_var( @@ -121,7 +122,20 @@ def do_post_build(self, thebuilder): for xml_result_file in xml_results_list: root = xml.etree.ElementTree.parse( xml_result_file).getroot() + # MU_CHANGE [BEGIN] - Check for invalid tests + if len(root) == 0: + error_messages.append(f'{os.path.basename(test)} did not generate a test suite(s).') + error_messages.append(' Review source code to ensure Test suites are created and tests ' + ' are registered!') + failure_count += 1 for suite in root: + if len(suite) == 0: + error_messages.append(f'TestSuite [{suite.attrib["name"]}] for test {test} did not ' + 'contain a test case(s).') + error_messages.append(' Review source code to ensure test cases are registered to ' + 'the suite!') + failure_count += 1 + # MU_CHANGE [END] - Check for invalid tests for case in suite: for result in case: if result.tag == 'failure': @@ -131,7 +145,7 @@ def do_post_build(self, thebuilder): " %s - %s" % (case.attrib['name'], result.text)) failure_count += 1 - if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE": + if thebuilder.env.GetValue("CODE_COVERAGE", "FALSE") == "TRUE": # MU_CHANGE if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": ret = self.gen_code_coverage_gcc(thebuilder) if ret != 0: @@ -152,6 +166,10 @@ def do_post_build(self, thebuilder): return -1 # MU_CHANGE end - reformat coverage data + # MU_CHANGE [BEGIN] - Check for invalid tests + for error in error_messages: + logging.error(error) + # MU_CHANGE [END] - Check for invalid tests return failure_count def gen_code_coverage_gcc(self, thebuilder): diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index e8bcc2e27b..131fae0713 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1489,6 +1489,18 @@ GTEST_OUTPUT=xml: This mode is used by the test running plugin to aggregate the results for CI test status reporting in the web view. +### XML Reporting Test Consolidation + +There exists multiple tools for consolidating and generating consolidated test results from the +test xml files that are generated. The arguably most convenient tool available is the +`xunit-viewer` node package, installed via `npm install -g xunit-viewer`. This tool can +consolidate all generated xml reports and create an html or cli summary of test results. + +The following command will generate a consolidated report at `.html` and +also print summary overview of the test results to the command line: + +`xunit-viewer --results Build --output .html --console` + ### Code Coverage Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report @@ -1496,10 +1508,17 @@ per package tested, and combined cobertura report for all packages tested. The p present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present at `Build/coverage.xml` -Code coverage generation has two config knobs: +Code coverage generation has three config knobs. Each can be turned on/off by setting it to TRUE +or FALSE e.g. `CC_REORGANIZE=TRUE`: + +1. `CC_REORGANIZE`: Controls if code coverage results are re-formatted into a "by-inf" folder + structure rather than the default "by-test" folder structure. Default: `TRUE` +1. `CC_FULL`: Generates zero'd out coverage data for untested source files in the package. + Default: `FALSE` +1. `CC_FLATTEN`: Groups all source files together, rather than by INF. Default: `FALSE` -1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package. -2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF. +** NOTE: `CC_FULL` and `CC_FLATTEN` values only matter if `CC_REORGANIZE=TRUE`, as they only +effect how the coverage report is reorganized. **TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files that are consumed by multiple INFs. diff --git a/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml b/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml index 4e0229bef5..9a888ddf16 100644 --- a/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml +++ b/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml @@ -115,7 +115,8 @@ "cobertura", # tool for code coverage "pycobertura", # tool for code coverage "loongarch", - "loongson" + "loongson", + "xunit", ], "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)