|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +"""Test that validates the JUnit XML output from pkl_test.""" |
| 17 | + |
| 18 | +import os |
| 19 | +import subprocess |
| 20 | +import sys |
| 21 | +import tempfile |
| 22 | +import unittest |
| 23 | +import xml.etree.ElementTree as ET |
| 24 | +from functools import cached_property |
| 25 | +from pathlib import Path |
| 26 | + |
| 27 | + |
| 28 | +class JUnitXMLValidationTest(unittest.TestCase): |
| 29 | + @classmethod |
| 30 | + def setUpClass(cls): |
| 31 | + with tempfile.NamedTemporaryFile(suffix=".xml", delete=False) as xml_file: |
| 32 | + cls.xml_file = Path(xml_file.name) |
| 33 | + |
| 34 | + # Set environment and get script path |
| 35 | + env = os.environ | {"XML_OUTPUT_FILE": str(cls.xml_file)} |
| 36 | + script_path = os.environ.get("SAMPLE_XML_GENERATOR_PATH") |
| 37 | + |
| 38 | + if not script_path: |
| 39 | + raise RuntimeError("SAMPLE_XML_GENERATOR_PATH environment variable not set") |
| 40 | + |
| 41 | + # Run the pkl_test with XML output enabled |
| 42 | + try: |
| 43 | + result = subprocess.run( |
| 44 | + [script_path], env=env, capture_output=True, text=True, check=True |
| 45 | + ) |
| 46 | + except subprocess.CalledProcessError as e: |
| 47 | + raise RuntimeError( |
| 48 | + f"""Failed to run pkl_test: {e} |
| 49 | + STDOUT: |
| 50 | + {result.stdout} |
| 51 | + STDERR: |
| 52 | + {result.stderr} |
| 53 | + """ |
| 54 | + ) from e |
| 55 | + |
| 56 | + if not cls.xml_file.exists(): |
| 57 | + raise RuntimeError(f"XML output file was not created at {cls.xml_file}") |
| 58 | + |
| 59 | + @cached_property |
| 60 | + def xml_root(self): |
| 61 | + return ET.parse(self.xml_file).getroot() |
| 62 | + |
| 63 | + def test_xml_file_exists_and_parseable(self): |
| 64 | + self.assertTrue( |
| 65 | + self.xml_file.exists(), f"XML file {self.xml_file} does not exist" |
| 66 | + ) |
| 67 | + |
| 68 | + try: |
| 69 | + self.assertIsNotNone(self.xml_root) |
| 70 | + except ET.ParseError as e: |
| 71 | + self.fail(f"Failed to parse XML: {e}") |
| 72 | + |
| 73 | + def test_root_element_structure(self): |
| 74 | + root = self.xml_root |
| 75 | + |
| 76 | + self.assertEqual( |
| 77 | + root.tag, |
| 78 | + "testsuites", |
| 79 | + f"Root element should be 'testsuites', got '{root.tag}'", |
| 80 | + ) |
| 81 | + |
| 82 | + required_attrs = ["name", "tests", "failures"] |
| 83 | + missing_attrs = [attr for attr in required_attrs if attr not in root.attrib] |
| 84 | + self.assertFalse( |
| 85 | + missing_attrs, f"testsuites missing attributes: {missing_attrs}" |
| 86 | + ) |
| 87 | + |
| 88 | + self.assertEqual(root.attrib["name"], "tests.junit_xml.sample_xml_generator") |
| 89 | + |
| 90 | + def test_testsuite_structure(self): |
| 91 | + testsuites = self.xml_root.findall("testsuite") |
| 92 | + self.assertGreater(len(testsuites), 0, "No testsuite elements found") |
| 93 | + |
| 94 | + required_attrs = ["name", "tests", "failures"] |
| 95 | + for i, testsuite in enumerate(testsuites): |
| 96 | + missing_attrs = [ |
| 97 | + attr for attr in required_attrs if attr not in testsuite.attrib |
| 98 | + ] |
| 99 | + self.assertFalse( |
| 100 | + missing_attrs, f"testsuite {i} missing attributes: {missing_attrs}" |
| 101 | + ) |
| 102 | + |
| 103 | + def test_testcase_structure(self): |
| 104 | + testcases = self.xml_root.findall(".//testcase") |
| 105 | + self.assertGreater(len(testcases), 0, "No testcase elements found") |
| 106 | + |
| 107 | + required_attrs = ["name", "classname"] |
| 108 | + for i, testcase in enumerate(testcases): |
| 109 | + missing_attrs = [ |
| 110 | + attr for attr in required_attrs if attr not in testcase.attrib |
| 111 | + ] |
| 112 | + self.assertFalse( |
| 113 | + missing_attrs, f"testcase {i} missing attributes: {missing_attrs}" |
| 114 | + ) |
| 115 | + |
| 116 | + def test_expected_test_cases(self): |
| 117 | + testcases = self.xml_root.findall(".//testcase") |
| 118 | + testcase_names = {tc.attrib["name"] for tc in testcases} |
| 119 | + expected_names = {"dummy test line item 1", "dummy test line item 2"} |
| 120 | + missing_names = expected_names - testcase_names |
| 121 | + self.assertFalse( |
| 122 | + missing_names, f"Expected test cases not found: {missing_names}" |
| 123 | + ) |
| 124 | + |
| 125 | + def test_xml_declaration(self): |
| 126 | + content = self.xml_file.read_text() |
| 127 | + self.assertIn("<?xml version", content, "XML declaration not found") |
| 128 | + |
| 129 | + def test_suite_name_matches_target(self): |
| 130 | + expected_name = "tests.junit_xml.sample_xml_generator" |
| 131 | + self.assertEqual( |
| 132 | + self.xml_root.attrib["name"], |
| 133 | + expected_name, |
| 134 | + "Root testsuites name should match target path", |
| 135 | + ) |
| 136 | + |
| 137 | + |
| 138 | +if __name__ == "__main__": |
| 139 | + unittest.main() |
0 commit comments