diff --git a/MODULE.bazel b/MODULE.bazel index b1ece2a..510cc0f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -30,7 +30,6 @@ bazel_dep(name = "rules_shell", version = "0.4.0") bazel_dep(name = "buildifier_prebuilt", version = "7.3.1", dev_dependency = True) bazel_dep(name = "gazelle", version = "0.44.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "rules_bazel_integration_test", version = "0.29.0", dev_dependency = True) bazel_dep(name = "rules_python", version = "1.4.1", dev_dependency = True) bazel_dep(name = "stardoc", version = "0.8.0", dev_dependency = True) @@ -99,6 +98,10 @@ python.toolchain( python_version = "3.12", ) +# moved here due to python toolchain resolution order issue. +# rules_bazel_integration_test needs to be declared after python.toolchain +bazel_dep(name = "rules_bazel_integration_test", version = "0.29.0", dev_dependency = True) + bazel_binaries = use_extension( "@rules_bazel_integration_test//:extensions.bzl", "bazel_binaries", diff --git a/pkl/private/pkl.bzl b/pkl/private/pkl.bzl index 0be2bee..8f9af92 100644 --- a/pkl/private/pkl.bzl +++ b/pkl/private/pkl.bzl @@ -82,6 +82,11 @@ def _prepare_pkl_script(ctx, is_test_target): ctx.actions.write(output = symlinks_json_file, content = json.encode(path_to_symlink_target)) pkl_symlink_tool = pkl_toolchain.symlink_tool[DefaultInfo].files_to_run.executable + suite_name_parts = [] + if is_test_target: + package_parts = [part for part in ctx.label.package.split("/") if part != ""] + suite_name_parts = package_parts + [ctx.label.name] + # The 'args' lists for 'pkl_eval' and 'pkl_test' differ because for `pkl_eval`, files are passed as file targets to enable # path stripping on the `ctx.Args` object when using the '--experimental_output_path=strip' flag. Currently, test rules # don't support using the `ctx.Args` object, which will be addressed by the following upstream PR @@ -95,6 +100,7 @@ def _prepare_pkl_script(ctx, is_test_target): ctx.attr.multiple_outputs, working_dir, cache_root_path if len(caches) else "", + ".".join(suite_name_parts) if is_test_target else "", ] for k, v in ctx.attr.properties.items(): diff --git a/pkl/private/run_pkl_script.sh b/pkl/private/run_pkl_script.sh index 334a66a..8b8d5cf 100755 --- a/pkl/private/run_pkl_script.sh +++ b/pkl/private/run_pkl_script.sh @@ -34,7 +34,8 @@ entrypoints=$7 multiple_outputs=$8 working_dir=$9 cache_dir=${10} -shift 10 +suite_name=${11} +shift 11 properties_and_expressions=("$@") @@ -53,8 +54,14 @@ if [ "$command" == "eval" ]; then else output_args=("--output-path" "$expected_output") fi + test_args=() elif [[ "$command" == "test" ]]; then - output_args=() + output_args=() + if [[ -n "${XML_OUTPUT_FILE}" ]]; then + test_args=("--junit-reports" "${working_dir}" "--junit-aggregate-reports" "--junit-aggregate-suite-name" "${suite_name}") + else + test_args=() + fi else echo "invalid command: $command" >&2 exit 1 @@ -66,9 +73,16 @@ if [[ -n "$cache_dir" ]]; then cache_args=("--cache-dir" "../cache") fi -output=$($executable "$command" $format_args "${properties_and_expressions[@]}" $expression_args --working-dir "${working_dir}" "${cache_args[@]}" "${output_args[@]}" $entrypoints) +output=$($executable "$command" $format_args "${properties_and_expressions[@]}" $expression_args --working-dir "${working_dir}" "${cache_args[@]}" "${test_args[@]}" "${output_args[@]}" $entrypoints) ret=$? + +if [[ "$command" == "test" ]]; then + if [ -f "${working_dir}/${suite_name}.xml" ]; then + mv "${working_dir}/${suite_name}.xml" "${XML_OUTPUT_FILE}" + fi +fi + if [[ $ret != 0 ]]; then if [ "$command" == eval ]; then echo "Failed processing PKL configuration with entrypoint(s) '$entrypoints' (PWD: $(pwd)):" >&2 diff --git a/tests/junit_xml/BUILD.bazel b/tests/junit_xml/BUILD.bazel new file mode 100644 index 0000000..c60caa5 --- /dev/null +++ b/tests/junit_xml/BUILD.bazel @@ -0,0 +1,32 @@ +# Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_pkl//pkl:defs.bzl", "pkl_test") +load("@rules_python//python:defs.bzl", "py_test") + +# Sample pkl test for XML generation +pkl_test( + name = "sample_xml_generator", + srcs = ["sample_xml_test.pkl"], +) + +# Python unittest that runs pkl_test with XML output and validates the XML content +py_test( + name = "junit_xml_validation_test", + srcs = ["junit_xml_validation_test.py"], + data = [":sample_xml_generator"], + env = { + "SAMPLE_XML_GENERATOR_PATH": "$(location :sample_xml_generator)", + }, +) diff --git a/tests/junit_xml/junit_xml_validation_test.py b/tests/junit_xml/junit_xml_validation_test.py new file mode 100644 index 0000000..646b61b --- /dev/null +++ b/tests/junit_xml/junit_xml_validation_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test that validates the JUnit XML output from pkl_test.""" + +import os +import subprocess +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET +from functools import cached_property +from pathlib import Path + + +class JUnitXMLValidationTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + with tempfile.NamedTemporaryFile(suffix=".xml", delete=False) as xml_file: + cls.xml_file = Path(xml_file.name) + + # Set environment and get script path + env = os.environ | {"XML_OUTPUT_FILE": str(cls.xml_file)} + script_path = os.environ.get("SAMPLE_XML_GENERATOR_PATH") + + if not script_path: + raise RuntimeError("SAMPLE_XML_GENERATOR_PATH environment variable not set") + + # Run the pkl_test with XML output enabled + try: + result = subprocess.run( + [script_path], env=env, capture_output=True, text=True, check=True + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"""Failed to run pkl_test: {e} + STDOUT: + {result.stdout} + STDERR: + {result.stderr} + """ + ) from e + + if not cls.xml_file.exists(): + raise RuntimeError(f"XML output file was not created at {cls.xml_file}") + + @cached_property + def xml_root(self): + return ET.parse(self.xml_file).getroot() + + def test_xml_file_exists_and_parseable(self): + self.assertTrue( + self.xml_file.exists(), f"XML file {self.xml_file} does not exist" + ) + + try: + self.assertIsNotNone(self.xml_root) + except ET.ParseError as e: + self.fail(f"Failed to parse XML: {e}") + + def test_root_element_structure(self): + root = self.xml_root + + self.assertEqual( + root.tag, + "testsuites", + f"Root element should be 'testsuites', got '{root.tag}'", + ) + + required_attrs = ["name", "tests", "failures"] + missing_attrs = [attr for attr in required_attrs if attr not in root.attrib] + self.assertFalse( + missing_attrs, f"testsuites missing attributes: {missing_attrs}" + ) + + self.assertEqual(root.attrib["name"], "tests.junit_xml.sample_xml_generator") + + def test_testsuite_structure(self): + testsuites = self.xml_root.findall("testsuite") + self.assertGreater(len(testsuites), 0, "No testsuite elements found") + + required_attrs = ["name", "tests", "failures"] + for i, testsuite in enumerate(testsuites): + missing_attrs = [ + attr for attr in required_attrs if attr not in testsuite.attrib + ] + self.assertFalse( + missing_attrs, f"testsuite {i} missing attributes: {missing_attrs}" + ) + + def test_testcase_structure(self): + testcases = self.xml_root.findall(".//testcase") + self.assertGreater(len(testcases), 0, "No testcase elements found") + + required_attrs = ["name", "classname"] + for i, testcase in enumerate(testcases): + missing_attrs = [ + attr for attr in required_attrs if attr not in testcase.attrib + ] + self.assertFalse( + missing_attrs, f"testcase {i} missing attributes: {missing_attrs}" + ) + + def test_expected_test_cases(self): + testcases = self.xml_root.findall(".//testcase") + testcase_names = {tc.attrib["name"] for tc in testcases} + expected_names = {"dummy test line item 1", "dummy test line item 2"} + missing_names = expected_names - testcase_names + self.assertFalse( + missing_names, f"Expected test cases not found: {missing_names}" + ) + + def test_xml_declaration(self): + content = self.xml_file.read_text() + self.assertIn("