Support junit xml output for pkl test

MODULE.bazel

@@ -30,7 +30,6 @@ bazel_dep(name = "rules_shell", version = "0.4.0")
 
 bazel_dep(name = "buildifier_prebuilt", version = "7.3.1", dev_dependency = True)
 bazel_dep(name = "gazelle", version = "0.44.0", dev_dependency = True, repo_name = "bazel_gazelle")
-bazel_dep(name = "rules_bazel_integration_test", version = "0.29.0", dev_dependency = True)
 bazel_dep(name = "rules_python", version = "1.4.1", dev_dependency = True)
 bazel_dep(name = "stardoc", version = "0.8.0", dev_dependency = True)
 
@@ -99,6 +98,10 @@ python.toolchain(
     python_version = "3.12",
 )
 
+# moved here due to python toolchain resolution order issue.
+# rules_bazel_integration_test needs to be declared after python.toolchain
+bazel_dep(name = "rules_bazel_integration_test", version = "0.29.0", dev_dependency = True)
+
 bazel_binaries = use_extension(
     "@rules_bazel_integration_test//:extensions.bzl",
     "bazel_binaries",

pkl/private/pkl.bzl

@@ -82,6 +82,11 @@ def _prepare_pkl_script(ctx, is_test_target):
     ctx.actions.write(output = symlinks_json_file, content = json.encode(path_to_symlink_target))
     pkl_symlink_tool = pkl_toolchain.symlink_tool[DefaultInfo].files_to_run.executable
 
+    suite_name_parts = []
+    if is_test_target:
+        package_parts = [part for part in ctx.label.package.split("/") if part != ""]
+        suite_name_parts = package_parts + [ctx.label.name]
+
     # The 'args' lists for 'pkl_eval' and 'pkl_test' differ because for `pkl_eval`, files are passed as file targets to enable
     # path stripping on the `ctx.Args` object when using the '--experimental_output_path=strip' flag. Currently, test rules
     # don't support using the `ctx.Args` object, which will be addressed by the following upstream PR
@@ -95,6 +100,7 @@ def _prepare_pkl_script(ctx, is_test_target):
         ctx.attr.multiple_outputs,
         working_dir,
         cache_root_path if len(caches) else "",
+        ".".join(suite_name_parts) if is_test_target else "",
     ]
 
     for k, v in ctx.attr.properties.items():

pkl/private/run_pkl_script.sh

@@ -34,7 +34,8 @@ entrypoints=$7
 multiple_outputs=$8
 working_dir=$9
 cache_dir=${10}
-shift 10
+suite_name=${11}
+shift 11
 
 properties_and_expressions=("$@")
 
@@ -53,8 +54,14 @@ if [ "$command" == "eval" ]; then
   else
     output_args=("--output-path" "$expected_output")
   fi
+  test_args=()
 elif [[ "$command" == "test" ]]; then
-    output_args=()
+  output_args=()
+  if [[ -n "${XML_OUTPUT_FILE}" ]]; then
+    test_args=("--junit-reports" "${working_dir}" "--junit-aggregate-reports" "--junit-aggregate-suite-name" "${suite_name}")
+  else
+    test_args=()
+  fi
 else
   echo "invalid command: $command" >&2
   exit 1
@@ -66,9 +73,16 @@ if [[ -n "$cache_dir" ]]; then
   cache_args=("--cache-dir" "../cache")
 fi
 
-output=$($executable "$command" $format_args "${properties_and_expressions[@]}" $expression_args --working-dir "${working_dir}" "${cache_args[@]}" "${output_args[@]}" $entrypoints)
+output=$($executable "$command" $format_args "${properties_and_expressions[@]}" $expression_args --working-dir "${working_dir}" "${cache_args[@]}" "${test_args[@]}" "${output_args[@]}" $entrypoints)
 
 ret=$?
+
+if [[ "$command" == "test" ]]; then
+  if [ -f "${working_dir}/${suite_name}.xml" ]; then
+    mv "${working_dir}/${suite_name}.xml" "${XML_OUTPUT_FILE}"
+  fi
+fi
+
 if [[ $ret != 0 ]]; then
   if [ "$command" == eval ]; then
     echo "Failed processing PKL configuration with entrypoint(s) '$entrypoints' (PWD: $(pwd)):" >&2

tests/junit_xml/BUILD.bazel

@@ -0,0 +1,32 @@
+# Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_pkl//pkl:defs.bzl", "pkl_test")
+load("@rules_python//python:defs.bzl", "py_test")
+
+# Sample pkl test for XML generation
+pkl_test(
+    name = "sample_xml_generator",
+    srcs = ["sample_xml_test.pkl"],
+)
+
+# Python unittest that runs pkl_test with XML output and validates the XML content
+py_test(
+    name = "junit_xml_validation_test",
+    srcs = ["junit_xml_validation_test.py"],
+    data = [":sample_xml_generator"],
+    env = {
+        "SAMPLE_XML_GENERATOR_PATH": "$(location :sample_xml_generator)",
+    },
+)

tests/junit_xml/junit_xml_validation_test.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test that validates the JUnit XML output from pkl_test."""
+
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+import xml.etree.ElementTree as ET
+from functools import cached_property
+from pathlib import Path
+
+
+class JUnitXMLValidationTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        with tempfile.NamedTemporaryFile(suffix=".xml", delete=False) as xml_file:
+            cls.xml_file = Path(xml_file.name)
+
+        # Set environment and get script path
+        env = os.environ | {"XML_OUTPUT_FILE": str(cls.xml_file)}
+        script_path = os.environ.get("SAMPLE_XML_GENERATOR_PATH")
+
+        if not script_path:
+            raise RuntimeError("SAMPLE_XML_GENERATOR_PATH environment variable not set")
+
+        # Run the pkl_test with XML output enabled
+        try:
+            result = subprocess.run(
+                [script_path], env=env, capture_output=True, text=True, check=True
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(
+                f"""Failed to run pkl_test: {e}
+                STDOUT:
+                {result.stdout}
+                STDERR:
+                {result.stderr}
+                """
+            ) from e
+
+        if not cls.xml_file.exists():
+            raise RuntimeError(f"XML output file was not created at {cls.xml_file}")
+
+    @cached_property
+    def xml_root(self):
+        return ET.parse(self.xml_file).getroot()
+
+    def test_xml_file_exists_and_parseable(self):
+        self.assertTrue(
+            self.xml_file.exists(), f"XML file {self.xml_file} does not exist"
+        )
+
+        try:
+            self.assertIsNotNone(self.xml_root)
+        except ET.ParseError as e:
+            self.fail(f"Failed to parse XML: {e}")
+
+    def test_root_element_structure(self):
+        root = self.xml_root
+
+        self.assertEqual(
+            root.tag,
+            "testsuites",
+            f"Root element should be 'testsuites', got '{root.tag}'",
+        )
+
+        required_attrs = ["name", "tests", "failures"]
+        missing_attrs = [attr for attr in required_attrs if attr not in root.attrib]
+        self.assertFalse(
+            missing_attrs, f"testsuites missing attributes: {missing_attrs}"
+        )
+
+        self.assertEqual(root.attrib["name"], "tests.junit_xml.sample_xml_generator")
+
+    def test_testsuite_structure(self):
+        testsuites = self.xml_root.findall("testsuite")
+        self.assertGreater(len(testsuites), 0, "No testsuite elements found")
+
+        required_attrs = ["name", "tests", "failures"]
+        for i, testsuite in enumerate(testsuites):
+            missing_attrs = [
+                attr for attr in required_attrs if attr not in testsuite.attrib
+            ]
+            self.assertFalse(
+                missing_attrs, f"testsuite {i} missing attributes: {missing_attrs}"
+            )
+
+    def test_testcase_structure(self):
+        testcases = self.xml_root.findall(".//testcase")
+        self.assertGreater(len(testcases), 0, "No testcase elements found")
+
+        required_attrs = ["name", "classname"]
+        for i, testcase in enumerate(testcases):
+            missing_attrs = [
+                attr for attr in required_attrs if attr not in testcase.attrib
+            ]
+            self.assertFalse(
+                missing_attrs, f"testcase {i} missing attributes: {missing_attrs}"
+            )
+
+    def test_expected_test_cases(self):
+        testcases = self.xml_root.findall(".//testcase")
+        testcase_names = {tc.attrib["name"] for tc in testcases}
+        expected_names = {"dummy test line item 1", "dummy test line item 2"}
+        missing_names = expected_names - testcase_names
+        self.assertFalse(
+            missing_names, f"Expected test cases not found: {missing_names}"
+        )
+
+    def test_xml_declaration(self):
+        content = self.xml_file.read_text()
+        self.assertIn("<?xml version", content, "XML declaration not found")
+
+    def test_suite_name_matches_target(self):
+        expected_name = "tests.junit_xml.sample_xml_generator"
+        self.assertEqual(
+            self.xml_root.attrib["name"],
+            expected_name,
+            "Root testsuites name should match target path",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/junit_xml/sample_xml_test.pkl

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+// Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+amends "pkl:test"
+
+facts {
+  ["dummy test line item 1"] {
+    true
+  }
+  ["dummy test line item 2"] {
+    true
+  }
+}