[caliper] rename the caliper marker file into __caliper_test_metadata__.yaml

Author: kpouget

projects/caliper/cli/main.py

@@ -99,7 +99,7 @@ def parse_mlflow_url(url: str) -> dict[str, str | None]:
 
 _ARTIFACTS_DIR_HELP = (
     "Root directory of the test artifact tree (directories containing "
-    "__test_labels__.yaml). Optional manifest files (e.g. caliper.yaml) are searched here "
+    "__caliper_test_metadata__.yaml). Optional manifest files (e.g. caliper.yaml) are searched here "
     "unless --postprocess-config is set."
 )
 _PLUGIN_MODULE_HELP = (

projects/caliper/engine/constants.py

@@ -0,0 +1,12 @@
+"""Caliper engine constants - central location for shared values."""
+
+# Test metadata file markers
+METADATA_FILE = "__caliper_test_metadata__.yaml"
+LEGACY_METADATA_FILE = "__test_labels__.yaml"
+
+# MatrixBenchmarking compatibility
+MATRIXBENCHMARKING_SETTINGS_FILE = "settings.yaml"
+
+# MLflow artifact files
+METRICS_FILE = "metrics.json"
+PARAMETERS_FILE = "parameters.json"

projects/caliper/engine/file_export/artifacts_export_run.py

@@ -14,6 +14,7 @@
 import yaml
 from click.core import Context, ParameterSource
 
+from projects.caliper.engine.constants import LEGACY_METADATA_FILE, METADATA_FILE
 from projects.caliper.engine.file_export.mlflow_config import (
     project_metadata_fields,
     validate_mlflow_config,
@@ -310,12 +311,20 @@ def run_artifacts_export(
 
 
 def discover_run_dirs(from_path: Path) -> list[Path]:
-    """Auto-detect test run directories via ``__test_labels__.yaml`` markers."""
-    run_dirs: list[Path] = []
-    for marker in sorted(from_path.rglob("__test_labels__.yaml")):
+    """Auto-detect test run directories via metadata markers (with backwards compatibility)."""
+    # Collect directories with either metadata file (new format or legacy)
+    metadata_dirs = set()
+
+    for marker in from_path.rglob(METADATA_FILE):
         if marker.is_file():
-            run_dirs.append(marker.parent)
-    return run_dirs
+            metadata_dirs.add(marker.parent)
+
+    # Look for legacy format (for directories that don't have new format)
+    for marker in from_path.rglob(LEGACY_METADATA_FILE):
+        if marker.is_file() and marker.parent not in metadata_dirs:
+            metadata_dirs.add(marker.parent)
+
+    return sorted(metadata_dirs)
 
 
 def run_multi_run_artifacts_export(

projects/caliper/engine/file_export/mlflow_backend.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Any
 
+from projects.caliper.engine.constants import LEGACY_METADATA_FILE, METADATA_FILE
 from projects.caliper.engine.file_export.mlflow_secrets import (
     assert_tracking_uri_has_no_userinfo,
     mlflow_connection_env,
@@ -506,14 +507,22 @@ def _log_curve_metrics(metrics_curve: dict[str, Any]) -> None:
 
 
 def _log_metrics_and_params_from_tree(artifact_root: Path) -> None:
-    """Find metrics.json/parameters.json under __test_labels__.yaml-marked dirs and log them."""
+    """Find metrics.json/parameters.json under metadata-marked dirs and log them (with backwards compatibility)."""
     import mlflow
 
-    for marker in sorted(artifact_root.rglob("__test_labels__.yaml")):
-        if not marker.is_file():
-            continue
-        run_dir = marker.parent
+    # Collect directories with either metadata file (new format or legacy)
+    metadata_dirs = set()
+
+    for marker in artifact_root.rglob(METADATA_FILE):
+        if marker.is_file():
+            metadata_dirs.add(marker.parent)
+
+    # Look for legacy format (for directories that don't have new format)
+    for marker in artifact_root.rglob(LEGACY_METADATA_FILE):
+        if marker.is_file() and marker.parent not in metadata_dirs:
+            metadata_dirs.add(marker.parent)
 
+    for run_dir in sorted(metadata_dirs):
         mf = run_dir / "metrics.json"
         if mf.is_file():
             for k, v in _load_json_file(mf).items():

projects/caliper/engine/kpi/kpis_to_mlflow.py

@@ -17,27 +17,46 @@
 from pathlib import Path
 from typing import Any
 
+from projects.caliper.engine.constants import (
+    LEGACY_METADATA_FILE,
+    METADATA_FILE,
+    METRICS_FILE,
+    PARAMETERS_FILE,
+)
 from projects.caliper.engine.kpi.dataclasses import HierarchicalKpiFormat
 
 logger = logging.getLogger(__name__)
 
-METRICS_FILE = "metrics.json"
-PARAMETERS_FILE = "parameters.json"
-TEST_LABELS_MARKER = "__test_labels__.yaml"
+# Metadata file markers (new format preferred, legacy for backwards compatibility)
+METADATA_MARKER = METADATA_FILE
+LEGACY_METADATA_MARKER = LEGACY_METADATA_FILE
 
 
 def _build_run_dir_index(artifact_tree: Path) -> dict[str, Path]:
-    """Map run directory names to their paths using __test_labels__.yaml markers."""
+    """Map run directory names to their paths using metadata markers (with backwards compatibility)."""
     index: dict[str, Path] = {}
-    for marker in sorted(artifact_tree.rglob(TEST_LABELS_MARKER)):
+
+    # Collect directories with either metadata file (new format or legacy)
+    metadata_dirs = set()
+
+    for marker in artifact_tree.rglob(METADATA_MARKER):
         if marker.is_file():
-            run_dir = marker.parent
-            try:
-                rel = run_dir.relative_to(artifact_tree)
-            except ValueError:
-                rel = Path(run_dir.name)
-            index[str(rel)] = run_dir
-            index[run_dir.name] = run_dir
+            metadata_dirs.add(marker.parent)
+
+    # Look for legacy format (for directories that don't have new format)
+    for marker in artifact_tree.rglob(LEGACY_METADATA_MARKER):
+        if marker.is_file() and marker.parent not in metadata_dirs:
+            metadata_dirs.add(marker.parent)
+
+    # Build index from collected directories
+    for run_dir in sorted(metadata_dirs):
+        try:
+            rel = run_dir.relative_to(artifact_tree)
+        except ValueError:
+            rel = Path(run_dir.name)
+        index[str(rel)] = run_dir
+        index[run_dir.name] = run_dir
+
     return index
 
 

projects/caliper/engine/model.py

@@ -10,7 +10,7 @@
 
 @dataclass
 class TestBaseNode:
-    """Directory containing __test_labels__.yaml or MatrixBenchmarking settings.yaml."""
+    """Directory containing __caliper_test_metadata__.yaml or MatrixBenchmarking settings.yaml."""
 
     directory: Path
     test_labels: dict[str, Any]
@@ -173,7 +173,7 @@ def get_ai_data_artifact_files_for_test(self, test_dir: Path) -> list[str]:
         not across the entire base directory. This provides better security isolation.
 
         Args:
-            test_dir: The specific test directory to search within (where __test_labels__.yaml is located)
+            test_dir: The specific test directory to search within (where test metadata file is located)
 
         Returns:
             List of relative file paths from test_dir to copy for AI evaluation

projects/caliper/engine/traverse.py

@@ -1,4 +1,4 @@
-"""Discover test base directories via __test_labels__.yaml or MatrixBenchmarking settings.yaml."""
+"""Discover test base directories via __caliper_test_metadata__.yaml or MatrixBenchmarking settings.yaml."""
 
 from __future__ import annotations
 
@@ -8,10 +8,18 @@
 
 import yaml
 
+from projects.caliper.engine.constants import (
+    LEGACY_METADATA_FILE,
+    MATRIXBENCHMARKING_SETTINGS_FILE,
+    METADATA_FILE,
+)
 from projects.caliper.engine.model import TestBaseNode
 
-MARKER = "__test_labels__.yaml"
-MATRIXBENCHMARKING_MARKER = "settings.yaml"
+# Primary marker
+MARKER = METADATA_FILE
+# Legacy marker for backwards compatibility
+LEGACY_MARKER = LEGACY_METADATA_FILE
+MATRIXBENCHMARKING_MARKER = MATRIXBENCHMARKING_SETTINGS_FILE
 
 
 def discover_test_bases(
@@ -40,6 +48,8 @@ def discover_test_bases(
         marker_found = None
         if MARKER in filenames:
             marker_found = MARKER
+        elif LEGACY_MARKER in filenames:
+            marker_found = LEGACY_MARKER  # Backwards compatibility
         elif MATRIXBENCHMARKING_MARKER in filenames:
             marker_found = MATRIXBENCHMARKING_MARKER
 
@@ -49,7 +59,7 @@ def discover_test_bases(
         path = Path(dirpath)
 
         # Use hierarchical label loading for both marker types
-        if marker_found == MARKER:
+        if marker_found == MARKER or marker_found == LEGACY_MARKER:
             test_labels = _load_hierarchical_test_labels(path, base_dir)
             # For filtering, use the labels directly (hierarchical loading returns the labels dict)
             # Normalize missing "labels" entry to empty mapping to allow discovery of empty marker files
@@ -144,15 +154,16 @@ def _matches_any_local(labels_dict: dict, key: str, filter_values: list[str]) ->
 
 
 def _load_hierarchical_test_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]:
-    """Load and merge __test_labels__.yaml files hierarchically from base_dir down to test_dir.
+    """Load and merge test metadata files hierarchically from base_dir down to test_dir.
 
-    Merges in order:
-    1. base_dir/__test_labels__.*.yaml (all variants)
-    2. parent_dir/__test_labels__.*.yaml (all variants)
-    3. test_dir/__test_labels__.*.yaml (all variants)
-    4. test_dir/__test_labels__.yaml (final, cannot be overridden)
+    Merges in order (for each directory):
+    1. __caliper_test_metadata__.*.yaml (all variants, new format)
+    2. __test_labels__.*.yaml (all variants, legacy format)
+    3. __caliper_test_metadata__.yaml (final, new format)
+    4. __test_labels__.yaml (final, legacy fallback)
 
-    Later files override earlier ones, with the main __test_labels__.yaml having final priority.
+    Later files override earlier ones, with the main metadata file having final priority.
+    New format files are preferred over legacy format when both exist.
     """
     import glob
 
@@ -172,14 +183,19 @@ def _load_hierarchical_test_labels(test_dir: Path, base_dir: Path) -> dict[str,
         # test_dir is not under base_dir, just use test_dir
         path_parts = [test_dir_abs]
 
-    # For each directory in the hierarchy, merge __test_labels__.*.yaml files (excluding plain __test_labels__.yaml)
+    # For each directory in the hierarchy, merge variant files (new format first, then legacy)
     for dir_path in path_parts:
         if not dir_path.is_dir():
             continue
 
-        # Find all __test_labels__.*.yaml files (but not __test_labels__.yaml itself)
-        pattern = str(dir_path / "__test_labels__.*.yaml")
-        variant_files = sorted(glob.glob(pattern))
+        # Find all variant files - new format first
+        new_pattern = str(dir_path / f"{METADATA_FILE.replace('.yaml', '.*.yaml')}")
+        legacy_pattern = str(dir_path / f"{LEGACY_METADATA_FILE.replace('.yaml', '.*.yaml')}")
+
+        # Collect all variant files, prioritizing new format
+        variant_files = []
+        variant_files.extend(sorted(glob.glob(new_pattern)))
+        variant_files.extend(sorted(glob.glob(legacy_pattern)))
 
         for variant_file in variant_files:
             variant_path = Path(variant_file)
@@ -192,8 +208,12 @@ def _load_hierarchical_test_labels(test_dir: Path, base_dir: Path) -> dict[str,
                     # Skip files that can't be loaded
                     pass
 
-    # Finally, load the main __test_labels__.yaml from the test directory (final priority)
+    # Finally, load the main metadata file from the test directory (final priority)
+    # Prefer new format, fall back to legacy format
     main_labels_path = test_dir / MARKER
+    if not main_labels_path.is_file():
+        main_labels_path = test_dir / LEGACY_MARKER
+
     if main_labels_path.is_file():
         try:
             main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=False)

projects/caliper/orchestration/postprocess.py

@@ -18,6 +18,7 @@
 
 from pydantic import ValidationError
 
+from projects.caliper.engine.constants import LEGACY_METADATA_FILE, METADATA_FILE
 from projects.caliper.orchestration.caliper_invocation import (
     _execute_caliper_command,
     _generate_automatic_status_file_path,
@@ -287,29 +288,40 @@ def _run_artifacts_to_ai_data(
 
 
 def _load_test_labels(test_dir: Path) -> dict[str, Any]:
-    """Load test labels from __test_labels__.yaml file if it exists.
+    """Load test labels from metadata file (new format preferred, legacy fallback).
 
     Args:
-        test_dir: Directory to search for __test_labels__.yaml
+        test_dir: Directory to search for metadata files
 
     Returns:
-        Dictionary containing test labels, or empty dict if file doesn't exist
+        Dictionary containing test labels, or empty dict if no file exists
     """
     import yaml
 
-    test_labels_file = test_dir / "__test_labels__.yaml"
-    if test_labels_file.exists():
+    # Try new format first
+    metadata_file = test_dir / METADATA_FILE
+    if metadata_file.exists():
         try:
-            with open(test_labels_file, encoding="utf-8") as f:
+            with open(metadata_file, encoding="utf-8") as f:
                 labels = yaml.safe_load(f)
-                logger.debug(f"Loaded test labels from {test_labels_file}: {labels}")
+                logger.debug(f"Loaded test metadata from {metadata_file}: {labels}")
                 return labels or {}
         except Exception as e:
-            logger.warning(f"Failed to load test labels from {test_labels_file}: {e}")
-            return {}
-    else:
-        logger.debug(f"No test labels file found at {test_labels_file}")
-        return {}
+            logger.error(f"Failed to load test metadata from {metadata_file}: {e}")
+
+    # Fallback to legacy format
+    legacy_file = test_dir / LEGACY_METADATA_FILE
+    if legacy_file.exists():
+        try:
+            with open(legacy_file, encoding="utf-8") as f:
+                labels = yaml.safe_load(f)
+                logger.debug(f"Loaded test labels from legacy file {legacy_file}: {labels}")
+                return labels or {}
+        except Exception as e:
+            logger.error(f"Failed to load test labels from {legacy_file}: {e}")
+
+    logger.debug(f"No metadata or test labels file found in {test_dir}")
+    return {}
 
 
 def _run_kpis_to_csv(

projects/caliper/tests/test_multi_run_export.py

@@ -2,7 +2,7 @@
 Tests for the multi-run caliper export pipeline.
 
 Covers:
-- Run directory auto-detection via __test_labels__.yaml markers
+- Run directory auto-detection via __caliper_test_metadata__.yaml markers
 - Shared vs run-specific file partitioning
 - metrics.json / parameters.json reading and MLflow logging
 - Parent + nested child run creation
@@ -20,15 +20,15 @@
 import pytest
 import yaml
 
+from projects.caliper.engine.constants import METADATA_FILE, METRICS_FILE, PARAMETERS_FILE
 from projects.caliper.engine.file_export.artifacts_export_run import discover_run_dirs
 
-METRICS_FILE = "metrics.json"
-PARAMETERS_FILE = "parameters.json"
-TEST_LABELS_MARKER = "__test_labels__.yaml"
+# Use new format for tests
+TEST_LABELS_MARKER = METADATA_FILE
 
 
 def _write_test_labels(directory: Path, labels: dict) -> None:
-    """Helper to write a __test_labels__.yaml marker file."""
+    """Helper to write a test metadata marker file."""
     directory.mkdir(parents=True, exist_ok=True)
     (directory / TEST_LABELS_MARKER).write_text(
         yaml.safe_dump({"version": "1", "labels": labels}, sort_keys=False),