378 feature request make sure that results obtained from different benchmark versions are not compared

src/jade/app/app.py

@@ -34,7 +34,11 @@
 
 
 class JadeApp:
-    def __init__(self, root: PathLike | None = None, skip_init: bool = False):
+    def __init__(
+        self,
+        root: PathLike | None = None,
+        skip_init: bool = False,
+    ):
         if root is None:
             root = os.getcwd()
 
@@ -51,12 +55,18 @@ def __init__(self, root: PathLike | None = None, skip_init: bool = False):
         # parse the post-processing config
         self.pp_cfg = PostProcessConfig(self.tree.cfg.bench_pp)
 
-        # Compute the global status
-        logging.info("Initializing the global status")
-        self.status = GlobalStatus(
-            simulations_path=self.tree.simulations,
-            raw_results_path=self.tree.raw,
-        )
+        self._status = None
+
+    @property
+    def status(self) -> GlobalStatus:
+        """Lazy-load the global status on first access."""
+        if self._status is None:
+            logging.info("Initializing the global status (lazy-loaded)")
+            self._status = GlobalStatus(
+                simulations_path=self.tree.simulations,
+                raw_results_path=self.tree.raw,
+            )
+        return self._status
 
     def initialize_log(self) -> None:
         """Initialize the custom python logger for JADE."""
@@ -304,6 +314,9 @@ def post_process(self):
                 )
                 continue
 
+            # check that the benchmark versions are consistent with each other
+            self.status.validate_codelibs(code_libs, benchmark)
+
             # prepare the new paths
             pp_path = self.tree.get_new_post_bench_path(benchmark)
             excel_folder = Path(pp_path, "excel")

src/jade/config/status.py

@@ -12,6 +12,7 @@
     print_code_lib,
 )
 from jade.helper.constants import CODE
+from jade.helper.errors import VersionInconsistencyError
 
 
 class GlobalStatus:
@@ -36,12 +37,39 @@ def __init__(self, simulations_path: PathLike, raw_results_path: PathLike):
         """
         self.simulations_path = simulations_path
         self.raw_results_path = raw_results_path
-        self.update()
-
-    def update(self):
-        """Update the status of the simulations and raw results"""
-        self.simulations = self._parse_simulations_folder(self.simulations_path)
-        self.raw_data = self._parse_raw_results_folder(self.raw_results_path)
+        self._simulations = None
+        self._raw_data = None
+        self._raw_metadata = None
+
+    @property
+    def simulations(self) -> dict[tuple[CODE, str, str], CodeLibRunStatus]:
+        if self._simulations is None:
+            self._simulations = self._parse_simulations_folder(self.simulations_path)
+        return self._simulations
+
+    @property
+    def raw_data(self) -> dict[tuple[CODE, str, str], list[str]]:
+        if self._raw_data is None:
+            self._raw_data, self._raw_metadata = self._parse_raw_results_folder(
+                self.raw_results_path
+            )
+        return self._raw_data
+
+    @property
+    def raw_metadata(self) -> dict[tuple[CODE, str, str], dict]:
+        if self._raw_metadata is None:
+            self._raw_data, self._raw_metadata = self._parse_raw_results_folder(
+                self.raw_results_path
+            )
+        return self._raw_metadata
+
+    def update_raw_results(self) -> None:
+        """Update the raw results by re-parsing the raw results folder. It should be used
+        after processing new raw results to update the status.
+        """
+        self._raw_data, self._raw_metadata = self._parse_raw_results_folder(
+            self.raw_results_path
+        )
 
     def _parse_simulations_folder(
         self, simulations_path: PathLike
@@ -65,7 +93,9 @@ def _parse_simulations_folder(
                 for sub_bench in os.listdir(bench_path):
                     # check if the run was successful
                     sub_bench_path = Path(bench_path, sub_bench)
-                    success = CODE_CHECKERS[code](sub_bench_path)
+                    success = CODE_CHECKERS[code].check_success(
+                        os.listdir(sub_bench_path)
+                    )
                     if success:
                         successful.append(sub_bench)
                     else:
@@ -93,9 +123,12 @@ def _parse_simulations_folder(
 
     def _parse_raw_results_folder(
         self, path_raw: PathLike
-    ) -> dict[tuple[CODE, str, str], list[str]]:
+    ) -> tuple[
+        dict[tuple[CODE, str, str], list[str]], dict[tuple[CODE, str, str], dict]
+    ]:
         # simply store a dictionary with the processed raw results
         available_raw_data = {}
+        metadata = {}
         for code_lib in os.listdir(path_raw):
             codelib_path = Path(path_raw, code_lib)
             if not codelib_path.is_dir():
@@ -108,7 +141,10 @@ def _parse_raw_results_folder(
                 available_raw_data[(CODE(code), lib, benchmark)] = os.listdir(
                     bench_path
                 )
-        return available_raw_data
+                if lib != "exp":  # for the experiments we don't have metadata
+                    with open(os.path.join(bench_path, "metadata.json")) as infile:
+                        metadata[(CODE(code), lib, benchmark)] = json.load(infile)
+        return available_raw_data, metadata
 
     def was_simulated(self, code: CODE, lib: str, benchmark: str) -> bool:
         """Check if a simulation was already performed and if it was successful.
@@ -230,6 +266,63 @@ def is_raw_available(self, codelib: str, benchmark: str) -> bool:
             return True
         return False
 
+    def _validate_libs_processing(
+        self, code: CODE, benchmark: str, libs: list[str]
+    ) -> None:
+        """Check that the post-processing can be performed for the given code and
+        benchmark. This is true if the benchmark version for the requested libs
+        are the same.
+
+        Parameters
+        ----------
+        code : CODE
+            code used in the simulation.
+        benchmark : str
+            benchmark name.
+        libs : list[str]
+            list of libraries to check.
+
+        Returns
+        -------
+        bool
+            True if the post-processing can be performed, False otherwise.
+        """
+        versions = {}
+        for lib in libs:
+            # I want only the major version
+            versions[lib] = self.raw_metadata[(code, lib, benchmark)][
+                "benchmark_version"
+            ].split(".")[0]
+        if len(set(versions.values())) > 1:
+            raise VersionInconsistencyError(
+                f"The versions of the requested libraries for {code} and benchmark {benchmark} are not consistent: {versions}"
+            )
+
+    def validate_codelibs(
+        self, codelibs: list[tuple[CODE, str]], benchmarks: str
+    ) -> None:
+        """Check that a list of codelibs can be post-processed together for a given benchmark.
+        This is true if the benchmark version for the requested libs are the same.
+
+        Parameters
+        ----------
+        codelibs : list[tuple[CODE, str]]
+            list of codelibs to check. Each codelib is a tuple with the code and library.
+        benchmark : str
+            benchmark name.
+        """
+        libs = {}
+        for code, lib in codelibs:
+            code = CODE(code)
+            if lib == "exp":  # for the experiments we don't have metadata
+                continue
+            if code not in libs:
+                libs[code] = []
+            libs[code].append(lib)
+
+        for code, lib_list in libs.items():
+            self._validate_libs_processing(code, benchmarks, lib_list)
+
 
 @dataclass
 class CodeLibRunStatus:

src/jade/helper/aux_functions.py

@@ -3,6 +3,7 @@
 import importlib.metadata
 import os
 import re
+from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Union
 
@@ -68,32 +69,61 @@ def print_code_lib(code: CODE, lib: Library | str, pretty: bool = False) -> str:
         return f"_{code.value}_-_{lib_name}_"
 
 
-def check_run_mcnp(folder: PathLike) -> bool:
-    """check if mcnp run was successful"""
-    try:
-        MCNPSimOutput.retrieve_files(folder)
-        return True
-    except FileNotFoundError:
-        return False
+class SimulationChecker(ABC):
+    """Abstract base class for simulation success checkers.
 
+    All code-specific checkers must inherit from this class and implement
+    the check_success method with the same signature.
+    """
 
-def check_run_openmc(folder: PathLike) -> bool:
-    """check if openmc run was successful"""
-    try:
-        OpenMCSimOutput.retrieve_file(folder)
-        return True
-    except FileNotFoundError:
-        return False
+    @abstractmethod
+    def check_success(self, files: list[str]) -> bool:
+        """Check if a simulation run was successful.
+
+        Parameters
+        ----------
+        files : list[str]
+            List of output files to check for success.
+
+        Returns
+        -------
+        bool
+            True if the simulation completed successfully, False otherwise.
+        """
+        pass
+
+
+class MCNPChecker(SimulationChecker):
+    """Checker for MCNP simulations."""
+
+    def check_success(self, files: list[str]) -> bool:
+        """Check if MCNP run was successful by verifying output files exist."""
+        return MCNPSimOutput.is_successfully_simulated(files)
+
+
+class OpenMCChecker(SimulationChecker):
+    """Checker for OpenMC simulations."""
+
+    def check_success(self, files: list[str]) -> bool:
+        """Check if OpenMC run was successful by verifying output files exist."""
+        return OpenMCSimOutput.is_successfully_simulated(files)
+
+
+class SerpentChecker(SimulationChecker):
+    """Checker for Serpent simulations."""
 
+    def check_success(self, files: list[str]) -> bool:
+        """Check if Serpent run was successful."""
+        # TODO implement the logic to check if the Serpent run was successful
+        raise NotImplementedError("Serpent checker not yet implemented")
 
-def check_run_serpent(folder: PathLike) -> bool:
-    # TODO implement the logic to check if the Serpent run was successful
-    raise NotImplementedError
 
+class D1SChecker(SimulationChecker):
+    """Checker for D1S simulations."""
 
-def check_run_d1s(folder: PathLike) -> bool:
-    """check if d1s run was successful"""
-    return check_run_mcnp(folder)
+    def check_success(self, files: list[str]) -> bool:
+        """Check if D1S run was successful (uses same logic as MCNP)."""
+        return MCNPChecker().check_success(files)
 
 
 def get_jade_version() -> str:
@@ -124,11 +154,13 @@ def add_rmode0(path: PathLike) -> None:
                             f.write("RMODE 0\n")
 
 
-CODE_CHECKERS = {
-    CODE.MCNP: check_run_mcnp,
-    CODE.OPENMC: check_run_openmc,
-    CODE.SERPENT: check_run_serpent,
-    CODE.D1S: check_run_d1s,
+# Dictionary mapping CODE enums to checker instances
+# All checkers implement the SimulationChecker interface
+CODE_CHECKERS: dict[CODE, SimulationChecker] = {
+    CODE.MCNP: MCNPChecker(),
+    CODE.OPENMC: OpenMCChecker(),
+    CODE.SERPENT: SerpentChecker(),
+    CODE.D1S: D1SChecker(),
 }
 
 

src/jade/helper/errors.py

@@ -6,6 +6,14 @@ def __init__(self, message="There is an error in the configuration"):
         super().__init__(self.message)
 
 
+class VersionInconsistencyError(ConfigError):
+    """Exception raised for version inconsistencies in the configuration."""
+
+    def __init__(self, message="There is a version inconsistency in the sims"):
+        self.message = message
+        super().__init__(self.message)
+
+
 class PostProcessConfigError(Exception):
     """Exception raised for errors in the configuration of post processing."""
 

src/jade/post/sim_output.py

@@ -262,6 +262,18 @@ def retrieve_files(results_path: PathLike) -> tuple[Path, Path, Path | None]:
 
         return mctal, outp, meshtal
 
+    @staticmethod
+    def is_successfully_simulated(files: list[str]) -> bool:
+        """Check if the simulation was successful by verifying output files exist."""
+        mctal_found = False
+        output_found = False
+        for file in files:
+            if file.endswith(".m"):
+                mctal_found = True
+            elif file.endswith(".o"):
+                output_found = True
+        return mctal_found and output_found
+
 
 class OpenMCSimOutput(AbstractSimOutput):
     def __init__(
@@ -333,6 +345,18 @@ def retrieve_file(
 
         return file1, file2, file3
 
+    @staticmethod
+    def is_successfully_simulated(files: list[str]) -> bool:
+        """Check if the simulation was successful by verifying output files exist."""
+        statepoint_found = False
+        output_found = False
+        for file in files:
+            if file.startswith("statepoint") and file.endswith(".h5"):
+                statepoint_found = True
+            elif file.endswith(".out"):
+                output_found = True
+        return statepoint_found and output_found
+
     def _create_dataframes(
         self, tallies: dict
     ) -> tuple[dict[int, pd.DataFrame], dict[int, pd.DataFrame]]:

src/jade/run/benchmark.py

@@ -440,7 +440,7 @@ def _get_continue_run_command(self, code: CODE, lib: Library) -> str:
         for single_run_folder in os.listdir(benchmark_root):
             single_run_root = Path(benchmark_root, single_run_folder)
             # check if the simulation has been completed
-            flag_run = CODE_CHECKERS[code](single_run_root)
+            flag_run = CODE_CHECKERS[code].check_success(os.listdir(single_run_root))
             if flag_run:
                 continue