diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 595215507..0c27561ef 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -5,43 +5,6 @@ on: BENCHPARK_CODECOV_TOKEN: required: true jobs: - check_errors: - runs-on: ubuntu-24.04 - steps: - - name: Checkout Benchpark - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - name: Add needed Python libs - run: | - pip install -r ./requirements.txt - - name: Run negative tests via helper - shell: bash - run: | - fail() { echo "TEST FAILED: $*" >&2; exit 1; } - - # one-time setup - ./bin/benchpark system init --dest dane llnl-cluster - - # 1) Non-MPIOnlyExperiment Test - set +e - # babelstream is not an MPIOnlyExperiment - stderr_output="$({ ./bin/benchpark experiment init --dest babelstream --system dane babelstream; } 2>&1)" - status=$? - set -e - [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status" - # Check for the specific BenchparkError message - expected="cannot run with MPI only" - grep -Fq "$expected" <<< "$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output" - - # 2) Multiple scaling options - set +e - # Cannot use both +strong and +weak - stderr_output="$({ ./bin/benchpark experiment init --dest kripke --system dane kripke+strong+weak; } 2>&1)" - status=$? - set -e - [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status" - # Check for the specific BenchparkError message - expected="spec cannot specify multiple scaling options" - grep -Fq "$expected" <<<"$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output" saxpy: runs-on: ubuntu-24.04 steps: diff --git a/experiments/babelstream/experiment.py b/experiments/babelstream/experiment.py index 44766d36c..590dfef13 100644 --- a/experiments/babelstream/experiment.py +++ b/experiments/babelstream/experiment.py @@ -6,17 +6,13 @@ from benchpark.directives import variant, maintainers from benchpark.experiment import Experiment from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment -from benchpark.rocm import ROCmExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.models import ModelsType, Models class Babelstream( Experiment, Caliper, - CudaExperiment, - ROCmExperiment, - OpenMPExperiment, + Models(ModelsType.Openmp, ModelsType.Cuda, ModelsType.Rocm), ): variant( "workload", diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index a482cb95f..0a2e41a0d 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -5,20 +5,14 @@ from benchpark.directives import variant, maintainers from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.cuda import CudaExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.models import ModelsType, Models from benchpark.scaling import ScalingMode, Scaling from benchpark.caliper import Caliper class Kripke( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + Models(ModelsType.Mpionly, ModelsType.Openmp, ModelsType.Cuda, ModelsType.Rocm), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/lib/benchpark/cuda.py b/lib/benchpark/cuda.py deleted file mode 100644 index 97434db4c..000000000 --- a/lib/benchpark/cuda.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class CudaExperiment: - requires("cuda", when="+cuda") - variant("cuda", default=False, description="Build and run with CUDA") - - def __init__(self): - super().__init__() - if self.spec.variants["cuda"][0]: - self.device_type = "gpu" - self.programming_models.append("cuda") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "cuda" if self.spec.satisfies("+cuda") else "" - - def get_spack_variants(self): - return ( - "+cuda cuda_arch={cuda_arch}" - if self.spec.satisfies("+cuda") - else "~cuda" - ) diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index 56c07e924..b76424c08 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -256,30 +256,39 @@ def __init__(self, spec): self.package_specs = {} + # Set available programming models for checks + models = set() + for cls in self.__class__.mro(): + models.update(getattr(cls, "_available_programming_models", ())) + self.programming_models = list(models) + # Explicitly ordered list. "mpi" first models = ["mpi"] + ["openmp", "cuda", "rocm"] + valid_models = [] invalid_models = [] for model in models: - # Experiment specifying model in add_package_spec that it doesn't implement - if ( - self.spec.satisfies("+" + model) - and model not in self.programming_models - ): - invalid_models.append(model) + if self.spec.satisfies("+" + model): + valid_models.append(model) + # Experiment specifying model in add_package_spec that it doesn't implement + if model not in self.programming_models: + invalid_models.append(model) + # MPI is always valid if with another programming model, even if no mpionly + if "mpi" in invalid_models and len(valid_models) > 1: + invalid_models.remove("mpi") # Case where there are no experiments specified in experiment.py if len(self.programming_models) == 0: - raise NotImplementedError( + raise BenchparkError( f"Please specify a programming model in your {self.name}/experiment.py (e.g. MpiOnlyExperiment, OpenMPExperiment, CudaExperiment, ROCmExperiment). See other experiments for examples." ) elif len(invalid_models) > 0: - raise NotImplementedError( + raise BenchparkError( f'{invalid_models} are not valid programming models for "{self.name}". Choose from {self.programming_models}.' ) # Check if experiment is trying to run in MpiOnly mode without being an MpiOnlyExperiment elif "mpi" not in str(self.spec) and not any( self.spec.satisfies("+" + model) for model in models[1:] ): - raise NotImplementedError( + raise BenchparkError( f'"{self.name}" cannot run with MPI only without inheriting from MpiOnlyExperiment. Choose from {self.programming_models}' ) @@ -456,7 +465,10 @@ def compute_applications_section_wrapper(self): for cls in self.helpers: helper_prefix = cls.get_helper_name_prefix() if helper_prefix: - expr_helper_list.append(helper_prefix) + if isinstance(helper_prefix, list): + expr_helper_list.extend(helper_prefix) + else: + expr_helper_list.append(helper_prefix) expr_name_suffix = "_".join(expr_helper_list + self.expr_var_names) self.check_required_variables() diff --git a/lib/benchpark/models.py b/lib/benchpark/models.py new file mode 100644 index 000000000..6b50324eb --- /dev/null +++ b/lib/benchpark/models.py @@ -0,0 +1,85 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + + +from benchpark.error import BenchparkError +from benchpark.directives import variant, requires +from benchpark.experiment import ExperimentHelper +from enum import Enum + + +class ModelsType(Enum): + Mpionly = "mpi" + Openmp = "openmp" + Cuda = "cuda" + Rocm = "rocm" + + +def Models(*types): + for ty in types: + if not isinstance(ty, ModelsType): + raise ValueError(f"Invalid programming model: {ty}") + + # Normalize once so we can reuse + _available = tuple(t.value for t in types) + + class BaseModel: + requires("mpi", when="+mpi") + requires("rocm", when="+rocm") + requires("cuda", when="+cuda") + requires("openmp", when="+openmp") + + variant("mpi", default=True, description="Run with MPI") + variant("rocm", default=False, description="Build and run with ROCm") + variant("cuda", default=False, description="Build and run with CUDA") + variant("openmp", default=False, description="Build and run with OpenMP") + + # Class-level list of supported models for any class that includes this mixin + _available_programming_models = _available + + # Handy instance-level property (works on Experiment instances) + @property + def available_programming_models(self): + # If multiple mixins contribute, merge them from MRO at runtime + models = set() + for cls in type(self).mro(): + models.update(getattr(cls, "_available_programming_models", ())) + return tuple(sorted(models)) + + # Quick check helper + @staticmethod + def supports_model(name: str) -> bool: + return name in _available + + # Helper class (unchanged except for optional new method) + class Helper(ExperimentHelper): + def get_helper_name_prefix(self): + models = [] + for s in [ + ModelsType.Mpionly.value, + ModelsType.Openmp.value, + ModelsType.Cuda.value, + ModelsType.Rocm.value, + ]: + if self.spec.satisfies("+" + s): + models.append(s) + if len(models) > 0: + return models + return "no_model" + + # Optional: expose *available* (not selected) models via helper, too + def get_available_models(self): + models = set() + for cls in type(self).__mro__: + models.update(getattr(cls, "_available_programming_models", ())) + return tuple(sorted(models)) + + return type( + "ProgrammingModel", + (BaseModel,), + { + "Helper": Helper, + }, + ) diff --git a/lib/benchpark/mpi.py b/lib/benchpark/mpi.py deleted file mode 100644 index b9273c5ad..000000000 --- a/lib/benchpark/mpi.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class MpiOnlyExperiment: - requires("mpi") - variant("mpi", default=True, description="Run with MPI only") - - def __init__(self): - super().__init__() - if self.spec.variants["mpi"][0]: - self.device_type = "cpu" - self.programming_models.append("mpi") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "mpi" if self.spec.satisfies("+mpi") else "" diff --git a/lib/benchpark/openmp.py b/lib/benchpark/openmp.py deleted file mode 100644 index 00c40355a..000000000 --- a/lib/benchpark/openmp.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class OpenMPExperiment: - requires("openmp", when="+openmp") - variant("openmp", default=False, description="Build and run with OpenMP") - - def __init__(self): - super().__init__() - if self.spec.variants["openmp"][0]: - self.device_type = "cpu" - self.programming_models.append("openmp") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "openmp" if self.spec.satisfies("+openmp") else "" - - def get_spack_variants(self): - return "+openmp" if self.spec.satisfies("+openmp") else "~openmp" diff --git a/lib/benchpark/rocm.py b/lib/benchpark/rocm.py deleted file mode 100644 index b0c60d887..000000000 --- a/lib/benchpark/rocm.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class ROCmExperiment: - requires("rocm", when="+rocm") - variant("rocm", default=False, description="Build and run with ROCm") - - def __init__(self): - super().__init__() - if self.spec.variants["rocm"][0]: - self.device_type = "gpu" - self.programming_models.append("rocm") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "rocm" if self.spec.satisfies("+rocm") else "" - - def get_spack_variants(self): - return ( - "+rocm amdgpu_target={rocm_arch}" - if self.spec.satisfies("+rocm") - else "~rocm" - ) diff --git a/lib/benchpark/test/experiment_errors.py b/lib/benchpark/test/experiment_errors.py new file mode 100644 index 000000000..743803adc --- /dev/null +++ b/lib/benchpark/test/experiment_errors.py @@ -0,0 +1,28 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +import benchpark.spec +from benchpark.error import BenchparkError + + +def test_programming_model_checks(): + # babelstream mpi-only not valid + with pytest.raises(BenchparkError, match=r"mpi.*are not valid programming models"): + spec = benchpark.spec.ExperimentSpec("babelstream").concretize() + experiment = spec.experiment # noqa: F841 + + # stream+openmp not valid + with pytest.raises(Exception, match="not a valid variant"): + spec = benchpark.spec.ExperimentSpec( + "stream+openmp workload=stream" + ).concretize() + experiment = spec.experiment + + # Multiple scaling options not valid + with pytest.raises(BenchparkError, match="cannot specify multiple scaling options"): + spec = benchpark.spec.ExperimentSpec("kripke+strong+weak").concretize() + experiment = spec.experiment