diff --git a/.gitlab/tests/nightly.yml b/.gitlab/tests/nightly.yml index b2c179e07..7f270fc55 100644 --- a/.gitlab/tests/nightly.yml +++ b/.gitlab/tests/nightly.yml @@ -49,6 +49,7 @@ include: BENCHMARK: - amg2023 - babelstream + - branson - kripke - laghos - lammps @@ -64,6 +65,7 @@ include: BENCHMARK: - amg2023 - babelstream + - branson - genesis - hpcg - hpl @@ -84,6 +86,7 @@ include: BENCHMARK: - amg2023 - babelstream + - branson - gromacs - kripke - laghos diff --git a/experiments/branson/experiment.py b/experiments/branson/experiment.py new file mode 100644 index 000000000..aace8ca42 --- /dev/null +++ b/experiments/branson/experiment.py @@ -0,0 +1,91 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from benchpark.caliper import Caliper +from benchpark.cuda import CudaExperiment +from benchpark.directives import variant +from benchpark.experiment import Experiment +from benchpark.mpi import MpiOnlyExperiment +from benchpark.openmp import OpenMPExperiment +from benchpark.rocm import ROCmExperiment +from benchpark.scaling import Scaling, ScalingMode + + +class Branson( + Experiment, + MpiOnlyExperiment, + OpenMPExperiment, + CudaExperiment, + ROCmExperiment, + Scaling(ScalingMode.Strong, ScalingMode.Weak), + Caliper, +): + variant( + "workload", + default="branson", + description="workload name", + ) + + variant( + "version", + default="develop", + values=("develop",), + description="app version", + ) + + variant( + "n_groups", + default="30", + values=int, + description="Number of groups", + ) + + def compute_applications_section(self): + if self.spec.satisfies("exec_mode=test"): + self.add_experiment_variable("num_particles", 1000000, True) + else: + self.add_experiment_variable("num_particles", 850000000, True) + self.add_experiment_variable("resource_count", 1, False) + + self.register_scaling_config( + { + ScalingMode.Strong: { + "resource_count": lambda var, itr, dim, scaling_factor: var.val(dim) + * scaling_factor, + }, + ScalingMode.Weak: { + "resource_count": lambda var, itr, dim, scaling_factor: var.val(dim) + * scaling_factor, + "num_particles": lambda var, itr, dim, scaling_factor: var.val(dim) + * scaling_factor, + }, + } + ) + + # Set the variables required by the experiment + self.set_required_variables( + n_resources="{resource_count}", + process_problem_size="{num_particles}", + total_problem_size="{num_particles} * {resource_count}", + ) + + if self.spec.satisfies("+openmp"): + self.add_experiment_variable("n_threads_per_proc", 1, True) + if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): + self.add_experiment_variable("use_gpu", "TRUE") + self.add_experiment_variable("n_gpus", "{n_resources}", True) + else: + self.add_experiment_variable("use_gpu", "FALSE") + self.add_experiment_variable("n_ranks", "{n_resources}", True) + + def compute_package_section(self): + # get package version + app_version = self.spec.variants["version"][0] + self.add_package_spec( + self.name, + [ + f"branson@{app_version} n_groups={self.spec.variants['n_groups'][0]} ", + ], + ) diff --git a/repo/branson/application.py b/repo/branson/application.py index 4abb7b14c..adfa0e351 100644 --- a/repo/branson/application.py +++ b/repo/branson/application.py @@ -19,14 +19,15 @@ class Branson(ExecutableApplication): executable('setup_experiment', template=[ 'cp {branson}/inputs/* {experiment_run_dir}/.', - 'sed -i "s|250000000|{num_particles}|g" {experiment_run_dir}/{input_file}' + 'sed -i "s|[0-9]*|{num_particles}|g" {experiment_run_dir}/{input_file}', + 'sed -i "s|.*|{use_gpu}|g" {experiment_run_dir}/{input_file}' ]) executable('p', '{branson}/bin/BRANSON {experiment_run_dir}/{input_file}', use_mpi=True) workload('branson', executables=['setup_experiment','p']) - workload_variable('input_file', default='3D_hohlraum_multi_node.xml', + workload_variable('input_file', default='3D_hohlraum_single_node.xml', description='input file name', workloads=['branson']) diff --git a/repo/branson/package.py b/repo/branson/package.py index b8610b09a..e3ecd9723 100644 --- a/repo/branson/package.py +++ b/repo/branson/package.py @@ -4,16 +4,14 @@ # SPDX-License-Identifier: Apache-2.0 from spack.package import * -from spack_repo.builtin.packages.boost.package import Boost -class Branson(CMakePackage): +class Branson(CMakePackage, CudaPackage, ROCmPackage): """Branson's purpose is to study different algorithms for parallel Monte Carlo transport. Currently it contains particle passing and mesh passing methods for domain decomposition.""" homepage = "https://github.com/lanl/branson" - url = "https://github.com/lanl/branson/archive/0.82.tar.gz" git = "https://github.com/lanl/branson.git" tags = ["proxy-app"] @@ -34,25 +32,81 @@ class Branson(CMakePackage): depends_on("c", type="build") depends_on("cxx", type="build") - #depends_on("mpi") + variant("openmp", default=False, description="Enable OpenMP support") + variant("caliper", default=False, description="Enable Caliper monitoring") + variant("n_groups", default=30, values=int, description="Number of groups") + depends_on("mpi@2:") - # TODO: replace this with an explicit list of components of Boost, - # for instance depends_on('boost +filesystem') - # See https://github.com/spack/spack/pull/22303 for reference - depends_on(Boost.with_default_variants, when="@:0.81") depends_on("metis") depends_on("parmetis", when="@:0.81") + depends_on("caliper", when="+caliper") + depends_on("adiak", when="+caliper") root_cmakelists_dir = "src" + flag_handler = build_system_flags + + def setup_run_environment(self, env): + if self.compiler.extra_rpaths: + for rpath in self.compiler.extra_rpaths: + env.prepend_path("LD_LIBRARY_PATH", rpath) + + def setup_build_environment(self, env): + if "+cuda" in self.spec: + env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") + def cmake_args(self): spec = self.spec args = [] - #args.append("--enable-mpi") + args.append(f"-DCMAKE_C_COMPILER={spec['mpi'].mpicc}") args.append(f"-DCMAKE_CXX_COMPILER={spec['mpi'].mpicxx}") args.append(f"-DCMAKE_Fortran_COMPILER={spec['mpi'].mpifc}") + args.append(f"-DCMAKE_CXX_STANDARD=17") + args.append(f"-DMPI_C_COMPILER={spec['mpi'].mpicc}") + args.append(f"-DMPI_CXX_COMPILER={spec['mpi'].mpicxx}") + + args.append(f"-DMETIS_ROOT_DIR={spec['metis'].prefix}") + + if '+cuda' in spec: + args.append("-DUSE_CUDA=ON") + args.append("-DUSE_GPU=ON") + args.append(f"-DCMAKE_CUDA_COMPILER={spec['cuda'].prefix}/bin/nvcc") + cuda_arch_vals = spec.variants["cuda_arch"].value + if cuda_arch_vals: + cuda_arch_sorted = list(sorted(cuda_arch_vals, reverse=True)) + cuda_arch = cuda_arch_sorted[0] + args.append(f"-DCUDA_ARCH={cuda_arch}") + else: + args.append("-DUSE_CUDA=OFF") + args.append("-DUSE_GPU=OFF") + + if '+rocm' in spec: + args.append("-DUSE_HIP=ON") + args.append("-DUSE_GPU=ON") + rocm_arch_vals = spec.variants["amdgpu_target"].value + args.append(f"-DROCM_PATH={spec['hip'].prefix}") + if rocm_arch_vals: + rocm_arch_sorted = list(sorted(rocm_arch_vals, reverse=True)) + rocm_arch = rocm_arch_sorted[0] + args.append(f"-DROCM_ARCH={rocm_arch}") + args.append(f"-DHIP_ARCH={rocm_arch}") + else: + args.append("-DUSE_HIP=OFF") + args.append("-DUSE_GPU=OFF") + + args.append(self.define_from_variant("USE_OPENMP", "openmp")) + + if '+caliper' in spec: + args.append(self.define_from_variant("USE_CALIPER", "caliper")) + args.append(f"-Dcaliper_DIR={spec['caliper'].prefix}") + + args.append("-DBUILD_TESTING=OFF") + args.append(f"-DN_GROUPS={self.spec.variants['n_groups'].value}") + + args.append(f"-DMPI_CXX_LINK_FLAGS={spec['mpi'].libs.ld_flags}") + return args def install(self, spec, prefix):