diff --git a/experiments/remhos/experiment.py b/experiments/remhos/experiment.py index f2cb34b13..19f56c012 100644 --- a/experiments/remhos/experiment.py +++ b/experiments/remhos/experiment.py @@ -31,10 +31,17 @@ class Remhos( variant( "version", default="gpu-opt", - values=("develop", "latest", "gpu-fom", "gpu-opt", "1.0"), + values=("develop", "latest", "1.0"), description="app version", ) + variant( + "gpu-aware-mpi", + default=False, + values=(True, False), + description="Use GPU-aware MPI", + ) + maintainers("rfhaque") def compute_applications_section(self): @@ -77,8 +84,18 @@ def compute_applications_section(self): if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): self.add_experiment_variable("n_gpus", "{n_resources}", True) + if self.spec.satisfies("+gpu-aware-mpi"): + self.add_experiment_variable("gam", "--gpu-aware-mpi") + else: + self.add_experiment_variable("gam", "--no-gpu-aware-mpi") else: self.add_experiment_variable("n_ranks", "{n_resources}", True) def compute_package_section(self): - self.add_package_spec(self.name, [f"remhos{self.determine_version()} +metis"]) + gam = "~gpu-aware-mpi" + if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): + if self.spec.satisfies("+gpu-aware-mpi"): + gam = "+gpu-aware-mpi" + self.add_package_spec( + self.name, [f"remhos{self.determine_version()} +metis {gam}"] + ) diff --git a/repo/mfem/package.py b/repo/mfem/package.py index 084f2e106..98e7110aa 100644 --- a/repo/mfem/package.py +++ b/repo/mfem/package.py @@ -38,7 +38,19 @@ def get_make_config_options(self, spec, prefix): def yes_no(varstr): return "YES" if varstr in self.spec else "NO" - options = super().get_make_config_options(spec, prefix) + options = [x for x in super().get_make_config_options(spec, prefix) if not x.startswith("UMPIRE_LIB=")] + + if "+umpire" in spec: + umpire = spec["umpire"] + umpire_libs = umpire.libs + if "^camp" in umpire: + umpire_libs += umpire["camp"].libs + if "^fmt" in umpire: + umpire_libs += umpire["fmt"].libs + options += [ + "UMPIRE_LIB=%s" % self.ld_flags_from_library_list(umpire_libs), + ] + options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper")) if "+caliper" in self.spec: options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix) diff --git a/repo/remhos/application.py b/repo/remhos/application.py index a4912cfaa..54cad0f42 100644 --- a/repo/remhos/application.py +++ b/repo/remhos/application.py @@ -20,8 +20,30 @@ class Remhos(ExecutableApplication): 'network-latency-bound','network-collectives','unstructured-grid'] #TODO: add -ms flag once it's implemented - executable('2d', 'remhos'+' -dim 2 -epm {epm}'+' -p 14'+' -rs {rs2d}'+' -o 3 -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True) - executable('3d', 'remhos'+' -dim 3 -epm {epm}'+' -p 10'+' -rs {rs3d}'+' -o 2'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True) + executable('2d', 'remhos' + + ' -dim 2 -epm {epm}' + + ' -p 14' + + ' -rs {rs2d}' + + ' -o 3 -dt {dt} -tf {tf}' + + ' -ho {ho} -lo {lo}' + + ' -fct {fct}' + + ' -vs {vs}' + + ' -ms {ms}' + + ' {gam}' + + ' -d {device}' + + ' -pa -no-vis', use_mpi=True) + executable('3d', 'remhos' + + ' -dim 3 -epm {epm}' + + ' -p 10' + + ' -rs {rs3d}' + + ' -o 2 -dt {dt} -tf {tf}' + + ' -ho {ho} -lo {lo}' + + ' -fct {fct}' + + ' -vs {vs}' + + ' -ms {ms}' + + ' {gam}' + + ' -d {device}' + + ' -pa -no-vis', use_mpi=True) workload('2d', executables=['2d']) workload('3d', executables=['3d']) @@ -76,8 +98,13 @@ class Remhos(ExecutableApplication): workload_variable('ms', default='5', description='ms', workloads=['2d','3d']) + workload_variable('device', default='cpu', - description='cpu or cuda', + description='cpu, cuda or hip', + workloads=['2d','3d']) + + workload_variable('gam', default='--no-gpu-aware-mpi', + description='--gpu-aware-mpi or --no-gpu-aware-mpi', workloads=['2d','3d']) figure_of_merit("FOM", log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'FOM:\s+(?P[0-9]*\.[0-9]*)', group_name='fom', units='megadofs x time steps / second') diff --git a/repo/remhos/package.py b/repo/remhos/package.py index 462c22046..84eb36ca0 100644 --- a/repo/remhos/package.py +++ b/repo/remhos/package.py @@ -24,12 +24,11 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage): license("BSD-2-Clause") version("develop", branch="master") - version("gpu-fom", branch="gpu-fom") - version("gpu-opt", branch="gpu-opt") version("1.0", sha256="e60464a867fe5b1fd694fbb37bb51773723427f071c0ae26852a2804c08bbb32") variant("metis", default=True, description="Enable/disable METIS support") variant("caliper", default=False, description="Enable/disable Caliper support") + variant("gpu-aware-mpi", default=False, description="Enable GPU aware MPI") depends_on("c", type="build") depends_on("cxx", type="build") @@ -43,8 +42,6 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage): depends_on("zlib+optimize+pic~shared") depends_on("mfem@develop", when="@develop") depends_on("mfem@4.1.0:", when="@1.0") - depends_on("mfem@develop", when="@gpu-fom") - depends_on("mfem@develop", when="@gpu-opt") depends_on("mfem+caliper", when="+caliper") depends_on("mfem cxxstd=17") @@ -62,14 +59,22 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage): depends_on(f"mfem cuda_arch={arch}", when=f"cuda_arch={arch}") depends_on("mfem +cuda+mpi", when="+cuda") depends_on("mfem +rocm+mpi", when="+rocm") + depends_on("mfem +umpire", when="+cuda") + depends_on("mfem +umpire", when="+rocm") - depends_on("hypre +rocm +mpi", when="+rocm") + depends_on("hypre +rocm+mpi", when="+rocm") requires("+rocm", when="^hypre+rocm") for target in ("none", "gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx942"): depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}") depends_on(f"mfem amdgpu_target={target}", when=f"amdgpu_target={target}") - depends_on("hypre+gpu-aware-mpi", when="^cray-mpich+gtl") + depends_on("hypre+umpire", when="+cuda") + depends_on("hypre+umpire", when="+rocm") + depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi") + + def setup_run_environment(self, env): + if "+gpu-aware-mpi" in self.spec: + env.set("MFEM_GPU_AWARE_MPI", "1") def setup_build_environment(self, env): if "+cuda" in self.spec: