diff --git a/repos/spack_repo/builtin/packages/gasnet/package.py b/repos/spack_repo/builtin/packages/gasnet/package.py index 21d28874df4..28aafce2b16 100644 --- a/repos/spack_repo/builtin/packages/gasnet/package.py +++ b/repos/spack_repo/builtin/packages/gasnet/package.py @@ -5,14 +5,14 @@ import os import warnings +from spack_repo.builtin.build_systems.autotools import AutotoolsPackage from spack_repo.builtin.build_systems.cuda import CudaPackage -from spack_repo.builtin.build_systems.generic import Package from spack_repo.builtin.build_systems.rocm import ROCmPackage from spack.package import * -class Gasnet(Package, CudaPackage, ROCmPackage): +class Gasnet(AutotoolsPackage, CudaPackage, ROCmPackage): """GASNet is a language-independent, networking middleware layer that provides network-independent, high-performance communication primitives including Remote Memory Access (RMA) and Active Messages (AM). It has been @@ -22,18 +22,22 @@ class Gasnet(Package, CudaPackage, ROCmPackage): writers (as opposed to end users), and the primary goals are high performance, interface portability, and expressiveness. - ***NOTICE***: The GASNet library built by this Spack package is ONLY intended for - unit-testing purposes, and is generally UNSUITABLE FOR PRODUCTION USE. - The RECOMMENDED way to build GASNet is as an embedded library as configured - by the higher-level client runtime package (UPC++, Legion, Chapel, etc), including - system-specific configuration. + ***NOTICE***: The Spack built version of GASNet is generally considered + UNSUITABLE FOR PRODUCTION USE by its authors. The RECOMMENDED way to build + GASNet is as an embedded library as configured by the higher-level client + runtime package (UPC++, Chapel, etc), including system-specific configuration. + + Despite this recommendation, this Spack package is a best effort to allow + supporting a range of platforms. For optimal performance and more + fine-grained control on a particular target platform, HPC admins can provide an + external build. """ homepage = "https://gasnet.lbl.gov" url = "https://gasnet.lbl.gov/EX/GASNet-2024.5.0.tar.gz" git = "https://bitbucket.org/berkeleylab/gasnet.git" - maintainers("PHHargrove", "bonachea") + maintainers("PHHargrove", "bonachea", "rbberger") tags = ["e4s", "ecp"] @@ -73,6 +77,49 @@ class Gasnet(Package, CudaPackage, ROCmPackage): ) variant("debug", default=False, description="Enable library debugging mode") + variant("pic", default=True, description="Produce position-independent code (for shared libs)") + + variant("seq", default=True, description="support SEQ-mode single-threaded GASNet clients") + variant("par", default=False, description="support PAR-mode pthreaded GASNet clients") + variant("parsync", default=False, description="support PARSYNC-mode pthreaded GASNet clients") + + variant("pshm", default=True, description="Enable/disable inter-process shared memory support") + variant("pthreads", default=False, description="enable use of pthreads") + variant( + "mpi_compat", default=False, description="Enable/disable MPI compatibility in all conduits" + ) + + variant( + "pmi", default="none", values=("none", "x", "1", "2", "cray"), description="PMI version" + ) + variant( + "segment", + default="off", + values=("off", "fast", "large", "everything"), + description="Build GASNet in the FAST/LARGE/EVERYTHING shared segment configuration", + ) + + with when("conduits=ofi"): + variant( + "ofi_provider", + default="auto", + values=("auto", "cxi"), + description="Statically configure ofi-conduit for the given provider", + ) + variant( + "ofi_spawner", + default="auto", + values=("auto", "ssh", "mpi", "pmi"), + description="ofi job spawner", + ) + + with when("conduits=ibv"): + variant( + "ibv_max_hcas", + default="1", + values=(str(x) for x in range(9)), + description="Maximum number of IBV HCAs to open", + ) variant( "cuda", @@ -99,9 +146,14 @@ class Gasnet(Package, CudaPackage, ROCmPackage): depends_on("cxx", type="build") # generated depends_on("gmake", type="build") + depends_on("mpi", when="+mpi_compat") depends_on("mpi", when="conduits=mpi") + depends_on("libfabric", when="conduits=ofi") + depends_on("pmix", when="pmi=x") + depends_on("cray-pmi", when="pmi=cray") + depends_on("autoconf@2.69", type="build", when=bootstrap_version) depends_on("automake@1.16:", type="build", when=bootstrap_version) @@ -111,8 +163,9 @@ class Gasnet(Package, CudaPackage, ROCmPackage): depends_on("oneapi-level-zero@1.8.0:", when="+level_zero") - def install(self, spec, prefix): - if spec.satisfies(self.bootstrap_version): + @run_before("configure") + def bootstrap(self): + if self.spec.satisfies(self.bootstrap_version): bootstrapsh = Executable("./Bootstrap") bootstrapsh() # Record git-describe when fetched from git: @@ -122,58 +175,89 @@ def install(self, spec, prefix): except ProcessError: warnings.warn("Omitting version stamp due to git error") - # The GASNet-EX library has a highly multi-dimensional configure space, - # to accomodate the varying behavioral requirements of each client runtime. - # The library's ABI/link compatibility is strongly dependent on these - # client-specific build-time settings, and that variability is deliberately NOT - # encoded in the variants of this package. The recommended way to build/deploy - # GASNet is as an EMBEDDED library within the build of the client package - # (eg. Berkeley UPC, UPC++, Legion, etc), some of which provide build-time - # selection of the GASNet library sources. This spack package provides - # the GASNet-EX sources, for use by appropriate client packages. - install_tree(".", prefix + "/src") - - # Library build is provided for unit-testing purposes only (see notice above) - if "conduits=none" not in spec: - options = ["--prefix=%s" % prefix] - - if spec.satisfies("+debug"): - options.append("--enable-debug") - - if spec.satisfies("+cuda"): - options.append("--enable-kind-cuda-uva") - options.append("--with-cuda-home=" + spec["cuda"].prefix) - - if spec.satisfies("+rocm"): - options.append("--enable-kind-hip") - options.append("--with-hip-home=" + spec["hip"].prefix) - - if spec.satisfies("+level_zero"): - options.append("--enable-kind-ze") - options.append("--with-ze-home=" + spec["oneapi-level-zero"].prefix) - - if spec.satisfies("conduits=mpi"): - options.append("--enable-mpi-compat") - else: - options.append("--disable-mpi-compat") - - options.append("--disable-auto-conduit-detect") - for c in spec.variants["conduits"].value: - options.append("--enable-" + c) - - options.append("--enable-rpath") + def configure_args(self): + spec = self.spec + options = ["--disable-auto-conduit-detect", "--enable-rpath"] + options += self.enable_or_disable("debug") + options += self.enable_or_disable("par") + options += self.enable_or_disable("seq") + options += self.enable_or_disable("parsync") + options += self.enable_or_disable("pshm") + options += self.enable_or_disable("pthreads") + options += self.enable_or_disable("mpi-compat", variant="mpi_compat") + + if not spec.satisfies("segment=off"): + options.append(f"--enable-segment-{spec.variants['segment'].value}") + + flags = {"cflags": [], "cxxflags": [], "mpi-cflags": []} + + if spec.satisfies("+pic"): + flags["cflags"].append(self.compiler.cc_pic_flag) + flags["mpi-cflags"].append(self.compiler.cc_pic_flag) + flags["cxxflags"].append(self.compiler.cxx_pic_flag) + + for key, value in sorted(flags.items()): + if value: + options.append(f"--with-{key}={' '.join(value)}") + + if not spec.satisfies("pmi=none"): + options.append("--enable-pmi") + options.append(f"--with-pmi-version={spec.variants['pmi'].value}") + + if spec.satisfies("pmi=x"): + options.append(f"--with-pmi-home={spec['pmix'].prefix}") + elif spec.satisfies("pmi=cray"): + options.append(f"--with-pmi-home={spec['cray-pmi'].prefix}") + else: + options.append("--disable-pmi") + + if spec.satisfies("+cuda"): + options.append("--enable-kind-cuda-uva") + options.append("--with-cuda-home=" + spec["cuda"].prefix) + + if spec.satisfies("+rocm"): + options.append("--enable-kind-hip") + options.append("--with-hip-home=" + spec["hip"].prefix) + + if spec.satisfies("+level_zero"): + options.append("--enable-kind-ze") + options.append("--with-ze-home=" + spec["oneapi-level-zero"].prefix) + + for c in spec.variants["conduits"].value: + options.append("--enable-" + c) + + if spec.satisfies("conduits=mpi") or spec.satisfies("+mpi_compat"): + options.append(f"--with-mpi-cc={spec['mpi'].mpicc}") + + if spec.satisfies("conduits=ibv"): + options.append(f"--with-ibv-max-hcas={spec.variants['ibv_max_hcas'].value}") + + if spec.satisfies("conduits=ofi"): + if not spec.satisfies("ofi_provider=auto"): + options.append(f"--with-ofi-provider={spec.variants['ofi_provider'].value}") + if not spec.satisfies("ofi_spawner=auto"): + options.append(f"--with-ofi-spawner={spec.variants['ofi_spawner'].value}") + return options + + @run_after("build") + def build_tests(self): + if not self.spec.satisfies("conduits=none"): + for c in self.spec.variants["conduits"].value: + make("-C", c + "-conduit", "testgasnet-par") + make("-C", c + "-conduit", "testtools-par") - configure(*options) - make() - make("install") + @run_after("install") + def install_source(self): + install_tree(self.stage.source_path, self.prefix + "/src") - for c in spec.variants["conduits"].value: + @run_after("install") + def install_tests(self): + if not self.spec.satisfies("conduits=none"): + for c in self.spec.variants["conduits"].value: testdir = join_path(self.prefix.tests, c) mkdirp(testdir) - make("-C", c + "-conduit", "testgasnet-par") install(c + "-conduit/testgasnet", testdir) - make("-C", c + "-conduit", "testtools-par") - install(c + "-conduit/testtools", self.prefix.tests) + install(c + "-conduit/testtools", prefix.tests) @run_after("install") @on_package_attributes(run_tests=True) diff --git a/repos/spack_repo/builtin/packages/legion/package.py b/repos/spack_repo/builtin/packages/legion/package.py index 384809a7848..3d18d7a6a39 100644 --- a/repos/spack_repo/builtin/packages/legion/package.py +++ b/repos/spack_repo/builtin/packages/legion/package.py @@ -11,7 +11,7 @@ from spack.package import * -class Legion(CMakePackage, ROCmPackage): +class Legion(CMakePackage, CudaPackage, ROCmPackage): """Legion is a data-centric parallel programming system for writing portable high performance programs targeted at distributed heterogeneous architectures. Legion presents abstractions which allow programmers to @@ -26,12 +26,14 @@ class Legion(CMakePackage, ROCmPackage): tuning of Legion applications to new architectures.""" homepage = "https://legion.stanford.edu/" - git = "https://github.com/StanfordLegion/legion.git" + git = "https://gitlab.com/StanfordLegion/legion.git" license("Apache-2.0") - maintainers("pmccormick", "streichler", "elliottslaughter") + maintainers("pmccormick", "streichler", "elliottslaughter", "rbberger") tags = ["e4s"] + version("25.09.0", tag="legion-25.09.0", commit="8759d840099a138b5f395e86c841848520b34b73") + version("25.06.0", tag="legion-25.06.0", commit="d8e35c48d089014b0f764181b7b90278a7558b21") version("25.03.0", tag="legion-25.03.0", commit="04716e3b3686d4af71e6a4398dfbe8cd869c057b") version("24.12.0", tag="legion-24.12.0", commit="2f087ebe433a19f9a3abd05382f951027933bad9") version("24.09.0", tag="legion-24.09.0", commit="4a03402467547b99530042cfe234ceec2cd31b2e") @@ -52,55 +54,77 @@ class Legion(CMakePackage, ROCmPackage): depends_on("cmake@3.16:", when="@21.03.0:24.12.0", type="build") depends_on("cmake@3.22:", when="@25.03.0:", type="build") + + depends_on("realm", when="@25.09.0:") + depends_on("realm+shared", when="@25.09.0: +shared") + depends_on("realm+kokkos", when="@25.09.0: +kokkos") + depends_on("realm+hdf5", when="@25.09.0: +hdf5") + depends_on("realm+hwloc", when="@25.09.0: +hwloc") + depends_on("realm+openmp", when="@25.09.0: +openmp") + depends_on("realm+cuda", when="@25.09.0: +cuda") + depends_on("realm+rocm", when="@25.09.0: +rocm") + depends_on("realm+cuda_unsupported_compiler", when="@25.09.0: +cuda_unsupported_compiler") + # TODO: Need to spec version of MPI v3 for use of the low-level MPI transport # layer. At present the MPI layer is still experimental and we discourge its # use for general (not legion development) use cases. - depends_on("mpi", when="network=mpi") - depends_on("mpi", when="network=gasnet") # MPI is required to build gasnet (needs mpicc). - depends_on("ucx", when="network=ucx") - depends_on("ucc", when="network=ucx @25.03.0:") - depends_on("ucc+cuda+nccl", when="network=ucx +cuda @25.03.0:") - depends_on("ucc+rocm+rccl", when="network=ucx +rocm @25.03.0:") - depends_on("ucx", when="conduit=ucx") - depends_on("mpi", when="conduit=mpi") + depends_on("mpi", when="@:25.06.0 network=mpi") + depends_on( + "mpi", when="@:25.06.0 network=gasnet" + ) # MPI is required to build gasnet (needs mpicc). + depends_on("ucx", when="@:25.06.0 network=ucx") + depends_on("ucc", when="@25.03.0:25.06.0 network=ucx") + depends_on("ucc+cuda+nccl", when="network=ucx +cuda @25.03.0:25.06.0") + depends_on("ucc+rocm+rccl", when="network=ucx +rocm @25.03.0:25.06.0") + depends_on("ucx", when="conduit=ucx @:25.06.0") + depends_on("mpi", when="conduit=mpi @:25.06.0") depends_on("cuda@10.0:11.9", when="+cuda_unsupported_compiler @21.03.0:23.03.0") depends_on("cuda@10.0:11.9", when="+cuda @21.03.0:23.03.0") - depends_on("cuda@11.7:12.8", when="+cuda_unsupported_compiler @23.06.0:") - depends_on("cuda@11.7:12.8", when="+cuda @23.06.0:") + depends_on("cuda@11.7:12.8", when="+cuda_unsupported_compiler @23.06.0:24.12.0") + depends_on("cuda@11.7:12.8", when="+cuda @23.06.0:24.12.0") depends_on("hip@5.1:5.7", when="+rocm @23.03.0:23.12.0") depends_on("hip@5.1:", when="+rocm") depends_on("hdf5", when="+hdf5") depends_on("hwloc", when="+hwloc") - depends_on("libfabric", when="network=gasnet conduit=ofi-slingshot11") + depends_on("libfabric", when="@:25.06.0 network=gasnet conduit=ofi-slingshot11") + + depends_on( + "gasnet+par~seq~parsync+pthreads segment=fast", when="@25.09.0: ^realm network=gasnet" + ) # Kokkos - depends_on("kokkos", when="+kokkos") + depends_on("kokkos", when="@:25.06.0,stable +kokkos") # OpenMP backend - depends_on("kokkos+openmp", when="+kokkos+openmp") - depends_on("kokkos~openmp", when="+kokkos~openmp") + depends_on("kokkos+openmp", when="@:25.06.0,stable +kokkos+openmp") + depends_on("kokkos~openmp", when="@:25.06.0,stable +kokkos~openmp") # cuda-centric cuda_arch_list = CudaPackage.cuda_arch_values for arch in cuda_arch_list: # UCX transport dependency when using CUDA - depends_on(f"ucc cuda_arch={arch}", when=f"@25.03.0: network=ucx +cuda cuda_arch={arch}") - - # Kokkos CUDA + compiler-specific wrapper + depends_on( + f"ucc cuda_arch={arch}", + when=f"@25.03.0:25.06.0 network=ucx +cuda cuda_arch={arch}", + ) depends_on( f"kokkos+cuda+cuda_lambda+wrapper cuda_arch={arch}", - when=f"+kokkos+cuda cuda_arch={arch} %gcc", + when=f"@:25.06.0 +kokkos+cuda cuda_arch={arch} %gcc", ) depends_on( f"kokkos+cuda+cuda_lambda~wrapper cuda_arch={arch}", - when=f"+kokkos+cuda cuda_arch={arch} %clang", + when=f"@:25.06.0 +kokkos+cuda cuda_arch={arch} %clang", ) + depends_on(f"realm cuda_arch={arch}", when=f"@25.09.0: +cuda cuda_arch={arch}") # https://github.com/spack/spack/issues/37232#issuecomment-1553376552 patch("hip-offload-arch.patch", when="@23.03.0 +rocm") def patch(self): - if self.spec.satisfies("network=gasnet conduit=ofi-slingshot11") and ( + if self.spec.satisfies("@25.09.0:"): + # conflicts with Realm FindGASNet.cmake + force_remove("cmake/FindGASNet.cmake") + if self.spec.satisfies("@:25.06.0 network=gasnet conduit=ofi-slingshot11") and ( self.spec.satisfies("^[virtuals=mpi] cray-mpich+wrappers") or self.spec.satisfies("^[virtuals=mpi] mpich netmod=ofi ^libfabric fabrics=cxi") or self.spec.satisfies("^[virtuals=mpi] openmpi fabrics=ofi ^libfabric fabrics=cxi") @@ -130,11 +154,16 @@ def patch(self): for arch in ROCmPackage.amdgpu_targets: depends_on( - f"ucc amdgpu_target={arch}", when=f"@25.03.0: network=ucx +rocm amdgpu_target={arch}" + f"ucc amdgpu_target={arch}", + when=f"@25.03.0:25.06.0 network=ucx +rocm amdgpu_target={arch}", ) - depends_on(f"kokkos+rocm amdgpu_target={arch}", when=f"+kokkos+rocm amdgpu_target={arch}") + depends_on( + f"kokkos+rocm amdgpu_target={arch}", + when=f"@:25.06.0 +rocm amdgpu_target={arch}", + ) + depends_on(f"realm amdgpu_target={arch}", when=f"@25.09.0: +rocm amdgpu_target={arch}") - depends_on("kokkos+rocm", when="+kokkos+rocm") + depends_on("kokkos+rocm", when="@:25.06.0 +kokkos+rocm") # https://github.com/StanfordLegion/legion/#dependencies depends_on("python@3.8:", when="+python") @@ -143,7 +172,7 @@ def patch(self): depends_on("py-pip", when="+python", type="build") depends_on("py-setuptools", when="+python", type="build") - depends_on("papi", when="+papi") + depends_on("papi", when="@:25.06.0 +papi") depends_on("zlib-api", when="+zlib") # A C++ standard variant to work-around some odd behaviors with apple-clang @@ -164,6 +193,7 @@ def patch(self): values=("gasnet", "mpi", "ucx", "none"), description="The network communications/transport layer to use.", multi=False, + when="@:25.06.0", ) # Add Gasnet tarball dependency in spack managed manner @@ -173,7 +203,7 @@ def patch(self): git="https://github.com/StanfordLegion/gasnet.git", destination="stanfordgasnet", branch="master", - when="network=gasnet", + when="@:25.06.0 network=gasnet", ) # We default to automatically embedding a gasnet build. To override this @@ -191,7 +221,7 @@ def validate_gasnet_root(value): else: return True - with when("network=gasnet"): + with when("@:25.06.0 network=gasnet"): variant( "gasnet_root", default="none", @@ -242,13 +272,6 @@ def validate_gasnet_root(value): default=False, description="Hijack application calls into the CUDA runtime (+cuda).", ) - variant( - "cuda_arch", - default="70", - values=cuda_arch_list, - description="GPU/CUDA architecture to build for.", - multi=False, - ) variant( "cuda_unsupported_compiler", default=False, @@ -277,7 +300,12 @@ def validate_gasnet_root(value): variant("openmp", default=False, description="Enable support for OpenMP within Legion tasks.") - variant("papi", default=False, description="Enable PAPI performance measurements.") + variant( + "papi", + default=False, + description="Enable PAPI performance measurements.", + when="@:25.06.0", + ) variant("python", default=False, description="Enable Python support.") requires("+bindings", when="+python") @@ -325,7 +353,7 @@ def validate_gasnet_root(value): def flag_handler(self, name, flags): if name == "cxxflags": - if self.spec.satisfies("%oneapi@2025:"): + if self.spec.satisfies("%oneapi@2025:") or self.spec.satisfies("%cxx=clang@20:"): flags.append("-Wno-error=missing-template-arg-list-after-template-kw") return (flags, None, None) diff --git a/repos/spack_repo/builtin/packages/realm/package.py b/repos/spack_repo/builtin/packages/realm/package.py new file mode 100644 index 00000000000..0cebac55d58 --- /dev/null +++ b/repos/spack_repo/builtin/packages/realm/package.py @@ -0,0 +1,223 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage + +from spack.package import * + + +class Realm(CMakePackage, CudaPackage, ROCmPackage): + """Realm is a distributed, event–based tasking runtime for building + high-performance applications that span clusters of CPUs, GPUs, and other + accelerators. It began life as the low-level substrate underneath the Legion + programming system but is now maintained as a standalone project for developers + who want direct, fine-grained control of parallel and heterogeneous + machines.""" + + homepage = "https://legion.stanford.edu/realm/" + git = "https://github.com/StanfordLegion/realm.git" + url = "https://github.com/StanfordLegion/realm/archive/refs/tags/v25.6.1-rc.4.tar.gz" + + license("Apache-2.0") + + maintainers("rbberger") + + version("main", branch="main") + version( + "25.6.1-rc.4", sha256="077776e553a3098e19b431e2db797c9749366c2cc2a7b85f8b1535a958f60857" + ) + + depends_on("c", type="build") + depends_on("cxx", type="build") + + depends_on("cmake@3.22:", when="@25.03.0:", type="build") + + # TODO: Need to spec version of MPI v3 for use of the low-level MPI transport + # layer. At present the MPI layer is still experimental and we discourge its + # use for general (not legion development) use cases. + depends_on("mpi", when="network=mpi") + depends_on("ucx", when="network=ucx") + depends_on("ucc", when="network=ucx") + depends_on("ucc+cuda+nccl", when="network=ucx +cuda") + depends_on("ucc+rocm+rccl", when="network=ucx +rocm") + depends_on("hdf5", when="+hdf5") + depends_on("hwloc", when="+hwloc") + + # Propagate CUDA architectures + for arch in CudaPackage.cuda_arch_values: + depends_on(f"ucc cuda_arch={arch}", when=f"network=ucx +cuda cuda_arch={arch}") + depends_on(f"gasnet +cuda cuda_arch={arch}", when=f"network=gasnet +cuda cuda_arch={arch}") + depends_on( + f"kokkos+cuda+cuda_lambda cuda_arch={arch}", when=f"+kokkos+cuda cuda_arch={arch}" + ) + + for arch in ROCmPackage.amdgpu_targets: + depends_on(f"ucc amdgpu_target={arch}", when=f"network=ucx +rocm amdgpu_target={arch}") + depends_on(f"gasnet +rocm amdgpu_target={arch}", when=f"network=gasnet +rocm amdgpu_target={arch}") + depends_on(f"kokkos+rocm amdgpu_target={arch}", when=f"+rocm amdgpu_target={arch}") + + depends_on("kokkos@4:", when="+kokkos") + depends_on("kokkos+openmp", when="+kokkos+openmp") + depends_on("kokkos~openmp", when="+kokkos~openmp") + depends_on("kokkos+cmake_lang", when="+kokkos+cuda") + depends_on("kokkos+cmake_lang", when="+kokkos+rocm") + + depends_on("python@3.8:", when="+python") + + depends_on("papi", when="+papi") + + # A C++ standard variant to work-around some odd behaviors with apple-clang + # but this might be helpful for other use cases down the road. Legion's + # current development policy is C++11 or greater so we capture that aspect + # here. + cpp_stds = ("17", "20") + variant("cxxstd", default="17", description="C++ standard", values=cpp_stds, multi=False) + + # Network transport layer: the underlying data transport API should be used for + # distributed data movement. For Legion, gasnet is the currently the most + # mature. We have many users that default to using no network layer for + # day-to-day development thus we default to 'none'. MPI support is new and + # should be considered as a beta release. + variant( + "network", + default="none", + values=("gasnet", "mpi", "ucx", "none"), + description="The network communications/transport layer to use.", + multi=False, + ) + + depends_on("gasnet+pic", when="network=gasnet") + + # configs based on https://github.com/StanfordLegion/gasnet + depends_on("gasnet+mpi_compat+pshm conduits=smp", when="network=gasnet conduit=smp") + depends_on("gasnet+mpi_compat~pshm conduits=mpi", when="network=gasnet conduit=mpi") + depends_on( + "gasnet+mpi_compat+pshm conduits=ibv ibv_max_hcas=4", when="network=gasnet conduit=ibv" + ) + depends_on( + "gasnet+mpi_compat+pshm conduits=ofi ofi_provider=cxi ofi_spawner=pmi", + when="network=gasnet conduit=ofi-slingshot11", + ) + depends_on("gasnet pmi=cray", when="^[virtuals=mpi] cray-mpich") + + with when("network=gasnet"): + variant( + "conduit", + default="smp", + values=("smp", "aries", "ibv", "udp", "mpi", "ucx", "ofi-slingshot11"), + description="The GASNet conduit(s) to enable.", + sticky=True, + multi=False, + ) + + with when("network=ucx"): + variant( + "ucx_backends", + default="p2p", + values=("p2p", "mpi"), + description="UCX Bootstraps to build and install", + multi=True, + ) + requires("ucx_backends=p2p", msg="p2p backend is always enabled") + + variant("shared", default=False, description="Build shared libraries.") + + variant( + "log_level", + default="warning", + # Note: these values are dependent upon those used in the cmake config. + values=("spew", "debug", "info", "print", "warning", "error", "fatal", "none"), + description="Set the compile-time logging level.", + multi=False, + ) + + with when("+cuda"): + variant( + "cuda_dynamic_load", + default=False, + description="Enable dynamic loading of CUDA libraries.", + ) + variant( + "cuda_unsupported_compiler", + default=False, + description="Disable nvcc version check (--allow-unsupported-compiler).", + ) + + variant("hdf5", default=False, description="Enable support for HDF5.") + variant("hwloc", default=False, description="Use hwloc for topology awareness.") + variant( + "kokkos", default=False, description="Enable support for interoperability with Kokkos." + ) + variant( + "libdl", default=True, description="Enable support for dynamic object/library loading." + ) + variant("openmp", default=False, description="Enable support for OpenMP.") + variant("papi", default=False, description="Enable PAPI performance measurements.") + variant("python", default=False, description="Enable Python support.") + requires("+shared", when="+python") + + variant( + "max_dims", + values=int, + default="3", + description="Set max number of dimensions for logical regions.", + ) + + variant( + "sysomp", default=False, description="Use system OpenMP implementation instead of Realm's" + ) + + def cmake_args(self): + spec = self.spec + from_variant = self.define_from_variant + options = [ + self.define("REALM_ENABLE_INSTALL", True), + self.define("REALM_INSTALL", True), # remove once inconsistency is fixed + from_variant("REALM_CXX_STANDARD", "cxxstd"), + from_variant("BUILD_SHARED_LIBS", "shared"), + self.define("REALM_ENABLE_UCX", spec.satisfies("network=ucx")), + self.define("REALM_INSTALL_UCX_BOOTSTRAPS", spec.satisfies("network=ucx")), + self.define( + "UCX_BOOTSTRAP_ENABLE_MPI", spec.satisfies("network=ucx ucx_backends=mpi") + ), + self.define("REALM_ENABLE_GASNETEX", spec.satisfies("network=gasnet")), + self.define("REALM_ENABLE_MPI", spec.satisfies("network=mpi")), + from_variant("REALM_ENABLE_CUDA", "cuda"), + from_variant("REALM_ENABLE_HIP", "rocm"), + from_variant("REALM_ENABLE_HDF5", "hdf5"), + from_variant("REALM_ENABLE_HWLOC", "hwloc"), + from_variant("REALM_ENABLE_KOKKOS", "kokkos"), + from_variant("REALM_ENABLE_LIBDL", "libdl"), + from_variant("REALM_ENABLE_OPENMP", "openmp"), + from_variant("REALM_ENABLE_PAPI", "papi"), + from_variant("REALM_ENABLE_PYTHON", "python"), + from_variant("REALM_CUDA_DYNAMIC_LOAD", "cuda_dynamic_load"), + self.define("REALM_OPENMP_SYSTEM_RUNTIME", spec.satisfies("+openmp +sysomp")), + ] + + options.append(f"-DREALM_LOG_LEVEL={str.upper(spec.variants['log_level'].value)}") + + if spec.satisfies("+cuda"): + if spec.satisfies("%cxx=clang"): + options.append(self.define("CMAKE_CUDA_COMPILER", self.compiler.cxx)) + else: + options.append( + self.define( + "CMAKE_CUDA_COMPILER", join_path(spec["cuda"].prefix.bin, "nvcc") + ) + ) + if spec.satisfies("+cuda_unsupported_compiler"): + options.append("-DCUDA_NVCC_FLAGS:STRING=--allow-unsupported-compiler") + + options.append( + self.define("CMAKE_CUDA_ARCHITECTURES", spec.variants["cuda_arch"].value) + ) + + maxdims = int(spec.variants["max_dims"].value) + # TODO: sanity check if maxdims < 0 || > 9??? + options.append(f"-DREALM_MAX_DIM={maxdims}") + + return options