ICON-DSL uenv: ICON-Exclaim with ICON4Py dycore.

config.yaml

@@ -39,6 +39,13 @@ uenvs:
         santis: [gh200]
         balfrin: [a100]
       develop: False
+  icon-dsl:
+    "25.8":
+      recipes:
+        gh200: '25.8/gh200'
+      deploy:
+        santis: [gh200]
+      develop: False
   mch:
     "v7":
       recipes:

recipes/icon-dsl/25.8/g200/compilers.yaml

@@ -0,0 +1,5 @@
+bootstrap:
+    spec: gcc@11
+gcc:
+    specs:
+    - gcc@11

recipes/icon-dsl/25.8/g200/config.yaml

@@ -0,0 +1,7 @@
+name: icon4pypoc
+store: /user-environment
+spack:
+    repo: https://github.com/spack/spack.git
+    commit: releases/v0.23
+modules: false
+description: icon4py proof-of-concept

recipes/icon-dsl/25.8/g200/environments.yaml

@@ -0,0 +1,11 @@
+myenv:
+  compiler:
+  - toolchain: gcc
+    spec: gcc
+  specs:
+  - icon4py@icon_20250328 +distributed +cuda
+  - py-mpi4py
+  - python
+  unify: true
+  views:
+    default:

recipes/icon-dsl/25.8/g200/repo/packages/ghex/package.py

@@ -0,0 +1,87 @@
+from spack.package import *
+
+
+class Ghex(CMakePackage, CudaPackage, ROCmPackage):
+    """GHEX is a generic halo-exchange library."""
+
+    homepage = "https://github.com/ghex-org/GHEX"
+    url = "https://github.com/ghex-org/GHEX/archive/refs/tags/v0.3.0.tar.gz"
+    git = "https://github.com/ghex-org/GHEX.git"
+    maintainers = ["boeschf"]
+
+    version("0.4.1", tag="v0.4.1", submodules=True)
+    version("0.4.0", tag="v0.4.0", submodules=True)
+    version("0.3.0", tag="v0.3.0", submodules=True)
+    version("master", branch="master", submodules=True)
+
+    depends_on("cxx", type="build")
+
+    generator("ninja")
+
+    backends = ("mpi", "ucx", "libfabric")
+    variant(
+        "backend", default="mpi", description="Transport backend", values=backends, multi=False
+    )
+    variant("xpmem", default=False, description="Use xpmem shared memory")
+    variant("python", default=True, description="Build Python bindings")
+
+    depends_on("cmake@3.21:", type="build")
+    depends_on("mpi")
+    depends_on("boost")
+    depends_on("xpmem", when="+xpmem", type=("build", "run"))
+
+    depends_on("oomph")
+    for backend in backends:
+        depends_on(f"oomph backend={backend}", when=f"backend={backend}")
+    depends_on("oomph+cuda", when="+cuda")
+    depends_on("oomph+rocm", when="+rocm")
+    depends_on("oomph@0.3:", when="@0.3:")
+
+    conflicts("+cuda+rocm")
+
+    with when("+python"):
+        extends("python")
+        depends_on("python@3.7:", type="build")
+        depends_on("py-pip", type="build")
+        depends_on("py-pybind11", type="build")
+        depends_on("py-mpi4py", type=("build", "run"))
+        depends_on("py-numpy", type=("build", "run"))
+
+        depends_on("py-pytest", when="+python", type=("test"))
+
+    def cmake_args(self):
+        spec = self.spec
+
+        args = [
+            self.define("GHEX_USE_BUNDLED_LIBS", True),
+            self.define("GHEX_USE_BUNDLED_GRIDTOOLS", True),
+            self.define("GHEX_USE_BUNDLED_GTEST", self.run_tests),
+            self.define("GHEX_USE_BUNDLED_OOMPH", False),
+            self.define("GHEX_TRANSPORT_BACKEND", spec.variants["backend"].value.upper()),
+            self.define_from_variant("GHEX_USE_XPMEM", "xpmem"),
+            self.define_from_variant("GHEX_BUILD_PYTHON_BINDINGS", "python"),
+            self.define("GHEX_WITH_TESTING", self.run_tests),
+        ]
+
+        if spec.satisfies("+python"):
+            args.append(self.define("GHEX_PYTHON_LIB_PATH", python_platlib))
+
+        if self.run_tests and spec.satisfies("^openmpi"):
+            args.append(self.define("MPIEXEC_PREFLAGS", "--oversubscribe"))
+
+        if "+cuda" in spec and spec.variants["cuda_arch"].value != "none":
+            arch_str = ";".join(spec.variants["cuda_arch"].value)
+            args.append(self.define("CMAKE_CUDA_ARCHITECTURES", arch_str))
+            args.append(self.define("GHEX_USE_GPU", True))
+            args.append(self.define("GHEX_GPU_TYPE", "CUDA"))
+
+        if "+rocm" in spec and spec.variants["amdgpu_target"].value != "none":
+            arch_str = ";".join(spec.variants["amdgpu_target"].value)
+            args.append(self.define("CMAKE_HIP_ARCHITECTURES", arch_str))
+            args.append(self.define("GHEX_USE_GPU", True))
+            args.append(self.define("GHEX_GPU_TYPE", "AMD"))
+
+        if spec.satisfies("~cuda~rocm"):
+            args.append(self.define("GHEX_USE_GPU", False))
+
+        return args

recipes/icon-dsl/25.8/g200/repo/packages/hwmalloc/cmake_install_path.patch

@@ -0,0 +1,27 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index d5420e0..35dbe56 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -105,11 +105,11 @@ install(FILES ${PROJECT_BINARY_DIR}/include/hwmalloc/config.hpp
+ install(EXPORT HWMALLOC-targets
+   FILE HWMALLOC-targets.cmake
+   NAMESPACE HWMALLOC::
+-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake)
++  DESTINATION ${CMAKE_INSTALL_LIBDIR}/hwmalloc/cmake)
+
+ configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/HWMALLOCConfig.cmake.in
+     ${CMAKE_CURRENT_BINARY_DIR}/HWMALLOCConfig.cmake
+-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake)
++    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/hwmalloc/cmake)
+
+ write_basic_package_version_file(HWMALLOCConfigVersion.cmake
+     VERSION ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion)
+@@ -120,7 +120,7 @@ install(
+         ${CMAKE_CURRENT_BINARY_DIR}/HWMALLOCConfigVersion.cmake
+         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindNUMA.cmake
+     DESTINATION
+-        ${CMAKE_INSTALL_LIBDIR}/cmake)
++        ${CMAKE_INSTALL_LIBDIR}/hwmalloc/cmake)
+
+ export(EXPORT HWMALLOC-targets
+   FILE "${CMAKE_CURRENT_BINARY_DIR}/HWMALLOC-targets.cmake")

recipes/icon-dsl/25.8/g200/repo/packages/hwmalloc/package.py

@@ -0,0 +1,52 @@
+from spack.package import *
+
+
+class Hwmalloc(CMakePackage, CudaPackage, ROCmPackage):
+    """HWMALLOC is a allocator which supports memory registration for e.g. remote memory access"""
+
+    homepage = "https://github.com/ghex-org/hwmalloc"
+    url = "https://github.com/ghex-org/hwmalloc/archive/refs/tags/v0.3.0.tar.gz"
+    git = "https://github.com/ghex-org/hwmalloc.git"
+    maintainers = ["boeschf"]
+
+    version("0.3.0", sha256="d4d4ac6087a806600d79fb62c02719ca3d58a412968fe1ef4a2fd58d9e7ee950")
+    version("0.2.0", sha256="734758a390a3258b86307e4aef50a7ca2e5d0e2e579f18aeefcd05397e114419")
+    version("0.1.0", sha256="06e9bfcef0ecce4d19531ccbe03592b502d1281c7a092bc0ff51ca187899b21c")
+    version("master", branch="master")
+
+    depends_on("cxx", type="build")
+
+    generator("ninja")
+
+    depends_on("numactl", type=("build", "run"))
+    depends_on("boost", type=("build"))
+    depends_on("cmake@3.19:", type="build")
+
+    variant(
+        "numa-throws",
+        default=False,
+        description="True if numa_tools may throw during initialization",
+    )
+    variant("numa-local", default=True, description="Use numa_tools for local node allocations")
+    variant("logging", default=False, description="print logging info to cerr")
+
+    patch("cmake_install_path.patch", when="@:0.3.0", level=1)
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant("HWMALLOC_NUMA_THROWS", "numa-throws"),
+            self.define_from_variant("HWMALLOC_NUMA_FOR_LOCAL", "numa-local"),
+            self.define_from_variant("HWMALLOC_ENABLE_LOGGING", "logging"),
+            self.define("HWMALLOC_WITH_TESTING", self.run_tests),
+        ]
+
+        if "+cuda" in self.spec:
+            args.append(self.define("HWMALLOC_ENABLE_DEVICE", True))
+            args.append(self.define("HWMALLOC_DEVICE_RUNTIME", "cuda"))
+        elif "+rocm" in self.spec:
+            args.append(self.define("HWMALLOC_ENABLE_DEVICE", True))
+            args.append(self.define("HWMALLOC_DEVICE_RUNTIME", "hip"))
+        else:
+            args.append(self.define("HWMALLOC_ENABLE_DEVICE", False))
+
+        return args

recipes/icon-dsl/25.8/g200/repo/packages/icon4py/package.py

@@ -0,0 +1,113 @@
+import json
+import os
+import pathlib
+
+import llnl
+from llnl.util import tty
+from spack import *
+
+
+class Icon4py(Package):
+    extends("python")
+    depends_on("python@3.11:")
+
+    variant("cuda", default=True, description="Enable CUDA support")
+    variant("distributed", default=True, description="Enable multinode support")
+
+    depends_on("git")
+    depends_on("boost@1.85:", type=("build", "run"))
+    depends_on("py-uv@0.7:")
+    depends_on("bzip2", type="build")
+    depends_on("py-numpy")
+    depends_on("py-cffi")
+    depends_on("py-pybind11")
+    depends_on("py-nanobind")
+    depends_on("py-mpi4py", when="+distributed")
+    depends_on("py-cupy +cuda", when="+cuda")
+    depends_on("ghex +python", when="+distributed")
+    depends_on("ghex +python +cuda", when="+distributed +cuda")
+
+    version(
+        "icon_20250328",
+        sha256="8573ef031d207438f549511e859f522c60163ea660aafea93ef4991b9010739a",
+        extension="zip",
+    )
+
+    def url_for_version(self, version):
+        return f"https://github.com/c2sm/icon4py/archive/refs/heads/{version}.zip"
+
+    def install(self, spec, prefix):
+        uv = prepare_uv()
+        python_spec = spec["python"]
+        venv_path = prefix.share.venv
+
+        tty.msg(f"creating venv using spack python at: {python_spec.command.path}")
+        uv(
+            "venv",
+            "--seed",
+            "--relocatable",
+            "--system-site-packages",
+            str(venv_path),
+            "--python",
+            python_spec.command.path,
+        )
+
+        tty.msg(f"grabbing spack installed packages (distributions)")
+        pip = Executable(venv_path.bin.pip)
+        spack_installed = get_installed_pkg(pip)
+
+        tty.msg(f"installing missing packages")
+        uv(
+            "sync",
+            "--active",
+            "--extra",
+            "fortran",
+            "--inexact",
+            "--no-editable",
+            "--python",
+            str(venv_path.bin.python),
+            *no_install_options(spack_installed),
+            extra_env={"VIRTUAL_ENV": str(venv_path)},
+        )
+
+        tty.msg(f"linking spack installed packages into venv")
+        pathlib.Path(
+            f"{venv_path.lib.python}{python_spec.version.up_to(2)}/site-packages/spack_installed.pth"
+        ).write_text(pythonpath_to_pth())
+
+        # uv("run", "--active", "py2fgen", "icon4py.tools.py2fgen.wrappers.all_bindings", "diffusion_init,diffusion_run,grid_init,solve_nh_init,solve_nh_run", "icon4py_bindings", "-o", prefix.src, extra_env={"VIRTUAL_ENV": str(venv_path)})
+
+        tty.msg(f"running py2fgen")
+        py2fgen = Executable(venv_path.bin.py2fgen)
+        py2fgen(
+            "icon4py.tools.py2fgen.wrappers.all_bindings",
+            "diffusion_init,diffusion_run,grid_init,solve_nh_init,solve_nh_run",
+            "icon4py_bindings",
+            "-o",
+            prefix.src,
+            extra_env={"VIRTUAL_ENV": str(venv_path)},
+        )
+
+
+def prepare_uv():
+    uv = which("uv")
+    uv.add_default_env("UV_NO_CACHE", "true")
+    uv.add_default_env("UV_NO_MANAGED_PYTHON", "true")
+    uv.add_default_env("UV_PYTHON_DOWNLOADS", "never")
+    return uv
+
+
+def get_installed_pkg(pip):
+    return [
+        item["name"] for item in json.loads(pip("list", "--format", "json", output=str))
+    ]
+
+
+def no_install_options(installed):
+    for name in installed:
+        yield "--no-install-package"
+        yield name
+
+
+def pythonpath_to_pth():
+    return "\n".join(os.environ["PYTHONPATH"].split(":"))