diff --git a/Jenkinsfile b/Jenkinsfile index 8f378fca83..f520fcc44c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,6 +12,10 @@ pipeline { name 'NODENAME' values 'balfrin' } + axis { + name 'NTHREADS' + values '32' + } } post { always { @@ -54,7 +58,7 @@ pipeline { sh """ source ./setup-env.sh ${upstream} source .venv/bin/activate - pytest -v -n auto test/integration_test.py + pytest -v -n ${NTHREADS} test/integration_test.py """ } } @@ -63,7 +67,7 @@ pipeline { sh """ source ./setup-env.sh ${upstream} source .venv/bin/activate - pytest -v -n auto test/common_system_test.py test/balfrin_system_test.py + pytest -v -n ${NTHREADS} test/common_system_test.py test/balfrin_system_test.py """ } } diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index b0b46e37a8..4993344399 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -32,9 +32,25 @@ def check_variant_extra_config_args(extra_config_arg): class Icon(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' - maintainers('jonasjucker', 'huppd') + maintainers('huppd') version('develop', submodules=True) + version("icon-2025.10", + tag="icon-2025.10", + git='git@gitlab.dkrz.de:icon/icon.git', + submodules=True) + version("icon-2025.04-2", + tag="icon-2025.04-2", + git='git@gitlab.dkrz.de:icon/icon.git', + submodules=True) + version("icon-2025.04-1", + tag="icon-2025.04-1", + git='git@gitlab.dkrz.de:icon/icon.git', + submodules=True) + version("icon-2025.04", + tag="icon-2025.04", + git='git@gitlab.dkrz.de:icon/icon.git', + submodules=True) version("2024.10", tag="icon-2024.10", git='git@gitlab.dkrz.de:icon/icon.git', @@ -43,6 +59,9 @@ class Icon(SpackIcon): tag="icon-2024.01-1", git='git@gitlab.dkrz.de:icon/icon.git', submodules=True) + version('2024.10-mch-1.0', tag='icon-2024.10-mch-1.0', submodules=True) + version('2024.01-mch-2.1', tag='icon-2024.01-mch-2.1', submodules=True) + version('2024.01-mch-2.0', tag='icon-2024.01-mch-2.0', submodules=True) version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) version('nwp-master', submodules=True) diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py deleted file mode 100644 index 5b2c95cedf..0000000000 --- a/repos/c2sm/packages/py-gt4py/package.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os -from spack import * - - -class PyGt4py(PythonPackage): - """Python library for generating high-performance implementations - of stencil kernels for weather and climate modeling from a - domain-specific language (DSL). """ - - homepage = "https://gridtools.github.io/gt4py/latest/index.html" - - url = "git@github.com:GridTools/gt4py.git" - - version('main', branch='main', git=url) - version('1.0.3.3', tag='icon4py_20240229', git=url) - version('1.0.3.7', tag='icon4py_20240521', git=url) - version('1.0.3.9', tag='icon4py_20240912', git=url) - version('1.0.3.10', tag='icon4py_20241113', git=url) - - maintainers = ['samkellerhals'] - - # Build dependencies - depends_on('py-wheel', type='build') - depends_on('py-cython', type='build') - - # downgrade to from 65 to 63 because of py-numpy - depends_on('py-setuptools@63:', type='build') - - depends_on('cmake@3.22:', type=('build', 'run')) - depends_on('boost@1.73.0:', type=('build', 'run')) - depends_on('clang-format@9:', type=('build', 'run')) - - # Python dependencies from setup.cfg - depends_on('python@3.10:', type=('build', 'run')) - depends_on('py-attrs@21.3:', type=('build', 'run')) - depends_on('py-black@22.3.0:', type=('build', 'run')) - depends_on('py-boltons@20.1:', type=('build', 'run')) - depends_on('py-cached-property@1.5.1:', type=('build', 'run')) - depends_on('py-click@8.0.0:', type=('build', 'run')) - depends_on('py-cytoolz@0.12:', type=('build', 'run')) - depends_on('py-deepdiff@5.6.0:', type=('build', 'run')) - depends_on('py-devtools@0.6:', type=('build', 'run')) - depends_on('py-frozendict@2.3:', type=('build', 'run')) - depends_on('py-gridtools-cpp@2.3.6:', type=( - 'build', 'run' - )) #gridtools-cpp is backwards compatible with older gt4py versions - depends_on('py-jinja2@3.0.0:', type=('build', 'run')) - depends_on('py-lark@1.1.2:', type=('build', 'run')) - depends_on('py-mako@1.1:', type=('build', 'run')) - depends_on('py-ninja@1.10:', type=('build', 'run')) - depends_on('py-numpy@1.24.2:', type=('build', 'run')) - depends_on('py-packaging@20.0:', type=('build', 'run')) - depends_on('py-pybind11@2.10.1:', type=('build', 'run')) - depends_on('py-nanobind@1.4.0:', when="@1.0.1.3:", type=('build', 'run')) - depends_on('py-tabulate@0.8.10:', type=('build', 'run')) - depends_on('py-typing-extensions@4.5.0', - when="@:1.0.3.5", - type=('build', 'run')) - depends_on('py-typing-extensions@4.10.0', - when="@1.0.3.6:", - type=('build', 'run')) - depends_on('py-toolz@0.12.0:', type=('build', 'run')) - depends_on('py-xxhash@1.4.4:3.0.9', type=('build', 'run')) - depends_on('py-hypothesis@6.0.0:', type=('build', 'run')) - - # Python dependencies from requirements-dev.txt - # Convert to type=('test') ? - depends_on('py-coverage@5.0:', type=('build', 'run')) - depends_on('py-devtools@0.6:', type=('build', 'run')) - depends_on('py-factory-boy@3.1:', type=('build', 'run')) - depends_on('py-psutil@5.0:', type=('build', 'run')) - depends_on('py-pytest-cache@1.0:', type=('build', 'run')) - depends_on('py-pytest-cov@2.8:', type=('build', 'run')) - depends_on('py-pytest-factoryboy@2.0.3:', type=('build', 'run')) - depends_on('py-pytest@7.0:', type=('build', 'run')) - depends_on('py-diskcache@5.2:', type=('build', 'run')) - # setup.cfg requires newer version, but not available yet - depends_on('py-tox@3.14:', type=('build', 'run')) - - # missing version constraint: pytest-xdist[psutil]>=2.4 - depends_on('py-pytest-xdist', type=('build', 'run')) - - def test(self): - # workaround for not finding own python module - python_spec = self.spec['python'] - python_version = python_spec.version.up_to(2) - install_path = join_path(self.prefix, 'lib', f"python{python_version}", - 'site-packages') - os.environ[ - 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" - - python('-m', 'pytest', '-v', '-s', '-n', 'auto', '-k', '.run_gtfn]', - 'tests/next_tests', 'tests/eve_tests') diff --git a/repos/c2sm/packages/py-gt4py/patches/functional/rm_cov_config_arg.patch b/repos/c2sm/packages/py-gt4py/patches/functional/rm_cov_config_arg.patch deleted file mode 100644 index d09f092d06..0000000000 --- a/repos/c2sm/packages/py-gt4py/patches/functional/rm_cov_config_arg.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/setup.cfg b/setup.cfg -index 5028dc4..ac438f7 100644 ---- a/setup.cfg -+++ b/setup.cfg -@@ -256,7 +256,7 @@ ignore_errors = True - #-- pytest -- - [tool:pytest] - # Set coverage options but do not run it by default --addopts = --cov-config=setup.cfg -+#addopts = --cov-config=setup.cfg - markers = - lit_suite: pytest item running a LLVM-lit test suite - norecursedirs = diff --git a/repos/c2sm/packages/py-icon4py/package.py b/repos/c2sm/packages/py-icon4py/package.py deleted file mode 100644 index c9dea8fb2e..0000000000 --- a/repos/c2sm/packages/py-icon4py/package.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import inspect - -from spack import * -from spack.build_systems.python import PythonPipBuilder - -import llnl.util.filesystem as fs - - -class PyIcon4py(PythonPackage): - """ICON4Py contains Python (GT4Py) implementations of ICON (inspired) - components for weather and climate models.""" - - url = "git@github.com:C2SM/icon4py.git" - git = 'git@github.com:C2SM/icon4py.git' - - homepage = "https://github.com/C2SM/icon4py" - - maintainers = ['agopal', 'samkellerhals'] - - version('main', branch='main', git=git) - version('0.0.10', tag='v0.0.10', git=git) - version('0.0.11', tag='v0.0.11', git=git) - version('0.0.13', tag='v0.0.13', git=git) - version('0.0.14', tag='v0.0.14', git=git) - - depends_on('py-wheel', type='build') - depends_on('py-setuptools', type='build') - - depends_on('python@3.10:', type=('build', 'run')) - depends_on('py-tabulate@0.8.9:', type=('build', 'run')) - # TODO: push new version to Spack official - depends_on('py-fprettify@0.3.7:', type=('build', 'run')) - depends_on('py-gt4py', type=('build', 'run')) - depends_on('py-pytest', type=('build', 'run')) - depends_on('boost@1.73.0:', type=('build', 'run')) - - def patch(self): - spack_pytest_ini = 'jenkins/spack/pytest.ini' - if os.path.exists(spack_pytest_ini): - fs.install(spack_pytest_ini, '.') - - # cmake in unit-tests needs this path - def setup_build_environment(self, env): - env.set("CMAKE_INCLUDE_PATH", self.spec['boost'].prefix.include) - - def test(self): - # workaround for not finding own python module - python_spec = self.spec['python'] - python_version = python_spec.version.up_to(2) - install_path = join_path(self.prefix, 'lib', f"python{python_version}", - 'site-packages') - os.environ[ - 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" - # check if all installed module can be imported - super().test() - # unit tests - - python('-m', 'pytest', '-v', '-s', '-n', 'auto', '-m', - 'not slow_tests') - - @property - def headers(self): - '''Workaround to hide the details of the installation path, - i.e "lib/python3.10/site-packages/icon4py/atm_dyn_iconam" - from upstream packages. It needs to be part of the "Spec" object, - therefore choose the headers-function - ''' - query_parameters = self.spec.last_query.extra_parameters - version = self.spec.version - - folder_mapping = { - ver('=0.0.3.1'): { - 'atm_dyn_iconam': 'atm_dyn_iconam', - 'utils': - 'liskov' #utils will eventually map to parent directory of liskov - }, - ver('=0.0.9'): { - 'atm_dyn_iconam': 'dycore', - 'tools': 'icon4pytools', - 'diffusion': 'diffusion/stencils', - 'interpolation': 'interpolation/stencils', - 'advection': 'advection', - }, - ver('=0.0.10'): { - 'atm_dyn_iconam': 'dycore', - 'tools': 'icon4pytools', - 'diffusion': 'diffusion/stencils', - 'interpolation': 'interpolation/stencils', - 'advection': 'advection', - }, - ver('=0.0.11'): { - 'atm_dyn_iconam': 'dycore', - 'tools': 'icon4pytools', - 'diffusion': 'diffusion/stencils', - 'interpolation': 'interpolation/stencils', - 'advection': 'advection', - }, - ver('=main'): { - 'atm_dyn_iconam': 'dycore', - 'tools': 'icon4pytools', - 'diffusion': 'diffusion/stencils', - 'interpolation': 'interpolation/stencils', - 'advection': 'advection', - }, - } - - if len(query_parameters) > 1: - raise ValueError('Only one query parameter allowed') - - folder_name = folder_mapping.get(version, {}) - - if not folder_name: - return HeaderList([]) - - for param, folder in folder_name.items(): - if param in query_parameters: - return self._find_folder_and_add_dummy_header( - self.prefix, folder) - - return HeaderList([]) - - def _find_folder_and_add_dummy_header(self, prefix, name): - folder = find(prefix, name) - headerlist = HeaderList(f'{folder[0]}/dummy.h') - return headerlist - - -class PythonPipBuilder(PythonPipBuilder): - - def install(self, pkg, spec, prefix): - """Install everything from build directory.""" - - args = PythonPipBuilder.std_args(pkg) + ["--prefix=" + prefix] - - for key, value in self.config_settings(spec, prefix).items(): - if spec["py-pip"].version < Version("22.1"): - raise SpecError( - "'{}' package uses 'config_settings' which is only supported by " - "pip 22.1+. Add the following line to the package to fix this:\n\n" - ' depends_on("py-pip@22.1:", type="build")'.format( - spec.name)) - - args.append("--config-settings={}={}".format(key, value)) - - for option in self.install_options(spec, prefix): - args.append("--install-option=" + option) - for option in self.global_options(spec, prefix): - args.append("--global-option=" + option) - - if pkg.stage.archive_file and pkg.stage.archive_file.endswith(".whl"): - args.append(pkg.stage.archive_file) - else: - args.append(".") - - pip = inspect.getmodule(pkg).pip - - if self.spec.version == ver('=0.0.3.1'): - build_dirs = [ - 'common', 'pyutils', 'testutils', 'liskov', 'atm_dyn_iconam' - ] - else: - build_dirs = [ - 'tools', 'model/atmosphere/dycore', - 'model/atmosphere/diffusion', 'model/atmosphere/advection', - 'model/common/' - ] - - for dir in build_dirs: - with fs.working_dir(os.path.join(self.build_directory, dir)): - pip(*args) diff --git a/test/balfrin_system_test.py b/test/balfrin_system_test.py index de5d0d12ed..966aaeea96 100644 --- a/test/balfrin_system_test.py +++ b/test/balfrin_system_test.py @@ -12,20 +12,10 @@ def test_install_icon_conditional_dependencies(): # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install( - 'icon @2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' + 'icon @2024.10-mch-1.0 %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' ) -# fails due to sql error -def test_build_only_py_gt4py_for_1_0_3_10(): - spack_install('py-gt4py @1.0.3.10', test_root=False) - - -# fails due to sql error -def test_build_only_py_icon4py_for_0_0_14(): - spack_install('py-icon4py@ 0.0.14 ^py-gt4py @1.0.3.10', test_root=False) - - def test_install_yaxt(): spack_install('yaxt') diff --git a/test/common_system_test.py b/test/common_system_test.py index 281367d5b1..439a9642f0 100644 --- a/test/common_system_test.py +++ b/test/common_system_test.py @@ -28,7 +28,8 @@ def test_install_makedepf90(): spack_install('makedepf90', test_root=False) -@pytest.mark.parametrize('version', ['2024.10', '2.6.6-mch2a', '2.6.6-mch2b']) +@pytest.mark.parametrize('version', + ['2024.10', '2024.10-mch-1.0', '2.6.6-mch2b']) def test_install_icon(version): # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install(f'icon @{version} %nvhpc ^cray-mpich%nvhpc') diff --git a/test/santis_system_test.py b/test/santis_system_test.py index 0d973d384a..7e4e238337 100644 --- a/test/santis_system_test.py +++ b/test/santis_system_test.py @@ -13,5 +13,5 @@ def test_install_icon_conditional_dependencies(): # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install( - 'icon @2.6.6-mch2b %nvhpc +coupling +emvorado +mpi gpu=nvidia-90 ^cray-mpich%nvhpc' + 'icon @2024.10-mch-1.0 %nvhpc +coupling +emvorado +mpi gpu=nvidia-90 ^cray-mpich%nvhpc' )