From 0026bd57d2dfd7eeeadd709534514e87b5e1d21d Mon Sep 17 00:00:00 2001 From: Luke Robison <108305344+lrbison@users.noreply.github.com> Date: Fri, 30 May 2025 11:47:16 -0500 Subject: [PATCH 1/4] Update aws-pcluster-amazonlinux-2 stacks These stacks start the build with gcc@12 installed from a local buildcache --- .ci/gitlab/.gitlab-ci.yml | 2 +- .../aws-pcluster-neoverse_v1/spack.yaml | 20 +++++-------------- .../stacks/aws-pcluster-x86_64_v4/spack.yaml | 17 +++++----------- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/.ci/gitlab/.gitlab-ci.yml b/.ci/gitlab/.gitlab-ci.yml index 0bbe2317e27..bc5b70598c9 100644 --- a/.ci/gitlab/.gitlab-ci.yml +++ b/.ci/gitlab/.gitlab-ci.yml @@ -759,7 +759,7 @@ hep-build: ######################################## .aws-pcluster-generate: - image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] } + image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:pr-62", "entrypoint": [""] } # X86_64_V4 (one pipeline per target) .aws-pcluster-x86_64_v4: diff --git a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml index 23c92893300..763abf2775e 100644 --- a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml @@ -17,7 +17,7 @@ spack: ci: pipeline-gen: - build-job: - image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']} + image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-62, entrypoint: ['']} tags: [aarch64] before_script: - - mkdir spack @@ -31,8 +31,10 @@ spack: - . /etc/profile.d/modules.sh - spack --version - spack arch - - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/binutils-2.37-2yxz3xsjfmesxujxtlrgcctxlyilynmp/bin:$PATH + - spack mirror add --signed boot /bootstrap-gcc-cache/ + - spack buildcache keys --install --trust --force - spack -c "config:install_tree:root:/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh" reindex + - spack install --use-buildcache package:only gcc@12 - signing-job: before_script: # Do not distribute Intel & ARM binaries @@ -47,23 +49,12 @@ spack: missing_library_policy: ignore # due to use of externals packages: - gcc: - externals: - - spec: gcc@=12.4.0 - prefix: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b - extra_attributes: - compilers: - c: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gcc - cxx: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/g++ - fortran: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gfortran acfl: require: - - "%gcc" - "target=aarch64" gromacs: require: - gromacs@2024.3 ^armpl-gcc ^openmpi - - "%gcc" - target=neoverse_v1 libfabric: buildable: true @@ -106,7 +97,7 @@ spack: - target=neoverse_v1 quantum-espresso: require: - - quantum-espresso@6.6 %gcc ^armpl-gcc + - quantum-espresso@6.6 ^armpl-gcc %gcc - target=neoverse_v1 slurm: buildable: false @@ -118,7 +109,6 @@ spack: all: target: ["neoverse_v1"] require: - - "%gcc" - "target=neoverse_v1" providers: blas: [armpl-gcc, openblas] diff --git a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml index b729c509653..04755abbe90 100644 --- a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml @@ -22,7 +22,7 @@ spack: ci: pipeline-gen: - build-job: - image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']} + image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-62, entrypoint: ['']} before_script: - - mkdir spack - cd spack @@ -35,8 +35,10 @@ spack: - . /etc/profile.d/modules.sh - spack --version - spack arch - - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/binutils-2.37-qvccg7zpskturysmr4bzbsfrx34kvazo/bin:$PATH + - spack mirror add --signed boot /bootstrap-gcc-cache/ + - spack buildcache keys --install --trust --force - spack -c "config:install_tree:root:/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh" reindex + - spack install --use-buildcache package:only gcc@12 - signing-job: before_script: # Do not distribute Intel & ARM binaries @@ -51,15 +53,6 @@ spack: missing_library_policy: ignore # due to use of externals packages: - gcc: - externals: - - spec: gcc@=12.4.0 - prefix: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb - extra_attributes: - compilers: - c: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/gcc - cxx: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/g++ - fortran: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/gfortran intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@=2024.1.0 @@ -110,7 +103,7 @@ spack: - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm openfoam: require: - - openfoam %gcc ^scotch@6.0.9 + - openfoam ^scotch@6.0.9 %gcc openmpi: variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm require: From 0184377f2af048c6d7d29506f7b1ba8af28d36c6 Mon Sep 17 00:00:00 2001 From: Luke Robison Date: Wed, 4 Jun 2025 21:18:28 +0000 Subject: [PATCH 2/4] WIP: Update neoverse_v1 stack --- .../aws-pcluster-neoverse_v1/spack.yaml | 52 ++++++++----------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml index 763abf2775e..7dab6b715a1 100644 --- a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml @@ -49,56 +49,59 @@ spack: missing_library_policy: ignore # due to use of externals packages: + all: + permissions: + read: world + write: user + target: ["neoverse_v1"] + providers: + blas: [armpl-gcc, openblas] + fftw-api: [armpl-gcc, fftw] + mpi: [openmpi, mpich] + lapack: [armpl-gcc, openblas] + scalapack: [netlib-scalapack] + c: [gcc@12, acfl] + cxx: [gcc@12, acfl] + fortran: [gcc@12, acfl] + c: + require: gcc@12 acfl: require: - "target=aarch64" gromacs: - require: - - gromacs@2024.3 ^armpl-gcc ^openmpi - - target=neoverse_v1 + version: [2024.3] libfabric: buildable: true externals: - prefix: /opt/amazon/efa/ spec: libfabric@1.17.0 - require: + variants: - fabrics=shm,efa - - target=neoverse_v1 llvm: variants: ~lldb mpas-model: - require: - - precision=single ^parallelio+pnetcdf - - "%gcc" - - target=neoverse_v1 + variants: + - precision=single ^parallelio+pnetcdf mpich: require: - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm - - target=neoverse_v1 nvhpc: require: - "target=aarch64" openfoam: require: - openfoam ^scotch@6.0.9 - - target=neoverse_v1 openmpi: variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm require: - '@4:' - - target=neoverse_v1 # Palace does not build correctly with armpl until https://github.com/awslabs/palace/pull/207 is merged into a version. # palace: # require: # - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"] pmix: require: - - "pmix@3:" - - target=neoverse_v1 - quantum-espresso: - require: - - quantum-espresso@6.6 ^armpl-gcc %gcc - - target=neoverse_v1 + - '@3:' slurm: buildable: false externals: @@ -106,16 +109,3 @@ spack: spec: slurm@22.05.8 +pmix require: - "+pmix" - all: - target: ["neoverse_v1"] - require: - - "target=neoverse_v1" - providers: - blas: [armpl-gcc, openblas] - fftw-api: [armpl-gcc, fftw] - lapack: [armpl-gcc, openblas] - mpi: [openmpi, mpich] - scalapack: [netlib-scalapack] - permissions: - read: world - write: user From 5faccf3281c9dddafbe27d5dea6245df7945b49b Mon Sep 17 00:00:00 2001 From: Luke Robison Date: Thu, 5 Jun 2025 21:58:20 +0000 Subject: [PATCH 3/4] WIP: Update x86_64 stack --- .../stacks/aws-pcluster-x86_64_v4/spack.yaml | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml index 04755abbe90..d1094181eca 100644 --- a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml @@ -39,6 +39,7 @@ spack: - spack buildcache keys --install --trust --force - spack -c "config:install_tree:root:/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh" reindex - spack install --use-buildcache package:only gcc@12 + - spack install intel-oneapi-compilers target=x86_64_v3 - signing-job: before_script: # Do not distribute Intel & ARM binaries @@ -53,20 +54,6 @@ spack: missing_library_policy: ignore # due to use of externals packages: - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@=2024.1.0 - prefix: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop - extra_attributes: - compilers: - c: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/icx - cxx: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/icpx - fortran: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx - extra_rpaths: - - /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/lib64 - fortran: - require: - - "intel-oneapi-compilers" gettext: # Newer gettext cannot build with gcc@12 and old AL2 glibc headers # Older gettext versions do not build correctly with oneapi. @@ -101,9 +88,6 @@ spack: mpich: require: - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm - openfoam: - require: - - openfoam ^scotch@6.0.9 %gcc openmpi: variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm require: @@ -128,12 +112,13 @@ spack: require: - wrf@4 build_type=dm+sm all: - require: - - "%oneapi" permissions: read: world write: user providers: + c: [intel-oneapi-compilers, gcc] + cxx: [intel-oneapi-compilers, gcc] + fortran: [intel-oneapi-compilers, gcc] blas: [intel-oneapi-mkl] daal: [intel-oneapi-dal] fftw-api: [intel-oneapi-mkl] From d702ab8ce2932014858feea419cee1876271a23a Mon Sep 17 00:00:00 2001 From: Luke Robison Date: Tue, 17 Jun 2025 15:03:43 +0000 Subject: [PATCH 4/4] WIP: Switch to pr-66. Final before tag and squash --- .ci/gitlab/.gitlab-ci.yml | 2 +- .ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml | 2 +- .ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/gitlab/.gitlab-ci.yml b/.ci/gitlab/.gitlab-ci.yml index bc5b70598c9..4c445ee5cf5 100644 --- a/.ci/gitlab/.gitlab-ci.yml +++ b/.ci/gitlab/.gitlab-ci.yml @@ -759,7 +759,7 @@ hep-build: ######################################## .aws-pcluster-generate: - image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:pr-62", "entrypoint": [""] } + image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:pr-66", "entrypoint": [""] } # X86_64_V4 (one pipeline per target) .aws-pcluster-x86_64_v4: diff --git a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml index 7dab6b715a1..2e246b6a296 100644 --- a/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-neoverse_v1/spack.yaml @@ -17,7 +17,7 @@ spack: ci: pipeline-gen: - build-job: - image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-62, entrypoint: ['']} + image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-66, entrypoint: ['']} tags: [aarch64] before_script: - - mkdir spack diff --git a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml index d1094181eca..17b30028875 100644 --- a/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml +++ b/.ci/gitlab/stacks/aws-pcluster-x86_64_v4/spack.yaml @@ -22,7 +22,7 @@ spack: ci: pipeline-gen: - build-job: - image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-62, entrypoint: ['']} + image: {name: ghcr.io/spack/pcluster-amazonlinux-2:pr-66, entrypoint: ['']} before_script: - - mkdir spack - cd spack @@ -39,7 +39,7 @@ spack: - spack buildcache keys --install --trust --force - spack -c "config:install_tree:root:/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh" reindex - spack install --use-buildcache package:only gcc@12 - - spack install intel-oneapi-compilers target=x86_64_v3 + - spack install --use-buildcache package:only intel-oneapi-compilers@2024.1.0 target=x86_64_v3 - signing-job: before_script: # Do not distribute Intel & ARM binaries