Merge pull request #4378 from psychocoderHPC/topic-ciContainerVersion3

switch to CI container version 3

.gitlab-ci.yml

@@ -6,7 +6,7 @@ stages:
   - test_pmacc
 
 variables:
-  CONTAINER_TAG: "2.0"
+  CONTAINER_TAG: "3.0"
 
 .base_generate-reduced-matrix:
   stage: generate

INSTALL.rst

@@ -31,12 +31,12 @@ Mandatory
 
 Compiler
 """"""""
-- C++17 supporting compiler, e.g. GCC 7+ or Clang 6+
+- C++17 supporting compiler, e.g. GCC 9+ or Clang 10+
 - if you want to build for Nvidia GPUs, check the `CUDA supported compilers <https://gist.github.com/ax3l/9489132>`_ page
 - *note:* be sure to build all libraries/dependencies with the *same* compiler version
 - *Debian/Ubuntu:*
 
-  - ``sudo apt-get install gcc-8 g++-8 build-essential``
+  - ``sudo apt-get install gcc-9 g++-9 build-essential``
   - ``sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-8``
 - *Arch Linux:*
 

include/pmacc/PMaccConfig.cmake

@@ -194,17 +194,21 @@ set(CMAKE_CXX_STANDARD 17)
 ###############################################################################
 # GNU
 if(CMAKE_COMPILER_IS_GNUCXX)
-    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)
-        message(FATAL_ERROR "GCC too old! Use GCC 4.9 or newer")
+    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0)
+        message(FATAL_ERROR "GCC too old! Use GCC 7.0 or newer")
+    elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+        message(WARNING "GCC < 9.0 is not a official supported, please try a newer!")
     endif()
 # Clang
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
     if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0)
         message(FATAL_ERROR "Apple Clang (Xcode) too old! Use Xcode 8.0 or newer")
     endif()
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.9)
-        message(FATAL_ERROR "Clang too old! Use Clang 3.9 or newer")
+    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0)
+        message(FATAL_ERROR "Clang too old! Use Clang 6.0 or newer")
+    elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0)
+        message(WARNING "Clang < 10.0 is not a official supported, please try a newer!")
     endif()
 endif()
 

include/pmacc/boost_workaround.hpp

@@ -29,7 +29,11 @@
 /* workaround for compile error with clang-cuda
  * boost/type_traits/is_base_and_derived.hpp:142:25: error: invalid application of 'sizeof' to an incomplete type
  * 'boost::in_place_factory_base' BOOST_STATIC_ASSERT(sizeof(B) != 0);
+ *
+ * https://github.com/boostorg/config/issues/406#issuecomment-928151025
  */
+#include <boost/utility/in_place_factory.hpp>
+#include <boost/utility/typed_in_place_factory.hpp>
 #include <boost/optional/optional.hpp>
 
 #if defined(__clang__) && defined(__CUDACC__)

include/pmacc/debug/VerboseLog.hpp

@@ -21,6 +21,8 @@
 
 #pragma once
 
+#include <pmacc/boost_workaround.hpp>
+
 #include "pmacc/debug/VerboseLogMakros.hpp"
 
 #include <boost/format.hpp>

include/pmacc/mappings/simulation/Filesystem.cpp

@@ -21,6 +21,8 @@
 
 #include "pmacc/mappings/simulation/Filesystem.hpp"
 
+#include <pmacc/boost_workaround.hpp>
+
 #include "pmacc/Environment.hpp"
 #include "pmacc/mappings/simulation/GridController.hpp"
 

include/pmacc/types.hpp

@@ -41,6 +41,8 @@
 #endif
 
 
+#include <pmacc/boost_workaround.hpp>
+
 #include "pmacc/attribute/Constexpr.hpp"
 #include "pmacc/attribute/FunctionSpecifier.hpp"
 #include "pmacc/cuplaHelper/ValidateCall.hpp"

share/ci/compiler_clang.yml

@@ -1,15 +1,16 @@
 ################################################################################
-#   [clang++-X] : X = {4.0, 5.0, 6.0, 7, 8, 9, 10, 11}
+#   [clang++-X] : X = {10, 11, 12, 13, 14, 15, 16}
 
 .base_clang:
-  image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-clang-pic:${CONTAINER_TAG}
+  image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-pic:${CONTAINER_TAG}
   variables:
     GIT_SUBMODULE_STRATEGY: normal
     # ISAAC is not supporting CPU
     DISABLE_ISAAC: "yes"
   script:
     - apt update
     - apt install -y curl libjpeg-dev
+    - source $CI_PROJECT_DIR/share/ci/install/clang.sh
     - $CI_PROJECT_DIR/share/ci/git_merge.sh
     - source $CI_PROJECT_DIR/share/ci/bash.profile
     - $CI_PROJECT_DIR/share/ci/run_tests.sh $PIC_TEST_CASE_FOLDER

share/ci/compiler_clang_cuda.yml

@@ -1,9 +1,9 @@
 ################################################################################
-#   [clang++-X] : X = {4.0, 5.0, 6.0, 7, 8, 9, 10, 11}
+#   [clang++-X] : X = {10, 11, 12, 13, 14, 15, 16}
 # cuda9.2Clang is not supporting clang-7
 
 .base_cuda_clang:
-  image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-cuda${CUDA_CONTAINER_VERSION}-clangpic:${CONTAINER_TAG}
+  image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-cuda${CUDA_CONTAINER_VERSION}-pic:${CONTAINER_TAG}
   variables:
     GIT_SUBMODULE_STRATEGY: normal
     PIC_CMAKE_ARGS: "-DCMAKE_CUDA_FLAGS=--no-cuda-version-check"
@@ -12,6 +12,7 @@
   script:
     - apt update
     - apt install -y curl libjpeg-dev
+    - source $CI_PROJECT_DIR/share/ci/install/clang.sh
     - $CI_PROJECT_DIR/share/ci/git_merge.sh
     - source $CI_PROJECT_DIR/share/ci/bash.profile
     - $CI_PROJECT_DIR/share/ci/run_tests.sh $PIC_TEST_CASE_FOLDER

share/ci/compiler_gcc.yml

@@ -1,5 +1,5 @@
 ################################################################################
-#   [g++-X] : X = {7, 8, 9 ,10}
+#   [g++-X] : X = {7, 8, 9 ,10, 11}
 
 .base_gcc:
   image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-gcc-pic:${CONTAINER_TAG}
@@ -10,6 +10,7 @@
   script:
     - apt update
     - apt install -y curl libjpeg-dev
+    - source $CI_PROJECT_DIR/share/ci/install/gcc.sh
     - $CI_PROJECT_DIR/share/ci/git_merge.sh
     - source $CI_PROJECT_DIR/share/ci/bash.profile
     - $CI_PROJECT_DIR/share/ci/run_tests.sh $PIC_TEST_CASE_FOLDER

share/ci/compiler_hipcc.yml

@@ -22,7 +22,7 @@
     - rocm-smi
     - hipcc --version
     # AMD container keys are outdated and must be updated
-    - wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
+    - wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
     - apt -y update
     - apt install -y curl libjpeg-dev
     - $CI_PROJECT_DIR/share/ci/git_merge.sh

share/ci/compiler_nvcc_cuda.yml

@@ -1,5 +1,5 @@
 ################################################################################
-#   [g++-X] : X = {5, 6, 7, 8, 9, 10}
+#   [g++-X] : X = {7, 8, 9, 10, 11}
 
 .base_nvcc:
   image: registry.hzdr.de/crp/alpaka-group-container/alpaka-ci-${CI_CONTAINER_NAME}-cuda${CUDA_CONTAINER_VERSION}-gccpic:${CONTAINER_TAG}
@@ -11,6 +11,7 @@
   script:
     - apt update
     - apt install -y curl libjpeg-dev
+    - source $CI_PROJECT_DIR/share/ci/install/gcc.sh
     - $CI_PROJECT_DIR/share/ci/git_merge.sh
     - source $CI_PROJECT_DIR/share/ci/bash.profile
     - $CI_PROJECT_DIR/share/ci/run_tests.sh $PIC_TEST_CASE_FOLDER

share/ci/install/clang.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+# merge the PR to the latest version of the destination branch
+
+cd $CI_PROJECT_DIR
+
+clang_version=$(echo $CXX_VERSION | tr -d "clang++-")
+echo "Clang-version: $clang_version"
+
+if ! agc-manager -e clang@${clang_version}; then
+    apt install -y clang-${clang_version}
+else
+    CLANG_BASE_PATH="$(agc-manager -b clang@${clang_version})"
+    export PATH=$CLANG_BASE_PATH/bin:$PATH
+fi

share/ci/install/gcc.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+# merge the PR to the latest version of the destination branch
+
+cd $CI_PROJECT_DIR
+
+gcc_version=$(echo $CXX_VERSION | tr -d "g++-")
+echo "GCC-version: $gcc_version"
+
+if ! agc-manager -e gcc@${GCC_version}; then
+    apt install -y gcc-${GCC_version}
+else
+    GCC_BASE_PATH="$(agc-manager -b gcc@${GCC_version})"
+    export PATH=$GCC_BASE_PATH/bin:$PATH
+fi

share/ci/n_wise_generator.py

@@ -123,15 +123,18 @@ def is_valid_combination(row):
         if is_clang_cuda:
             if not is_cuda:
                 return False
+            # clang cuda containers shipped without clang-11 and clang-12
+            # https://codebase.helmholtz.cloud/crp/alpaka-group-container/-/issues/28#note_3118676
+            if v_compiler == 11 or v_compiler == 12:
+                return False
             # native CMake CUDA compile is not supporting clang 8
             if v_compiler == 8:
                 return False
             if v_cuda == 10.0 and v_compiler >= 8:
                 return True
             if v_cuda == 10.1 and v_compiler >= 9:
                 return True
-            # CUDA 11.4 is the last container called '*cuda114-clang'
-            if 11.0 <= v_cuda <= 11.4 and v_compiler >= 11:
+            if 11.0 <= v_cuda <= 11.6 and v_compiler >= 11:
                 return True
 
             return False
@@ -154,8 +157,7 @@ def is_valid_combination(row):
                 # for C++17 support CUDA >= 11 is required
                 if v_cuda == 11.0 and v_compiler <= 9:
                     return True
-                # CUDA 11.4 is the last container called '*cuda114-clang'
-                if 11.1 <= v_cuda <= 11.4 and v_compiler <= 10:
+                if 11.1 <= v_cuda <= 11.6 and v_compiler <= 10:
                     return True
 
             return False
@@ -165,10 +167,9 @@ def is_valid_combination(row):
 
 # compiler list
 # tuple with two components (compiler name, version)
-clang_compiers = [("clang++", 6.0), ("clang++", 7),
-                  ("clang++", 8), ("clang++", 9), ("clang++", 10),
-                  ("clang++", 11), ("clang++", 12), ("clang++", 13)]
-gnu_compilers = [("g++", 7), ("g++", 8), ("g++", 9), ("g++", 10)]
+clang_compiers = [("clang++", 10), ("clang++", 11), ("clang++", 12),
+                  ("clang++", 13), ("clang++", 14), ("clang++", 15)]
+gnu_compilers = [("g++", 9), ("g++", 10), ("g++", 11)]
 compilers = [
     clang_compiers,
     gnu_compilers
@@ -198,14 +199,14 @@ def is_valid_combination(row):
 # PIConGPU backend list
 # tuple with two components (backend name, version)
 # version is only required for the cuda backend
-backends = [("hip", 4.3), ("hip", 4.5), ("hip", 5.0),
-            ("hip", 5.1), ("hip", 5.2),
+backends = [("hip", 5.0), ("hip", 5.1), ("hip", 5.2),
             ("cuda", 10.0), ("cuda", 10.1), ("cuda", 10.2),
             ("cuda", 11.0), ("cuda", 11.1), ("cuda", 11.2),
             ("cuda", 11.3), ("cuda", 11.4),
             ("omp2b", ), ("serial", )]
 
-boost_libs_all = ["1.74.0", "1.75.0"]
+boost_libs_all = ["1.74.0", "1.75.0", "1.76.0",
+                  "1.77.0", "1.78.0"]
 
 operating_system = [("ubuntu", 18.04), ("ubuntu", 20.04)]
 
@@ -296,7 +297,7 @@ def is_valid_combination(row):
             if backend == "hip":
                 print("    - wget -q -O - "
                       "https://repo.radeon.com/rocm/rocm.gpg.key | "
-                      "sudo apt-key add -")
+                      "apt-key add -")
             if backend == "cuda":
                 print("    - apt-key adv --fetch-keys "
                       "https://developer.download.nvidia.com/compute"

share/ci/run_picongpu_tests.sh

@@ -41,6 +41,7 @@ CMAKE_ARGS="${PIC_CONST_ARGS} ${PIC_CMAKE_ARGS} -DCMAKE_CXX_COMPILER=${CXX_VERSI
 
 # check and activate if clang should be used as CUDA device compiler
 if [ -n "$CI_CLANG_AS_CUDA_COMPILER" ] ; then
+  export PATH="$(agc-manager -b cuda)/bin:$PATH"
   CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_COMPILER=${CXX_VERSION}"
 fi
 

share/ci/run_pmacc_tests.sh

@@ -24,6 +24,7 @@ CMAKE_ARGS="$CMAKE_ARGS -DUSE_MPI_AS_ROOT_USER=ON"
 
 # check and activate if clang should be used as CUDA device compiler
 if [ -n "$CI_CLANG_AS_CUDA_COMPILER" ] ; then
+    export PATH="$(agc-manager -b cuda)/bin:$PATH"
     CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_COMPILER=${CXX_VERSION}"
 fi