From 084db7b81822ca9cc8584513572bf82aa061936c Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 15 Jun 2023 13:13:34 -0700 Subject: [PATCH 1/4] cuda related container updates --- azure-pipelines.yml | 14 +++++++++++++- .../Dockerfile | 2 +- .../build.sh | 2 +- .../Dockerfile | 2 +- .../build.sh | 2 +- .../docker_env_setup.sh | 0 .../docker_uberenv_build.sh | 0 7 files changed, 17 insertions(+), 5 deletions(-) rename scripts/ci/docker/{alpinedav_ubuntu_22.04_cuda_11.8.0_devel => alpinedav_ubuntu_20.04_cuda_12.1.1_devel}/Dockerfile (96%) rename scripts/ci/docker/{alpinedav_ubuntu_22.04_cuda_11.8.0_devel => alpinedav_ubuntu_20.04_cuda_12.1.1_devel}/build.sh (84%) rename scripts/ci/docker/{_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls => alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls}/Dockerfile (90%) rename scripts/ci/docker/{_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls => alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls}/build.sh (82%) rename scripts/ci/docker/{_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls => alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls}/docker_env_setup.sh (100%) rename scripts/ci/docker/{_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls => alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls}/docker_uberenv_build.sh (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec371bd60..41a6b1edd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,8 @@ variables: ubuntu_22_04_tag: alpinedav/ascent-ci:ubuntu-22.04-devel-tpls_2023-05-05-shaf57286 ubuntu_18_04_cuda_11_4_0_tag: alpinedav/ascent-ci:ubuntu-18.04-cuda-11.4.0-devel-tpls_2023-05-05-shaf57286 ubuntu_20_04_rocm_5_1_3_tag: alpinedav/ascent-ci:ubuntu-20.04-rocm-5.1.3-devel-tpls_2023-05-05-shaf57286 - + ubuntu_20_04_cuda_12_1_1_tag: alpinedav/ascent-ci:ubuntu-20.04-cuda-12.1.1-devel-tpls_2023-06-13-sha647407 + # only build merge target pr to develop trigger: none @@ -438,6 +439,17 @@ stages: COMPILER_FC: gfortran strategy: matrix: + ubuntu_20.04_static_cuda_12.1.1: + containerImage: ${{ variables.ubuntu_20_04_cuda_12_1_1_tag }} + BUILD_SHARED_LIBS: OFF + CMAKE_BUILD_TYPE: Release + ENABLE_DRAY: ON + ENABLE_MFEM: ON + BLT_CXX_STD: c++14 + # static linking unit tests eats up alot of disk space, + # so skip building tests in CI to keep us away from clif + ENABLE_TESTS: OFF + ubuntu_18.04_static_cuda_11.4.0: containerImage: ${{ variables.ubuntu_18_04_cuda_11_4_0_tag }} BUILD_SHARED_LIBS: OFF diff --git a/scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/Dockerfile similarity index 96% rename from scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/Dockerfile rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/Dockerfile index 3baa64fb0..671cbbfae 100644 --- a/scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/Dockerfile +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/Dockerfile @@ -2,7 +2,7 @@ # Project developers. See top-level LICENSE AND COPYRIGHT files for dates and # other details. No copyright assignment is required to contribute to Ascent. -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 +FROM nvidia/cuda:12.1.1-devel-ubuntu20.04 # add sudo to base cuda devel env # so we can install additional packages as diff --git a/scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/build.sh similarity index 84% rename from scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/build.sh rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/build.sh index 77a3a26ed..4a82abe26 100644 --- a/scripts/ci/docker/alpinedav_ubuntu_22.04_cuda_11.8.0_devel/build.sh +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel/build.sh @@ -4,7 +4,7 @@ # other details. No copyright assignment is required to contribute to Ascent. set -ev -export TAG_NAME=alpinedav/ascent-ci:ubuntu-22.04-cuda-11.8.0-devel +export TAG_NAME=alpinedav/ascent-ci:ubuntu-20.04-cuda-12.1.1-devel # exec docker build to create image echo "docker build -t ${TAG_NAME} ." diff --git a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/Dockerfile similarity index 90% rename from scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/Dockerfile rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/Dockerfile index 6794ba710..bdae1f7c7 100644 --- a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/Dockerfile +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/Dockerfile @@ -2,7 +2,7 @@ # Project developers. See top-level LICENSE AND COPYRIGHT files for dates and # other details. No copyright assignment is required to contribute to Ascent. -FROM alpinedav/ascent-ci:ubuntu-22.04-cuda-11.8.0-devel +FROM alpinedav/ascent-ci:ubuntu-20.04-cuda-12.1.1-devel # obtain a copy of ascent source from host env, # which we use to call uberenv diff --git a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/build.sh similarity index 82% rename from scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/build.sh rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/build.sh index db28aec36..1070d45dc 100644 --- a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/build.sh +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/build.sh @@ -4,7 +4,7 @@ # other details. No copyright assignment is required to contribute to Ascent. set -ev -export TAG_BASE=alpinedav/ascent-ci:ubuntu-22.04-cuda-11.8.0-devel-tpls +export TAG_BASE=alpinedav/ascent-ci:ubuntu-20.04-cuda-12.1.1-devel-tpls date diff --git a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/docker_env_setup.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/docker_env_setup.sh similarity index 100% rename from scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/docker_env_setup.sh rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/docker_env_setup.sh diff --git a/scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/docker_uberenv_build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/docker_uberenv_build.sh similarity index 100% rename from scripts/ci/docker/_alpinedav_ubuntu_22.04_cuda_11.8.0_devel_tpls/docker_uberenv_build.sh rename to scripts/ci/docker/alpinedav_ubuntu_20.04_cuda_12.1.1_devel_tpls/docker_uberenv_build.sh From 050c6271889c3e90c97f7124eb828f110df53c51 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 15 Jun 2023 15:41:26 -0700 Subject: [PATCH 2/4] hip ci build config change --- azure-pipelines.yml | 2 +- .../docker_uberenv_build.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 41a6b1edd..61d61ab92 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -609,7 +609,7 @@ stages: ${CMAKE_EXE} --version cat /ascent-config.cmake # configure - ${CMAKE_EXE} -S src -B build -C /ascent-config.cmake -DCMAKE_INSTALL_PREFIX=install + ${CMAKE_EXE} -S src -B build -C /ascent-config.cmake -DENABLE_TESTS=ON -DCMAKE_INSTALL_PREFIX=install displayName: 'Configure with CMake' - script: | diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_5.1.3_devel_tpls/docker_uberenv_build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_5.1.3_devel_tpls/docker_uberenv_build.sh index 4fa3536f1..8da2723af 100755 --- a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_5.1.3_devel_tpls/docker_uberenv_build.sh +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_5.1.3_devel_tpls/docker_uberenv_build.sh @@ -13,7 +13,8 @@ export PATH=$PATH:/${cmake_install_dir}/bin/ # build rocm tpls with helper script chmod +x ascent/scripts/build_ascent/build_ascent.sh -env build_ascent=false enable_hip=true ascent/scripts/build_ascent/build_ascent.sh +# enable tests is of b/c one of the vtk-m tests fails to link +env enable_tests=OFF build_ascent=false enable_hip=ON ascent/scripts/build_ascent/build_ascent.sh ############################ # TODO: get spack working From 370c0cff6fa5453a2ba2cc43c3cd6533c7e3ef27 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 15 Jun 2023 16:20:03 -0700 Subject: [PATCH 3/4] simplify dray config header logic --- src/libs/dray/CMakeLists.txt | 19 ++++++++++++++----- src/libs/dray/dray_config.h.in | 14 ++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libs/dray/CMakeLists.txt b/src/libs/dray/CMakeLists.txt index a5fd688f7..601ee5af4 100644 --- a/src/libs/dray/CMakeLists.txt +++ b/src/libs/dray/CMakeLists.txt @@ -4,6 +4,20 @@ # adjust cmake options into proper defs for generate dray_config.h +if(ENABLE_OPENMP) + set(DRAY_OPENMP_ENABLED TRUE) + message(STATUS "DRay OpenMP Support Enabled") +endif() + +if(ENABLE_CUDA) + set(DRAY_CUDA_ENABLED TRUE) + message(STATUS "DRay CUDA Support Enabled") +endif() + +if(ENABLE_HIP) + set(DRAY_HIP_ENABLED TRUE) + message(STATUS "DRay HIP Support Enabled") +endif() if(MFEM_FOUND) set(DRAY_MFEM_ENABLED TRUE) @@ -20,11 +34,6 @@ if(DRAY_USE_DOUBLE_PRECISION) message(STATUS "DRay using double precision") endif() -if(ENABLE_OPENMP) - set(DRAY_OPENMP_ENABLED TRUE) - message(STATUS "DRay OpenMP Support Enabled") -endif() - if(DRAY_ENABLE_STATS) set(DRAY_STATS TRUE) message(STATUS "DRay Stats enabled") diff --git a/src/libs/dray/dray_config.h.in b/src/libs/dray/dray_config.h.in index 56bc76d9e..8840d8f19 100644 --- a/src/libs/dray/dray_config.h.in +++ b/src/libs/dray/dray_config.h.in @@ -7,24 +7,18 @@ #define DRAY_CONFIG_H // config time defines -#cmakedefine DRAY_MFEM_ENABLED #cmakedefine DRAY_OPENMP_ENABLED +#cmakedefine DRAY_CUDA_ENABLED +#cmakedefine DRAY_HIP_ENABLED +#cmakedefine DRAY_MFEM_ENABLED #cmakedefine DRAY_ENABLE_LOGGING #cmakedefine DRAY_USE_DOUBLE_PRECISION #cmakedefine DRAY_STATS #include -#if defined(__CUDACC__) && !defined(DEBUG_CPU_ONLY) - -#define DRAY_CUDA_ENABLED -#define DRAY_EXEC RAJA_INLINE RAJA_HOST_DEVICE -#define DRAY_EXEC_ONLY RAJA_INLINE RAJA_DEVICE -#define DRAY_LAMBDA RAJA_DEVICE - -#elif defined(__HIP_PLATFORM_HCC__) && !defined(DEBUG_CPU_ONLY) +#if defined(DRAY_CUDA_ENABLED) ||defined(DRAY_HIP_ENABLED) -#define DRAY_HIP_ENABLED #define DRAY_EXEC RAJA_INLINE RAJA_HOST_DEVICE #define DRAY_EXEC_ONLY RAJA_INLINE RAJA_DEVICE #define DRAY_LAMBDA RAJA_DEVICE From 7373b965f4d90000fa40a413a908cbfdcd400957 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 15 Jun 2023 17:07:00 -0700 Subject: [PATCH 4/4] container update --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 61d61ab92..ee9fa15b3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,10 +21,9 @@ variables: ubuntu_20_04_tag: alpinedav/ascent-ci:ubuntu-20.04-devel-tpls_2023-05-05-shaf57286 ubuntu_22_04_tag: alpinedav/ascent-ci:ubuntu-22.04-devel-tpls_2023-05-05-shaf57286 ubuntu_18_04_cuda_11_4_0_tag: alpinedav/ascent-ci:ubuntu-18.04-cuda-11.4.0-devel-tpls_2023-05-05-shaf57286 - ubuntu_20_04_rocm_5_1_3_tag: alpinedav/ascent-ci:ubuntu-20.04-rocm-5.1.3-devel-tpls_2023-05-05-shaf57286 + ubuntu_20_04_rocm_5_1_3_tag: alpinedav/ascent-ci:ubuntu-20.04-rocm-5.1.3-devel-tpls_2023-06-15-sha370c0c ubuntu_20_04_cuda_12_1_1_tag: alpinedav/ascent-ci:ubuntu-20.04-cuda-12.1.1-devel-tpls_2023-06-13-sha647407 - # only build merge target pr to develop trigger: none pr: