From 407a07d7a97418c08452061b8b9ca2b8fe06f66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20And=C3=A9n?= Date: Wed, 12 Jul 2023 22:40:26 -0500 Subject: [PATCH 01/10] tests: Make separate build script --- tools/cufinufft/build-library.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 tools/cufinufft/build-library.sh diff --git a/tools/cufinufft/build-library.sh b/tools/cufinufft/build-library.sh new file mode 100755 index 000000000..988096879 --- /dev/null +++ b/tools/cufinufft/build-library.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e -u -x + +mkdir -p /io/build +cd /io/build + +cmake -D FINUFFT_USE_CUDA=ON \ + -D FINUFFT_USE_CPU=OFF \ + -D FINUFFT_BUILD_TESTS=ON \ + -D CMAKE_CUDA_ARCHITECTURES="35;50;60;70;75;80" \ + -D BUILD_TESTING=ON \ + .. + +make -j4 From bf84f2b64d8e62b6d9d80608f8a95c1793422028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20And=C3=A9n?= Date: Wed, 12 Jul 2023 22:41:04 -0500 Subject: [PATCH 02/10] tests: Remove compilation in cu110 Dockerfile --- .../docker/cuda11.0/Dockerfile-x86_64 | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 b/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 index 3ac5b8561..d9a7bb270 100644 --- a/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 +++ b/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 @@ -67,7 +67,9 @@ RUN yum install -y \ devtoolset-9-gcc-c++ && \ rm -rf /var/cache/yum/* -RUN scl enable devtoolset-9 -- g++ --version +ENV PATH /opt/rh/devtoolset-9/root/usr/bin:${PATH} + +RUN g++ --version # finufft reqs RUN yum install -y \ @@ -86,22 +88,11 @@ COPY perftest /io/perftest COPY examples /io/examples COPY cmake /io/cmake COPY contrib /io/contrib -COPY CMakeLists.txt /io/ - -# Set up cmake and build the code. -RUN mkdir -p /io/build -WORKDIR /io/build -RUN scl enable devtoolset-9 -- cmake -D FINUFFT_USE_CUDA=ON -D FINUFFT_USE_CPU=OFF -D CMAKE_CUDA_ARCHITECTURES="35;50;60;70;75;80" -DBUILD_TESTING=ON -DFINUFFT_BUILD_TESTS=ON .. -RUN scl enable devtoolset-9 -- make -j4 - -WORKDIR /io - -# And we need to pack it in our LD path -ENV LD_LIBRARY_PATH /io/build:${LD_LIBRARY_PATH} - -# Now copy the rest of the repository in. COPY python /io/python COPY docs /io/docs COPY tools /io/tools +COPY CMakeLists.txt /io/ + +WORKDIR /io CMD ["/bin/bash"] From 599a50058c6828408db2d63891a873713149a4ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20And=C3=A9n?= Date: Wed, 12 Jul 2023 22:41:18 -0500 Subject: [PATCH 03/10] tests: add library compilation step to Jenkins --- Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 89951e3ba..a83f4495a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,10 @@ pipeline { sh '''#!/bin/bash -ex nvidia-smi ''' + sh '''#!/bin/bash -ex + /io/tools/cufinufft/build-library.sh + cp /io/build/libcufinufft.so /usr/lib + ''' sh '''#!/bin/bash -ex cp -r /io/build/test/cuda cuda_tests cd cuda_tests From 661f3bd8e0c29d296ef73712e83445160b2a7512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20And=C3=A9n?= Date: Wed, 12 Jul 2023 22:41:52 -0500 Subject: [PATCH 04/10] cuda: update distribution helper script --- tools/cufinufft/distribution_helper.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tools/cufinufft/distribution_helper.sh b/tools/cufinufft/distribution_helper.sh index 901473dad..014d51ecd 100755 --- a/tools/cufinufft/distribution_helper.sh +++ b/tools/cufinufft/distribution_helper.sh @@ -10,19 +10,32 @@ dockerhub=janden echo "# build the wheel" docker build \ - --file ci/docker/cuda${cuda_version}/Dockerfile-x86_64 \ + --file tools/cufinufft/docker/cuda${cuda_version}/Dockerfile-x86_64 \ --tag ${dockerhub}/cufinufft-${cufinufft_version}-${manylinux_version} . -echo "# Run the container, invoking the build-wheels script to generate the wheels" -docker run \ +echo "# Create the container and start it" +docker create \ --gpus all \ --interactive \ --tty \ --volume $(pwd)/wheelhouse:/io/wheelhouse \ --env PLAT=${manylinux_version}_x86_64 \ - ${dockerhub}/cufinufft-${cufinufft_version}-${manylinux_version} \ - /io/ci/build-wheels.sh + --name cufinufft \ + ${dockerhub}/cufinufft-${cufinufft_version}-${manylinux_version} + +docker start cufinufft + +echo "# Build the library and install it" +docker exec cufinufft /io/tools/cufinufft/build-library.sh +docker exec cufinufft cp /io/build/libcufinufft.so /usr/lib + +echo "# Build the wheels" +docker exec cufinufft /io/tools/cufinufft/build-wheels.sh + +echo "# Shut down the container and remove it" +docker stop cufinufft +docker rm cufinufft echo "# Copy the wheels we care about to the dist folder" mkdir -p dist From d710ccabe8f43591301c4c85f0d68abb647cc1aa Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Thu, 27 Jul 2023 14:20:08 -0400 Subject: [PATCH 05/10] Pass at removing build entirely from dockerfile --- Jenkinsfile | 20 +++++++++++----- .../docker/cuda11.0/Dockerfile-x86_64 | 23 ------------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a83f4495a..6cac1b99b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,27 +14,35 @@ pipeline { } } environment { - HOME = "$WORKSPACE/build" + HOME = "$WORKSPACE" PYBIN = "/opt/python/cp38-cp38/bin" + LIBRARY_PATH = "$WORKSPACE/build" + LD_LIBRARY_PATH = "$WORKSPACE/build" } steps { sh '''#!/bin/bash -ex nvidia-smi ''' sh '''#!/bin/bash -ex - /io/tools/cufinufft/build-library.sh - cp /io/build/libcufinufft.so /usr/lib + echo $HOME ''' sh '''#!/bin/bash -ex - cp -r /io/build/test/cuda cuda_tests - cd cuda_tests + cmake -B build . -DFINUFFT_USE_CUDA=ON \ + -DFINUFFT_USE_CPU=OFF \ + -DFINUFFT_BUILD_TESTS=ON \ + -DCMAKE_CUDA_ARCHITECTURES="35;50;60;70;75;80" \ + -DBUILD_TESTING=ON + ninja -C build -j 4 + ''' + sh '''#!/bin/bash -ex + cd build/test/cuda ctest --output-on-failure ''' sh '${PYBIN}/python3 -m venv $HOME' sh '''#!/bin/bash -ex source $HOME/bin/activate python3 -m pip install --upgrade pip - LIBRARY_PATH=/io/build python3 -m pip install -e python/cufinufft + python3 -m pip install -e python/cufinufft python3 -m pip install pytest python3 -m pytest ''' diff --git a/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 b/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 index d9a7bb270..4b09fc392 100644 --- a/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 +++ b/tools/cufinufft/docker/cuda11.0/Dockerfile-x86_64 @@ -69,30 +69,7 @@ RUN yum install -y \ ENV PATH /opt/rh/devtoolset-9/root/usr/bin:${PATH} -RUN g++ --version - # finufft reqs RUN yum install -y \ cmake && \ rm -rf /var/cache/yum/* - -# Okay, so now we can begin cufinufft -# -# Only copy the C++/CUDA code (this way the cache is not invalidated by changes -# to the Python code or to the docs). -COPY src /io/src -COPY fortran /io/fortran -COPY include /io/include -COPY test /io/test -COPY perftest /io/perftest -COPY examples /io/examples -COPY cmake /io/cmake -COPY contrib /io/contrib -COPY python /io/python -COPY docs /io/docs -COPY tools /io/tools -COPY CMakeLists.txt /io/ - -WORKDIR /io - -CMD ["/bin/bash"] From 519c0b24c4100c2611b4e5b4f0fa0066a96a41e4 Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Wed, 26 Jul 2023 08:41:40 -0400 Subject: [PATCH 06/10] tests: switch build from ninja to makefile --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6cac1b99b..000461a7d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,8 @@ pipeline { -DFINUFFT_BUILD_TESTS=ON \ -DCMAKE_CUDA_ARCHITECTURES="35;50;60;70;75;80" \ -DBUILD_TESTING=ON - ninja -C build -j 4 + cd build + make -j4 ''' sh '''#!/bin/bash -ex cd build/test/cuda From 5516d0308d18fd386d0d5d39d0dd0b2c0419b367 Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Wed, 26 Jul 2023 08:56:10 -0400 Subject: [PATCH 07/10] tests-cuda: call pytest on proper tests directory --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 000461a7d..a27816064 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { python3 -m pip install --upgrade pip python3 -m pip install -e python/cufinufft python3 -m pip install pytest - python3 -m pytest + python3 -m pytest python/cufinufft ''' } } From b28479ec8c8f8ad58f9bd1eb73b34aed3646f557 Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Thu, 27 Jul 2023 14:29:26 -0400 Subject: [PATCH 08/10] cuda-jenkins: only compile for local cuda arch --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a27816064..bc7c64f02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,10 +27,11 @@ pipeline { echo $HOME ''' sh '''#!/bin/bash -ex + cuda_arch=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader|head -n 1| sed "s/\\.//") cmake -B build . -DFINUFFT_USE_CUDA=ON \ -DFINUFFT_USE_CPU=OFF \ -DFINUFFT_BUILD_TESTS=ON \ - -DCMAKE_CUDA_ARCHITECTURES="35;50;60;70;75;80" \ + -DCMAKE_CUDA_ARCHITECTURES="$cuda_arch" \ -DBUILD_TESTING=ON cd build make -j4 From 69f05c8dde68df989881fbcf9aad64769bff0951 Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Thu, 27 Jul 2023 14:45:50 -0400 Subject: [PATCH 09/10] cuda-performance: change default evaluator to horner and minor tweaks --- src/cuda/3d/spread3d_wrapper.cu | 2 +- src/cuda/cufinufft.cu | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cuda/3d/spread3d_wrapper.cu b/src/cuda/3d/spread3d_wrapper.cu index 9ae0d4d56..f652411f8 100644 --- a/src/cuda/3d/spread3d_wrapper.cu +++ b/src/cuda/3d/spread3d_wrapper.cu @@ -191,7 +191,7 @@ int cuspread3d_nuptsdriven(int nf1, int nf2, int nf3, int M, cufinufft_plan_t cuda_complex *d_c = d_plan->c; cuda_complex *d_fw = d_plan->fw; - threadsPerBlock.x = 16; + threadsPerBlock.x = 64; threadsPerBlock.y = 1; blocks.x = (M + threadsPerBlock.x - 1) / threadsPerBlock.x; blocks.y = 1; diff --git a/src/cuda/cufinufft.cu b/src/cuda/cufinufft.cu index 64a37310d..4f09be66c 100644 --- a/src/cuda/cufinufft.cu +++ b/src/cuda/cufinufft.cu @@ -106,7 +106,7 @@ int cufinufft_default_opts(int type, int dim, cufinufft_opts *opts) switch (dim) { case 1: { - opts->gpu_kerevalmeth = 0; // using exp(sqrt()) + opts->gpu_kerevalmeth = 1; // using horner if (type == 1) { opts->gpu_method = 2; } @@ -120,7 +120,7 @@ int cufinufft_default_opts(int type, int dim, cufinufft_opts *opts) } } break; case 2: { - opts->gpu_kerevalmeth = 0; // using exp(sqrt()) + opts->gpu_kerevalmeth = 1; // using horner if (type == 1) { opts->gpu_method = 2; } @@ -134,7 +134,7 @@ int cufinufft_default_opts(int type, int dim, cufinufft_opts *opts) } } break; case 3: { - opts->gpu_kerevalmeth = 0; // using exp(sqrt()) + opts->gpu_kerevalmeth = 1; // using horner if (type == 1) { opts->gpu_method = 2; } From c714ff2c60a0e0594b6e1fdd257eb836378d3ab5 Mon Sep 17 00:00:00 2001 From: Robert Blackwell Date: Thu, 27 Jul 2023 14:55:38 -0400 Subject: [PATCH 10/10] cuda-performance: revert minor tweaks -- flawed analysis :( --- src/cuda/3d/spread3d_wrapper.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cuda/3d/spread3d_wrapper.cu b/src/cuda/3d/spread3d_wrapper.cu index f652411f8..9ae0d4d56 100644 --- a/src/cuda/3d/spread3d_wrapper.cu +++ b/src/cuda/3d/spread3d_wrapper.cu @@ -191,7 +191,7 @@ int cuspread3d_nuptsdriven(int nf1, int nf2, int nf3, int M, cufinufft_plan_t cuda_complex *d_c = d_plan->c; cuda_complex *d_fw = d_plan->fw; - threadsPerBlock.x = 64; + threadsPerBlock.x = 16; threadsPerBlock.y = 1; blocks.x = (M + threadsPerBlock.x - 1) / threadsPerBlock.x; blocks.y = 1;