From a53d0321f971318d73add7b077269d13725e1c31 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 14:33:18 -0500 Subject: [PATCH 01/34] Add support for nvcc-specific matrix. --- .github/workflows/compute-matrix.sh | 35 +++++++++++++++++ ...ute-matrix.yml => compute-nvcc-matrix.yml} | 11 +----- .github/workflows/pr.yml | 30 +++++++------- ci/matrix.yaml | 39 +++++++++---------- 4 files changed, 71 insertions(+), 44 deletions(-) create mode 100755 .github/workflows/compute-matrix.sh rename .github/workflows/{compute-matrix.yml => compute-nvcc-matrix.yml} (69%) diff --git a/.github/workflows/compute-matrix.sh b/.github/workflows/compute-matrix.sh new file mode 100755 index 0000000000..3133872619 --- /dev/null +++ b/.github/workflows/compute-matrix.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euo pipefail + +# Ensure the script is being executed in its containing directory +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; + +# Check for the correct number of arguments +if [ $# -ne 2 ]; then + echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" + echo "MATRIX_FILE: The path to the matrix file." + echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"'" + exit 1 +fi + +MATRIX_FILE="$1" +MATRIX_QUERY="$2" +MATRIX_QUERY='.["'"$MATRIX_QUERY"'"]["nvcc"]' + +# Use /dev/null as the default value for GITHUB_OUTPUT if it isn't set, i.e., not running in GitHub Actions +GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" + +echo "Input matrix file:" +cat "$MATRIX_FILE" +echo "Query: $MATRIX_QUERY" +echo $(yq -o=json "$MATRIX_FILE" | jq -c -r "$MATRIX_QUERY | map(. as \$o | {std: .std[]} + del(\$o.std))") + +#FULL_MATRIX=$(yq -o=json "$MATRIX_FILE" | jq -c --arg matrix_type "$MATRIX_QUERY" '[ .[$matrix_type][] | . as $o | {std: .std[]} + del($o.std)]') +#echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT" +#CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') +#echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" +#COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') +#echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT" +#PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add') +#echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml similarity index 69% rename from .github/workflows/compute-matrix.yml rename to .github/workflows/compute-nvcc-matrix.yml index bbd09e628f..efbd43b92a 100644 --- a/.github/workflows/compute-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -54,13 +54,4 @@ jobs: - name: Compute matrix id: compute-matrix run: | - echo "Input matrix file:" - cat "$MATRIX_FILE" - FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg matrix_type "$MATRIX_TYPE" '[ .[$matrix_type][] | . as $o | {std: .std[]} + del($o.std)]') - echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT" - CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') - echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" - COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') - echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT" - PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add') - echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT" \ No newline at end of file + compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8708cc66e0..cd0d349d17 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,51 +18,53 @@ concurrency: cancel-in-progress: true jobs: - compute-matrix: - uses: ./.github/workflows/compute-matrix.yml + compute-nvcc-matrix: + name: Compute NVCC matrix + runs-on: ubuntu-latest + uses: ./.github/workflows/compute-nvcc-matrix.yml with: matrix_file: "./ci/matrix.yaml" matrix_type: "pull-request" thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: compute-matrix + needs: compute-nvcc-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_thrust.sh" test_script: "./ci/test_thrust.sh" cub: name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: compute-matrix + needs: compute-nvcc-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_cub.sh" test_script: "./ci/test_cub.sh" libcudacxx: name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: compute-matrix + needs: compute-nvcc-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_libcudacxx.sh" test_script: "./ci/test_libcudacxx.sh" diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 0343ad6fe2..a5deddf188 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -22,23 +22,22 @@ gpus: # Configurations that will run for every PR pull-request: -# gcc -- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']} -- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']} -# clang -- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} -- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']} + nvcc: + - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']} + - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']} + - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']} + - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']} From 663e2f4e65e4018060cf023412bcd953f21e3d90 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 14:50:07 -0500 Subject: [PATCH 02/34] Remove errnoeous runs-on --- .github/workflows/pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cd0d349d17..fd75f9869e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,7 +20,6 @@ concurrency: jobs: compute-nvcc-matrix: name: Compute NVCC matrix - runs-on: ubuntu-latest uses: ./.github/workflows/compute-nvcc-matrix.yml with: matrix_file: "./ci/matrix.yaml" From 9405359c923fb419fad7e870f616d6c6c69d36c0 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 15:45:42 -0500 Subject: [PATCH 03/34] Fix path to compute-matrix script. --- .github/workflows/compute-nvcc-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index efbd43b92a..00a2175b01 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -54,4 +54,4 @@ jobs: - name: Compute matrix id: compute-matrix run: | - compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file + ./.github/workflows/compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From be00cb2f976449a3cd7c50bef85bcc451b0e6083 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 16:08:49 -0500 Subject: [PATCH 04/34] debug --- .github/workflows/compute-nvcc-matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 00a2175b01..db38e6c30c 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -54,4 +54,5 @@ jobs: - name: Compute matrix id: compute-matrix run: | + pwd ./.github/workflows/compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From 18debe39b9642798051964b34cb134d6d6f46545 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:12:47 +0000 Subject: [PATCH 05/34] debug. --- .github/workflows/compute-nvcc-matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index db38e6c30c..29889c2018 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -55,4 +55,5 @@ jobs: id: compute-matrix run: | pwd + ls -l ./.github/workflows/compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From 677b8fc8ad74345dbb65e47b16252a4e3bacf466 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:17:21 +0000 Subject: [PATCH 06/34] debug. --- .github/workflows/compute-nvcc-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 29889c2018..948c74dbd0 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -55,5 +55,5 @@ jobs: id: compute-matrix run: | pwd - ls -l + ls -la ./.github/workflows/compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From ee821cc1ba454a31a72db224cf75d666b08ae8e4 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:18:55 +0000 Subject: [PATCH 07/34] lol, use hyphen not underscore. --- .github/workflows/compute-nvcc-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 948c74dbd0..306c642d39 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -56,4 +56,4 @@ jobs: run: | pwd ls -la - ./.github/workflows/compute_matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file + ./.github/workflows/compute-matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From 597cbc2cd916017e69215cd7a55a9ecbb9a2bcd8 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:23:35 +0000 Subject: [PATCH 08/34] Get absolute path for matrix file. --- .github/workflows/compute-nvcc-matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 306c642d39..438f6e23fc 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -56,4 +56,5 @@ jobs: run: | pwd ls -la + MATRIX_FILE=$(realpath MATRIX_FILE) ./.github/workflows/compute-matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From ba3af6883619c305b5d79105c880242ff2135998 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:24:51 +0000 Subject: [PATCH 09/34] Fix reference to matrix_file var. --- .github/workflows/compute-nvcc-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 438f6e23fc..33be6cc927 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -56,5 +56,5 @@ jobs: run: | pwd ls -la - MATRIX_FILE=$(realpath MATRIX_FILE) - ./.github/workflows/compute-matrix.sh $MATRIX_FILE '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file + MATRIX_FILE=$(realpath "${MATRIX_FILE}") + ./.github/workflows/compute-matrix.sh "$MATRIX_FILE" '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From fdf757b9a41f0476cf9104c293f89dc4d196bbeb Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 21:26:19 +0000 Subject: [PATCH 10/34] Remove extraneous matrix_query --- .github/workflows/compute-matrix.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/compute-matrix.sh b/.github/workflows/compute-matrix.sh index 3133872619..07835e6ed0 100755 --- a/.github/workflows/compute-matrix.sh +++ b/.github/workflows/compute-matrix.sh @@ -15,7 +15,6 @@ fi MATRIX_FILE="$1" MATRIX_QUERY="$2" -MATRIX_QUERY='.["'"$MATRIX_QUERY"'"]["nvcc"]' # Use /dev/null as the default value for GITHUB_OUTPUT if it isn't set, i.e., not running in GitHub Actions GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" From 42c4f6f7869224d0bee24a9fa9e39311b4e5f434 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:04:19 +0000 Subject: [PATCH 11/34] Use composite action instead of reusable workflow for matrix. --- .github/actions/compute-matrix/action.yml | 26 ++++++++++++++ .../actions/compute-matrix/compute-matrix.sh | 26 ++++++++++++++ .github/workflows/compute-matrix.sh | 34 ------------------- .github/workflows/pr.yml | 11 +++--- 4 files changed, 59 insertions(+), 38 deletions(-) create mode 100644 .github/actions/compute-matrix/action.yml create mode 100755 .github/actions/compute-matrix/compute-matrix.sh delete mode 100755 .github/workflows/compute-matrix.sh diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml new file mode 100644 index 0000000000..72e3716099 --- /dev/null +++ b/.github/actions/compute-matrix/action.yml @@ -0,0 +1,26 @@ + +name: Compute Matrix +description: "Compute the matrix for a given matrix type from the specified matrix file" + +inputs: + matrix_query: + description: 'The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"' + required: true + matrix_file: + description: 'The file containing the matrix' + required: true +outputs: + matrix: + description: 'The requested matrix' + value: ${{ steps.compute-matrix.outputs.MATRIX }} + +runs: + using: "composite" + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Compute matrix + id: compute-matrix + run: | + MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh "$matrix_file" "$matrix_query") + echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh new file mode 100755 index 0000000000..372f86197d --- /dev/null +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +# Check for the correct number of arguments +if [ $# -ne 2 ]; then + echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" + echo "MATRIX_FILE: The path to the matrix file." + echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"'" + exit 1 +fi + +# Get realpath before changing directory +MATRIX_FILE="realpath $1" +MATRIX_QUERY="$2" + +# Ensure the script is being executed in its containing directory +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; + +# Use /dev/null as the default value for GITHUB_OUTPUT if it isn't set, i.e., not running in GitHub Actions +GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" + +echo "Input matrix file:" >&2 +cat "$MATRIX_FILE" >&2 +echo "Query: $MATRIX_QUERY" >&2 +echo $(yq -o=json "$MATRIX_FILE" | jq -c -r "$MATRIX_QUERY | map(. as \$o | {std: .std[]} + del(\$o.std))") diff --git a/.github/workflows/compute-matrix.sh b/.github/workflows/compute-matrix.sh deleted file mode 100755 index 07835e6ed0..0000000000 --- a/.github/workflows/compute-matrix.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Ensure the script is being executed in its containing directory -cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; - -# Check for the correct number of arguments -if [ $# -ne 2 ]; then - echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" - echo "MATRIX_FILE: The path to the matrix file." - echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"'" - exit 1 -fi - -MATRIX_FILE="$1" -MATRIX_QUERY="$2" - -# Use /dev/null as the default value for GITHUB_OUTPUT if it isn't set, i.e., not running in GitHub Actions -GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" - -echo "Input matrix file:" -cat "$MATRIX_FILE" -echo "Query: $MATRIX_QUERY" -echo $(yq -o=json "$MATRIX_FILE" | jq -c -r "$MATRIX_QUERY | map(. as \$o | {std: .std[]} + del(\$o.std))") - -#FULL_MATRIX=$(yq -o=json "$MATRIX_FILE" | jq -c --arg matrix_type "$MATRIX_QUERY" '[ .[$matrix_type][] | . as $o | {std: .std[]} + del($o.std)]') -#echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT" -#CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') -#echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" -#COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') -#echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT" -#PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add') -#echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index fd75f9869e..24153afc8d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,10 +20,13 @@ concurrency: jobs: compute-nvcc-matrix: name: Compute NVCC matrix - uses: ./.github/workflows/compute-nvcc-matrix.yml - with: - matrix_file: "./ci/matrix.yaml" - matrix_type: "pull-request" + runs-on: ubuntu-latest + steps: + - name: Get full nvcc matrix + uses: ./.github/actions/compute-matrix + with: + matrix_file: "./ci/matrix.yaml" + matrix_query: "'.["pull-request"]["nvcc"]'" thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} From 2cb57031dae9c3a76377fdd43c7c8ba8f6617130 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:06:55 +0000 Subject: [PATCH 12/34] Fix quotes around composite action inputs. --- .github/workflows/pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 24153afc8d..e24e29ef8f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -25,8 +25,8 @@ jobs: - name: Get full nvcc matrix uses: ./.github/actions/compute-matrix with: - matrix_file: "./ci/matrix.yaml" - matrix_query: "'.["pull-request"]["nvcc"]'" + matrix_file: './ci/matrix.yaml' + matrix_query: '.["pull-request"]["nvcc"]' thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} From c61c78f744e467f26e0ceadaee653a1590acd25d Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:10:24 +0000 Subject: [PATCH 13/34] Need to checkout repo to get local composite action. --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e24e29ef8f..23ec0ad7a5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,8 @@ jobs: name: Compute NVCC matrix runs-on: ubuntu-latest steps: + - name: Checkout repo + uses: actions/checkout@v3 - name: Get full nvcc matrix uses: ./.github/actions/compute-matrix with: From 52de697bd0a0780a0fc690a942344884fb19e16d Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:13:11 +0000 Subject: [PATCH 14/34] Fix quotes. --- .github/actions/compute-matrix/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 72e3716099..507361290c 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -4,7 +4,7 @@ description: "Compute the matrix for a given matrix type from the specified matr inputs: matrix_query: - description: 'The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"' + description: 'The jq query used to specify the desired matrix. e.g., .["pull-request"]["nvcc]"' required: true matrix_file: description: 'The file containing the matrix' @@ -17,8 +17,6 @@ outputs: runs: using: "composite" steps: - - name: Checkout repo - uses: actions/checkout@v3 - name: Compute matrix id: compute-matrix run: | From ed7200142b72f9e18d266579ad12b31909ce1eb2 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:15:28 +0000 Subject: [PATCH 15/34] Add shell to composite action. --- .github/actions/compute-matrix/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 507361290c..6cf03e0d14 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -2,6 +2,10 @@ name: Compute Matrix description: "Compute the matrix for a given matrix type from the specified matrix file" +defaults: + run: + shell: bash -euxo pipefail {0} + inputs: matrix_query: description: 'The jq query used to specify the desired matrix. e.g., .["pull-request"]["nvcc]"' From a6df8747c766c0ab06029590738c74caee3fa503 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:16:40 +0000 Subject: [PATCH 16/34] Specify shell in step. --- .github/actions/compute-matrix/action.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 6cf03e0d14..44863b0649 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -2,10 +2,6 @@ name: Compute Matrix description: "Compute the matrix for a given matrix type from the specified matrix file" -defaults: - run: - shell: bash -euxo pipefail {0} - inputs: matrix_query: description: 'The jq query used to specify the desired matrix. e.g., .["pull-request"]["nvcc]"' @@ -25,4 +21,5 @@ runs: id: compute-matrix run: | MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh "$matrix_file" "$matrix_query") - echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT \ No newline at end of file + echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT + shell: bash -euxo pipefail {0} \ No newline at end of file From 661935ffa42d015541217a0d8a59319b4ee12629 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:18:15 +0000 Subject: [PATCH 17/34] Correctly use inputs for composite action. --- .github/actions/compute-matrix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 44863b0649..1c4342a94c 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -20,6 +20,6 @@ runs: - name: Compute matrix id: compute-matrix run: | - MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh "$matrix_file" "$matrix_query") + MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh ${{inputs.matrix_file}} ${{inputs.matrix_query}} ) echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT shell: bash -euxo pipefail {0} \ No newline at end of file From 7205c237b8197a7675f92f3153f052e94a88fc73 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:19:49 +0000 Subject: [PATCH 18/34] debug --- .github/actions/compute-matrix/compute-matrix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 372f86197d..8ec276bf46 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -11,6 +11,8 @@ if [ $# -ne 2 ]; then fi # Get realpath before changing directory +pwd +ls -la MATRIX_FILE="realpath $1" MATRIX_QUERY="$2" From c098fedf60a0c6e71071cb3c56c0e4895cdf1f0e Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:20:46 +0000 Subject: [PATCH 19/34] Fix expansion. --- .github/actions/compute-matrix/compute-matrix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 8ec276bf46..759ad3ff7c 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -13,7 +13,7 @@ fi # Get realpath before changing directory pwd ls -la -MATRIX_FILE="realpath $1" +MATRIX_FILE=$(realpath "$1") MATRIX_QUERY="$2" # Ensure the script is being executed in its containing directory From 214537d4bba555f848bfcccbeb57ec50af274146 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:21:49 +0000 Subject: [PATCH 20/34] Remove debug. --- .github/workflows/compute-nvcc-matrix.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml index 33be6cc927..040118a3da 100644 --- a/.github/workflows/compute-nvcc-matrix.yml +++ b/.github/workflows/compute-nvcc-matrix.yml @@ -43,18 +43,3 @@ jobs: COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }} PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }} steps: - - name: Validate matrix type - run: | - if [[ "$MATRIX_TYPE" != "pull-request" && "$MATRIX_TYPE" != "nightly" ]]; then - echo "Invalid build type: $MATRIX_TYPE. Must be 'nightly' or 'pull-request'" - exit 1 - fi - - name: Checkout repo - uses: actions/checkout@v3 - - name: Compute matrix - id: compute-matrix - run: | - pwd - ls -la - MATRIX_FILE=$(realpath "${MATRIX_FILE}") - ./.github/workflows/compute-matrix.sh "$MATRIX_FILE" '.["'"$MATRIX_TYPE"'"]["nvcc"]' \ No newline at end of file From 95dfcc95d06020681befd417dc0082c07848b82b Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Tue, 18 Jul 2023 22:21:58 +0000 Subject: [PATCH 21/34] Remove debug. --- .github/actions/compute-matrix/compute-matrix.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 759ad3ff7c..afeafb5bf6 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -11,8 +11,6 @@ if [ $# -ne 2 ]; then fi # Get realpath before changing directory -pwd -ls -la MATRIX_FILE=$(realpath "$1") MATRIX_QUERY="$2" From bcaef1d4e0df8413d46971d62bc6e5a3889da019 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 14:41:40 +0000 Subject: [PATCH 22/34] Escape quotes in jq query string. --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 23ec0ad7a5..6b87751ab7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/compute-matrix with: matrix_file: './ci/matrix.yaml' - matrix_query: '.["pull-request"]["nvcc"]' + matrix_query: '.[\"pull-request\"][\"nvcc\"]' thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} From 98227784f219ff3d92c6929413239671d5dc3084 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 14:45:40 +0000 Subject: [PATCH 23/34] Use underscore instead of hyphen to avoid needing quotes in query. --- .github/workflows/pr.yml | 2 +- ci/matrix.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6b87751ab7..20608fc142 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/compute-matrix with: matrix_file: './ci/matrix.yaml' - matrix_query: '.[\"pull-request\"][\"nvcc\"]' + matrix_query: '.[pull_request][nvcc]' thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} diff --git a/ci/matrix.yaml b/ci/matrix.yaml index a5deddf188..7148db63d6 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -21,7 +21,7 @@ gpus: # This field is unique as it will generate an independent build/test job for each value # Configurations that will run for every PR -pull-request: +pull_request: nvcc: - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']} - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']} From c194bfde4f0e51fbdca991d9e8ea45f2f40a389a Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 14:52:15 +0000 Subject: [PATCH 24/34] Use alternative syntax for jq query. --- .github/actions/compute-matrix/compute-matrix.sh | 2 +- .github/workflows/pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index afeafb5bf6..6f1ecf4b49 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -6,7 +6,7 @@ set -euo pipefail if [ $# -ne 2 ]; then echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" echo "MATRIX_FILE: The path to the matrix file." - echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.["pull-request"]["nvcc]"'" + echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.pull-request.nvcc' exit 1 fi diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 20608fc142..7243761713 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/compute-matrix with: matrix_file: './ci/matrix.yaml' - matrix_query: '.[pull_request][nvcc]' + matrix_query: '.pull_request.nvcc' thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} From 8d09b5c2eca8b15fafcfa27ab2eeadabcb55d274 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 14:53:51 +0000 Subject: [PATCH 25/34] Missing quote. --- .github/actions/compute-matrix/compute-matrix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 6f1ecf4b49..8a26d659d9 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -6,7 +6,7 @@ set -euo pipefail if [ $# -ne 2 ]; then echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" echo "MATRIX_FILE: The path to the matrix file." - echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.pull-request.nvcc' + echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.pull-request.nvcc'" exit 1 fi From 27f0e7946906ac48a55d02cfbf88ca80f2a29b17 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 15:23:44 +0000 Subject: [PATCH 26/34] Add post processing to set nvcc-specific matrix outputs. --- .github/workflows/pr.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7243761713..bc9b1e0e3b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -4,7 +4,7 @@ name: pr defaults: run: - shell: bash + shell: bash -euo pipefail {0} on: push: @@ -21,14 +21,29 @@ jobs: compute-nvcc-matrix: name: Compute NVCC matrix runs-on: ubuntu-latest + outputs: + CUDA_VERSIONS: ${{ steps.set-outputs.outputs.CUDA_VERSIONS }} + HOST_COMPILERS: ${{ steps.set-outputs.outputs.HOST_COMPILERS }} + PER_CUDA_COMPILER_MATRIX: ${{ steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX }} steps: - name: Checkout repo uses: actions/checkout@v3 - name: Get full nvcc matrix + id: compute-nvcc-matrix uses: ./.github/actions/compute-matrix with: matrix_file: './ci/matrix.yaml' matrix_query: '.pull_request.nvcc' + - name: Set outputs + id: set-outputs + run: | + FULL_MATRIX=${{steps.compute-nvcc-matrix.outputs.matrix}} + CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') + echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" + HOST_COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') + echo "HOST_COMPILERS=$HOST_COMPILERS" | tee -a "$GITHUB_OUTPUT" + PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add') + echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT" thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} @@ -38,7 +53,7 @@ jobs: fail-fast: false matrix: cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} with: per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_thrust.sh" @@ -52,7 +67,7 @@ jobs: fail-fast: false matrix: cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} with: per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_cub.sh" @@ -66,7 +81,7 @@ jobs: fail-fast: false matrix: cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.COMPILERS) }} + compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} with: per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_libcudacxx.sh" From b7e0fa6fa2b6b9490247d5b514ad1ef75043056c Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 15:56:05 +0000 Subject: [PATCH 27/34] Quotes around matrix output. --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bc9b1e0e3b..005cc2371b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -37,7 +37,7 @@ jobs: - name: Set outputs id: set-outputs run: | - FULL_MATRIX=${{steps.compute-nvcc-matrix.outputs.matrix}} + FULL_MATRIX="${{steps.compute-nvcc-matrix.outputs.matrix}}" CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" HOST_COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') From 100bdb7f6a0ecb9492cace16813d8960dda71190 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 18:59:08 +0000 Subject: [PATCH 28/34] Single quotes. --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 005cc2371b..be3e7822e9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -37,7 +37,7 @@ jobs: - name: Set outputs id: set-outputs run: | - FULL_MATRIX="${{steps.compute-nvcc-matrix.outputs.matrix}}" + FULL_MATRIX='${{steps.compute-nvcc-matrix.outputs.matrix}}' CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" HOST_COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') From 4cd2f80abf25fe5ce5b3cd1a22736337a4f76d56 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 20:14:01 +0000 Subject: [PATCH 29/34] Remove old compute-matrix reusable workflow. --- .github/workflows/compute-nvcc-matrix.yml | 45 ----------------------- 1 file changed, 45 deletions(-) delete mode 100644 .github/workflows/compute-nvcc-matrix.yml diff --git a/.github/workflows/compute-nvcc-matrix.yml b/.github/workflows/compute-nvcc-matrix.yml deleted file mode 100644 index 040118a3da..0000000000 --- a/.github/workflows/compute-nvcc-matrix.yml +++ /dev/null @@ -1,45 +0,0 @@ - -name: Compute the matrix for a given matrix type from the specified matrix file - -on: - workflow_call: - inputs: - matrix_file: - type: string - required: true - matrix_type: - description: "The desired matrix type. Currently, only 'pull-request' and 'nightly' are supported." - type: string - required: true - outputs: - FULL_MATRIX: - description: "The full matrix of build configurations for the given matrix type." - value: ${{ jobs.compute-matrix.outputs.FULL_MATRIX }} - CUDA_VERSIONS: - description: "The unique CUDA versions in the full matrix." - value: ${{ jobs.compute-matrix.outputs.CUDA_VERSIONS }} - COMPILERS: - description: "The unique compilers in the full matrix." - value: ${{ jobs.compute-matrix.outputs.COMPILERS }} - PER_CUDA_COMPILER_MATRIX: - description: "The matrix of build configurations for each CUDA version and compiler." - value: ${{ jobs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }} - -defaults: - run: - shell: bash - -env: - MATRIX_TYPE: ${{ inputs.matrix_type}} - MATRIX_FILE: ${{ inputs.matrix_file }} - -jobs: - compute-matrix: - name: ${{inputs.matrix_type}} - runs-on: ubuntu-latest - outputs: - FULL_MATRIX: ${{ steps.compute-matrix.outputs.FULL_MATRIX }} - CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }} - COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }} - PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }} - steps: From 1cd15818cb8c27e227291b3ecba81597e4250f35 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 20:14:50 +0000 Subject: [PATCH 30/34] Fix usage message. --- .github/actions/compute-matrix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 1c4342a94c..459cf8bf4d 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -4,7 +4,7 @@ description: "Compute the matrix for a given matrix type from the specified matr inputs: matrix_query: - description: 'The jq query used to specify the desired matrix. e.g., .["pull-request"]["nvcc]"' + description: 'The jq query used to specify the desired matrix. e.g., '.pull_request.nvcc' required: true matrix_file: description: 'The file containing the matrix' From e32fce7aac9faab7e703592170662e968cb55014 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 20:16:19 +0000 Subject: [PATCH 31/34] Update devcontainer script to reference nvcc field. --- .devcontainer/make_devcontainers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh index 9fc56afca2..56a6fe42a9 100755 --- a/.devcontainer/make_devcontainers.sh +++ b/.devcontainer/make_devcontainers.sh @@ -20,7 +20,7 @@ IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-" matrix_json=$(yq -o json ../ci/matrix.yaml) # Get unique combinations of cuda version, compiler name/version, and Ubuntu version -combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, os: .os}] | unique | .[]') +combinations=$(echo "$matrix_json" | jq -c '[.pull_request.nvcc[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, os: .os}] | unique | .[]') # For each unique combination for combination in $combinations; do From 705207c5b32047d639ea3a0103198e572d6a11ca Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 20:31:24 +0000 Subject: [PATCH 32/34] Removed unused GITHUB_OUTPUT var. --- .github/actions/compute-matrix/compute-matrix.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 8a26d659d9..3a2d1337a0 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -17,9 +17,6 @@ MATRIX_QUERY="$2" # Ensure the script is being executed in its containing directory cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; -# Use /dev/null as the default value for GITHUB_OUTPUT if it isn't set, i.e., not running in GitHub Actions -GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" - echo "Input matrix file:" >&2 cat "$MATRIX_FILE" >&2 echo "Query: $MATRIX_QUERY" >&2 From 468bab73b2a556a49d0d26eeb906598f3394f0bf Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Wed, 19 Jul 2023 20:31:48 +0000 Subject: [PATCH 33/34] EOF --- .github/actions/compute-matrix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 459cf8bf4d..70b808a565 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -22,4 +22,4 @@ runs: run: | MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh ${{inputs.matrix_file}} ${{inputs.matrix_query}} ) echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT - shell: bash -euxo pipefail {0} \ No newline at end of file + shell: bash -euxo pipefail {0} From dda1646710148e510b96f8aec44cf85b907f4ca8 Mon Sep 17 00:00:00 2001 From: Jake Hemstad Date: Thu, 20 Jul 2023 17:43:46 +0000 Subject: [PATCH 34/34] Fix quotes. --- .github/actions/compute-matrix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml index 70b808a565..b8155e7aa6 100644 --- a/.github/actions/compute-matrix/action.yml +++ b/.github/actions/compute-matrix/action.yml @@ -4,7 +4,7 @@ description: "Compute the matrix for a given matrix type from the specified matr inputs: matrix_query: - description: 'The jq query used to specify the desired matrix. e.g., '.pull_request.nvcc' + description: "The jq query used to specify the desired matrix. e.g., .pull_request.nvcc" required: true matrix_file: description: 'The file containing the matrix'