diff --git a/.github/workflows/wheels-build.yaml b/.github/workflows/wheels-build.yaml index b8334e16..24f89aae 100644 --- a/.github/workflows/wheels-build.yaml +++ b/.github/workflows/wheels-build.yaml @@ -15,7 +15,7 @@ on: build_type: required: true type: string - build_script: + script: required: true type: string @@ -148,7 +148,7 @@ jobs: run: | # Store internal pypi credentials before any step that may download wheels printf 'machine pypi.k8s.rapids.ai\n\tlogin cibuildwheel\n\tpassword ${{ secrets.RAPIDSAI_PYPI_CI_PASSWORD }}\n' > ~/.netrc - ${{ inputs.build_script }} + ${{ inputs.script }} env: GH_TOKEN: ${{ github.token }} # Use a shell that loads the rc file so that we get the compiler settings diff --git a/.github/workflows/wheels-publish.yaml b/.github/workflows/wheels-publish.yaml new file mode 100644 index 00000000..8492345c --- /dev/null +++ b/.github/workflows/wheels-publish.yaml @@ -0,0 +1,75 @@ +name: Publish RAPIDS wheels + +on: + workflow_call: + inputs: + # repo and branch + repo: + type: string + branch: + type: string + date: + type: string + sha: + type: string + build_type: + required: true + type: string + + # general settings + package-name: + required: true + type: string + +permissions: + actions: read + checks: none + contents: read + deployments: none + discussions: none + id-token: write + issues: none + packages: read + pages: none + pull-requests: read + repository-projects: none + security-events: none + statuses: none + +jobs: + wheel-publish: + name: wheels publish + runs-on: ubuntu-latest + container: + # ctk version of the container is irrelevant in the publish step + # it's simply a launcher for twine + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10" + env: + RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} + TWINE_USERNAME: cibuildwheel + TWINE_PASSWORD: ${{ secrets.RAPIDSAI_PYPI_CI_PASSWORD }} + TWINE_REPOSITORY_URL: "https://pypi.k8s.rapids.ai/simple/" + steps: + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - name: checkout code repo + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.sha }} + fetch-depth: 0 # unshallow fetch for setuptools-scm + persist-credentials: false + + - name: Standardize repository information + uses: rapidsai/shared-action-workflows/rapids-github-info@branch-23.08 + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + + - name: Download wheels from downloads.rapids.ai and publish to internal PyPI + run: rapids-twine "${{ inputs.package-name }}" diff --git a/.github/workflows/wheels-test.yaml b/.github/workflows/wheels-test.yaml new file mode 100644 index 00000000..4ac18d69 --- /dev/null +++ b/.github/workflows/wheels-test.yaml @@ -0,0 +1,142 @@ +name: Test RAPIDS wheels + +on: + workflow_call: + inputs: + # repo and branch + repo: + type: string + branch: + type: string + date: + type: string + sha: + type: string + build_type: + required: true + type: string + script: + type: string + default: "ci/test_wheel.sh" + + # general settings + matrix_filter: + type: string + default: "." + + # test settings + test-docker-options: + required: false + type: string + default: '-e _NOOP' + +defaults: + run: + shell: bash + +permissions: + actions: read + checks: none + contents: read + deployments: none + discussions: none + id-token: write + issues: none + packages: read + pages: none + pull-requests: read + repository-projects: none + security-events: none + statuses: none + +jobs: + compute-matrix: + runs-on: ubuntu-latest + env: + BUILD_TYPE: ${{ inputs.build_type }} + outputs: + MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }} + steps: + - name: Validate test type + run: | + if [[ "$BUILD_TYPE" != "pull-request" ]] && [[ "$BUILD_TYPE" != "nightly" ]]; then + echo "Invalid build type! Must be 'nightly' or 'pull-request'." + exit 1 + fi + - name: Compute test matrix + id: compute-matrix + run: | + set -eo pipefail + + export MATRICES=" + pull-request: + - { ARCH: 'amd64', PY_VER: '3.9', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.9', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', gpu: 'a100', driver: 'latest' } + nightly: + - { ARCH: 'amd64', PY_VER: '3.9', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.9', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', gpu: 'a100', driver: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', gpu: 'a100', driver: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.9', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu18.04', gpu: 'v100', driver: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.9', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', gpu: 'a100', driver: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', gpu: 'a100', driver: 'latest' } + " + + TEST_MATRIX=$(yq -n 'env(MATRICES) | .[strenv(BUILD_TYPE)]') + export TEST_MATRIX + + echo "MATRIX=$( + yq -n -o json 'env(TEST_MATRIX)' | \ + jq -c '${{ inputs.matrix_filter }} | {include: .}' \ + )" | tee --append "${GITHUB_OUTPUT}" + + test: + name: ${{ matrix.CUDA_VER }}, ${{ matrix.PY_VER }}, ${{ matrix.ARCH }}, ${{ matrix.LINUX_VER }}, ${{ matrix.gpu }} + needs: compute-matrix + env: + RAPIDS_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts + strategy: + matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} + runs-on: "linux-${{ matrix.ARCH }}-gpu-${{ matrix.gpu }}-${{ matrix.driver }}-1" + container: + image: "rapidsai/citestwheel:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + options: ${{ inputs.test-docker-options }} + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} # GPU jobs must set this container env variable + RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} + steps: + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - name: Run nvidia-smi to make sure GPU is working + run: nvidia-smi + + - name: Install private index credentials in cibuildwheel container + run: printf 'machine pypi.k8s.rapids.ai\n\tlogin cibuildwheel\n\tpassword ${{ secrets.RAPIDSAI_PYPI_CI_PASSWORD }}\n' > ~/.netrc + + - name: checkout code repo + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.sha }} + fetch-depth: 0 # unshallow fetch for setuptools-scm + persist-credentials: false + + - name: Standardize repository information + uses: rapidsai/shared-action-workflows/rapids-github-info@branch-23.08 + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + + - name: Run tests + run: ${{ inputs.script }} + + - name: Upload additional artifacts + if: "!cancelled()" + run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}