pypi-release-pipeline.yml

name: PyPI release pipeline

on:
  workflow_dispatch:
    inputs:
      variant-id:
        description: "Variant id"
        required: true
        type: string
      deploy:
        description: "Deploy to official pypi if true"
        required: false
        type: boolean
        default: false

jobs:
  variants:
    name: Populate supported AIMET variants
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.final.outputs.value }}
    steps:
      - name: Torch variants
        run: |
          if [ "${{ inputs.variant-id }}" == "torch-cpu" ] ; then
            VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
                {
                   "id":              "torch-cpu",
                   "runs-on":         "ubuntu-latest",
                   "VER_PYTHON":      "3.10",
                   "VER_TENSORFLOW":  "",
                   "VER_TORCH":       "2.1.2",
                   "VER_ONNXRUNTIME": "",
                   "VER_CUDA":        "",
                   "ENABLE_TESTS":    "ON",
                   "BUILD_TARGETS":   "all"
                }
              ]')
            elif [ "${{ inputs.variant-id }}" == "torch-gpu" ] ; then
              VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
                {
                   "id":              "torch-gpu",
                   "runs-on":         "k8s-gpu",
                   "VER_PYTHON":      "3.10",
                   "VER_TENSORFLOW":  "",
                   "VER_TORCH":       "2.1.2",
                   "VER_ONNXRUNTIME": "",
                   "VER_CUDA":        "12.1.1",
                   "ENABLE_TESTS":    "ON",
                   "BUILD_TARGETS":   "all"
                }
              ]')
            elif [ "${{ inputs.variant-id }}" == "onnx-cpu" ] ; then
              VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
                {
                   "id":              "onnx-cpu",
                   "runs-on":         "ubuntu-latest",
                   "VER_PYTHON":      "3.10",
                   "VER_TENSORFLOW":  "",
                   "VER_TORCH":       "",
                   "VER_ONNXRUNTIME": "1.18.1",
                   "VER_CUDA":        "",
                   "ENABLE_TESTS":    "ON",
                   "BUILD_TARGETS":   "all"
                }
              ]')
            elif [ "${{ inputs.variant-id }}" == "onnx-gpu" ] ; then
              VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
                {
                   "id":              "onnx-gpu",
                   "runs-on":         "k8s-gpu",
                   "VER_PYTHON":      "3.10",
                   "VER_TENSORFLOW":  "",
                   "VER_TORCH":       "",
                   "VER_ONNXRUNTIME": "1.18.1",
                   "VER_CUDA":        "11.8.0",
                   "ENABLE_TESTS":    "ON",
                   "BUILD_TARGETS":   "all"
                }
              ]')  
            fi
          echo "VALUE=$VALUE" >> $GITHUB_ENV
      - name: (Last step) Generate few extra properties for each variant
        id: final
        run: |
          echo "value=$VALUE" >> $GITHUB_OUTPUT

  docker-build-image:
    name: Docker image ${{ matrix.id }}
    runs-on: ubuntu-latest
    needs: [ variants ]
    strategy:
      matrix: ${{ fromJSON(needs.variants.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/docker-build-image
        with:
          dockerfile: Jenkins/fast-release/Dockerfile.ci
          docker-login: ${{ secrets.DOCKER_LOGIN }}
          docker-password: ${{ secrets.DOCKER_CREDENTIALS }}
          docker-registry: ${{ vars.DOCKER_REGISTRY }}
          image-name: "${{ vars.DOCKER_IMAGE }}-${{ inputs.variant-id }}"
          image-tag: "release"
          build-args: |
            VER_PYTHON=${{ matrix.VER_PYTHON }}
            VER_CUDA=${{ matrix.VER_CUDA }}
            VER_TORCH=${{ matrix.VER_TORCH }}
            VER_TENSORFLOW=${{ matrix.VER_TENSORFLOW }}
            VER_ONNXRUNTIME=${{ matrix.VER_ONNXRUNTIME }}

  call-build-wheels:
    name: Call build-wheels
    needs: [ variants, docker-build-image ]
    uses: ./.github/workflows/build-wheels.yml
    with:
      variants: ${{ needs.variants.outputs.matrix }}
      image-tag: "release"
    secrets: inherit

  call-run-unit-tests:
    name: Run unit tests
    needs: [ variants, call-build-wheels ]
    uses: ./.github/workflows/run-unit-tests.yml
    with:
      variants: ${{ needs.variants.outputs.matrix }}
    secrets: inherit

  call-run-acceptance-tests:
    name: Run acceptance tests
    needs: [ variants, call-build-wheels, call-run-unit-tests ]
    uses: ./.github/workflows/run-acceptance-tests.yml
    with:
      variants: ${{ needs.variants.outputs.matrix }}
    secrets: inherit

  publish:
    name: Publish the wheel packages
    runs-on: k8s-gpu
    needs: [ variants, call-build-wheels, call-run-unit-tests, call-run-acceptance-tests ]
    env:
      DEPLOY: ${{ inputs.deploy }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.variants.outputs.matrix) }}
    container:
      image: "ubuntu:22.04"
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: "${{ matrix.id }}"
          path: "downloads"
      - run: |
          apt-get update
          apt-get install python3 python3-pip -y
          python3 -m pip install --upgrade pip
          python3 -m pip install twine
          ls -al downloads/wheelhouse
      - name: Push to testpypi
        run: |
          case $DEPLOY in
          (true)
          echo "Deploy variable set to true, deploying. . . "
          twine upload --verbose --repository testpypi --config-file ~/.pypirc downloads/wheelhouse/*
          ;;
          (false)
          echo "Deploy variable is false, deployment for twine upload --verbose --repository pypi --config-file ~/.pypirc downloads/wheelhouse/* will not proceed."
          ;;
          esac
      - name: Push to pypi
        run: |          
          case $DEPLOY in
          (true)
          echo "Deploy variable set to true, deploying. . . "
          twine upload --verbose --repository pypi --config-file ~/.pypirc downloads/wheelhouse/*
          ;;
          (false)
          echo "Deploy variable is false, deployment for twine upload --verbose --repository pypi --config-file ~/.pypirc downloads/wheelhouse/* will not proceed."
          ;;
          esac

  cleanup:
    needs: [ variants, docker-build-image, call-build-wheels ]
    runs-on: ubuntu-latest
    strategy:
      matrix: ${{ fromJSON(needs.variants.outputs.matrix) }}
    steps:
      - name: Delete temp docker image
        run: curl -k -H "Authorization:Bearer ${{ secrets.DOCKER_CREDENTIALS }}" -X DELETE "https://${{ vars.DOCKER_REGISTRY }}/v2/${{ vars.DOCKER_IMAGE }}-${{ inputs.variant-id }}/manifests/release" || true