Maximize number of parallel jobs and readme update #5
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "[Main] Build, test, and push jupyter images" | |
| on: | |
| pull_request: | |
| paths: | |
| - ".github/workflows/main.yml" | |
| - ".github/workflows/build-base.yml" | |
| - ".github/workflows/build-datascience.yml" | |
| - ".github/workflows/build-spark.yml" | |
| - ".github/workflows/docker-build-test-push-latest.yml" | |
| - ".github/workflows/docker-tag-push.yml" | |
| - ".github/actions/generate-build-matrix/action.yml" | |
| - ".github/actions/install-patchs-and-extension/action.yml" | |
| - ".build/.versions.yml" | |
| - "python/okdp/**" | |
| - "docker-stacks/images/**" | |
| - "docker-stacks/tests/**" | |
| - "docker-stacks/tagging/**" | |
| - "images/**" | |
| - "!python/okdp/patch/README.md" | |
| - "!images/README.md" | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - ".github/workflows/main.yml" | |
| - ".github/workflows/build-base.yml" | |
| - ".github/workflows/build-datascience.yml" | |
| - ".github/workflows/build-spark.yml" | |
| - ".github/workflows/docker-build-test-push-latest.yml" | |
| - ".github/workflows/docker-tag-push.yml" | |
| - ".github/actions/generate-build-matrix/action.yml" | |
| - ".github/actions/install-patchs-and-extension/action.yml" | |
| - ".build/.versions.yml" | |
| - "python/okdp/**" | |
| - "docker-stacks/images/**" | |
| - "docker-stacks/tests/**" | |
| - "docker-stacks/tagging/**" | |
| - "images/**" | |
| - "!python/okdp/patch/README.md" | |
| - "!images/README.md" | |
| workflow_dispatch: | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| working-directory: ./docker-stacks | |
| jobs: | |
| run-unit-tests: | |
| uses: ./.github/workflows/unit-tests.yml | |
| with: | |
| runs-on: ubuntu-latest | |
| build-version-compatibility-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| spark: ${{ steps.set-matrix.outputs.spark }} | |
| python: ${{ steps.set-matrix.outputs.python }} | |
| steps: | |
| - name: Checkout Repo ⚡️ | |
| uses: actions/checkout@v4 | |
| - name: Setup dev env patchs 📦 | |
| uses: ./.github/actions/install-patchs-and-extension | |
| - name: Create dev environment 📦 | |
| uses: ./docker-stacks/.github/actions/create-dev-env | |
| - name: Get current branch 📦 | |
| id: branch | |
| uses: tj-actions/branch-names@v8 | |
| - name: Build version compatibility matrix 🛠 | |
| id: set-matrix | |
| run: | | |
| python3 -m okdp.extension.matrix.version_compatibility_matrix \ | |
| --versions-matrix-path ../.build/.versions.yml \ | |
| --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT | |
| cat $GITHUB_OUTPUT | |
| shell: bash | |
| needs: [run-unit-tests] | |
| build-base: | |
| name: build-test-base (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/build-test-base.yml | |
| with: | |
| python_version: ${{ matrix.python.python_version }} | |
| python_dev_tag: ${{ matrix.python.python_dev_tag }} | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix] | |
| build-datascience: | |
| name: build-test-datascience (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/build-test-datascience.yml | |
| with: | |
| python_dev_tag: ${{ matrix.python.python_dev_tag }} | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix, build-base] | |
| build-spark: | |
| name: build-test-spark (python-${{ matrix.spark.python_version }}) | |
| strategy: | |
| matrix: | |
| spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} | |
| uses: ./.github/workflows/build-test-spark.yml | |
| with: | |
| spark_download_url: ${{ matrix.spark.spark_download_url }} | |
| python_version: ${{ matrix.spark.python_version }} | |
| spark_version: ${{ matrix.spark.spark_version }} | |
| java_version: ${{ matrix.spark.java_version }} | |
| scala_version: ${{ matrix.spark.scala_version }} | |
| hadoop_version: ${{ matrix.spark.hadoop_version }} | |
| python_dev_tag: ${{ matrix.spark.python_dev_tag }} | |
| spark_dev_tag: ${{ matrix.spark.spark_dev_tag }} | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix, build-base] | |
| ### 3 push in // | |
| push-base: | |
| if: github.ref == 'refs/heads/main' | |
| name: push-base (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| image: [ docker-stacks-foundation, base-notebook, minimal-notebook, scipy-notebook,] | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/docker-tag-push.yml | |
| with: | |
| image: "${{ matrix.image }}:${{ matrix.python.python_dev_tag }}" | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix, build-datascience, build-spark,] | |
| push-datascience: | |
| if: github.ref == 'refs/heads/main' | |
| name: push-datascience (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| image: [r-notebook, datascience-notebook, ] | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/docker-tag-push.yml | |
| with: | |
| image: "${{ matrix.image }}:${{ matrix.python.python_dev_tag }}" | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix, build-datascience, build-spark] | |
| push-spark: | |
| if: github.ref == 'refs/heads/main' | |
| strategy: | |
| matrix: | |
| image: [ pyspark-notebook, all-spark-notebook,] | |
| spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} | |
| uses: ./.github/workflows/docker-tag-push.yml | |
| with: | |
| image: "${{ matrix.image }}:${{ matrix.spark.spark_dev_tag }}" | |
| registry: ${{ vars.REGISTRY || 'ghcr.io' }} | |
| runs-on: ubuntu-latest | |
| secrets: | |
| registry_username: ${{ secrets.REGISTRY_USERNAME }} | |
| registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} | |
| needs: [build-version-compatibility-matrix, build-datascience, build-spark] | |
| # Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 | |
| partial-rerun-on-failure: | |
| # Allow re-run the main branch only | |
| if: failure() && github.ref == 'refs/heads/main' && fromJSON(github.run_attempt) < 3 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| actions: write | |
| packages: write | |
| steps: | |
| - name: "Re-run failed jobs" | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ github.token }} | |
| GH_DEBUG: api | |
| run: | | |
| gh workflow run auto-rerun.yml \ | |
| -F run_id=${{ github.run_id }} \ | |
| -F workflow_name=${{ github.workflow }} \ | |
| -F workflow_sha=${{ github.workflow_sha }} | |
| shell: bash | |
| needs: [push-base, push-datascience, push-spark] | |