publish #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: publish | |
| on: | |
| ### Periodically rebuild all the images to fix os security vulnerabilities | |
| schedule: | |
| # At 05:00 AM, only on Monday | |
| #- cron: "0 5 * * 1" | |
| # https://crontab.cronhub.io/ | |
| # At 05:"0 AM, only on Monday | |
| - cron: "0 5 * * 1" | |
| # The release should be created manually (or with user token=pr approval/merge) in order to trigger the event | |
| ### https://github.com/orgs/community/discussions/25281 | |
| ### Instead of using the event, we call the workflow from release-please workflow (more secure) | |
| #release: | |
| # types: [published] | |
| workflow_dispatch: | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| working-directory: ./docker-stacks | |
| permissions: | |
| packages: write | |
| jobs: | |
| latest-github-release: | |
| if: github.repository_owner == 'OKDP' | |
| runs-on: "ubuntu-latest" | |
| outputs: | |
| tag_name: ${{ steps.git-release-tag.outputs.tag_name }} | |
| steps: | |
| - name: Checkout Repo ⚡️ | |
| uses: actions/checkout@v4 | |
| - name: Get latest GitHub Release tag name 📥 | |
| id: git-release-tag | |
| uses: InsonusK/[email protected] | |
| with: | |
| myToken: ${{ github.token }} | |
| exclude_types: "draft" | |
| view_top: 1 | |
| - name: Info - Found latest release tag | |
| run: | | |
| echo "id: ${{ steps.git-release-tag.outputs.id }}" | |
| echo "name: ${{ steps.git-release-tag.outputs.name }}" | |
| echo "tag_name: ${{ steps.git-release-tag.outputs.tag_name }}" | |
| echo "created_at: ${{ steps.git-release-tag.outputs.created_at }}" | |
| echo "draft: ${{ steps.git-release-tag.outputs.draft }}" | |
| echo "prerelease: ${{ steps.git-release-tag.outputs.prerelease }}" | |
| shell: bash | |
| build-version-compatibility-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| spark: ${{ steps.set-matrix.outputs.spark }} | |
| python: ${{ steps.set-matrix.outputs.python }} | |
| steps: | |
| - name: Checkout Repo ⚡️ | |
| uses: actions/checkout@v4 | |
| - name: Setup dev env patchs 📦 | |
| uses: ./.github/actions/install-patchs-and-extension | |
| - name: Create dev environment 📦 | |
| uses: ./docker-stacks/.github/actions/create-dev-env | |
| - name: Run unit tests (versions compatibility matrix) | |
| run: pytest python/tests -v --color=yes | |
| working-directory: ./.build | |
| shell: bash | |
| - name: Get current branch 📦 | |
| id: branch | |
| uses: tj-actions/branch-names@v8 | |
| - name: Build version compatibility matrix 🛠 | |
| id: set-matrix | |
| run: | | |
| python3 -m okdp.extension.matrix.version_compatibility_matrix \ | |
| --versions-matrix-path ../.build/.versions.yml \ | |
| --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT | |
| cat $GITHUB_OUTPUT | |
| shell: bash | |
| publish-base: | |
| if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' | |
| name: publish-base (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/build-base-images-template.yml | |
| with: | |
| python_version: ${{ matrix.python.python_version }} | |
| python_dev_tag: ${{ matrix.python.python_dev_tag }} | |
| publish_to_registry: "true" | |
| registry: ${{ vars.REGISTRY || 'quay.io' }} | |
| git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} | |
| runs-on: ubuntu-latest | |
| secrets: inherit | |
| needs: [latest-github-release,build-version-compatibility-matrix] | |
| publish-datascience: | |
| if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' | |
| name: publish-datascience (python-${{ matrix.python.python_version }}) | |
| strategy: | |
| matrix: | |
| python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} | |
| uses: ./.github/workflows/build-datascience-images-template.yml | |
| with: | |
| python_dev_tag: ${{ matrix.python.python_dev_tag }} | |
| publish_to_registry: "true" | |
| registry: ${{ vars.REGISTRY || 'quay.io' }} | |
| git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} | |
| runs-on: ubuntu-latest | |
| secrets: inherit | |
| needs: [latest-github-release, build-version-compatibility-matrix, publish-base] | |
| publish-spark: | |
| if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' | |
| strategy: | |
| matrix: | |
| spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} | |
| uses: ./.github/workflows/build-spark-images-template.yml | |
| with: | |
| spark_download_url: ${{ matrix.spark.spark_download_url }} | |
| python_version: ${{ matrix.spark.python_version }} | |
| spark_version: ${{ matrix.spark.spark_version }} | |
| java_version: ${{ matrix.spark.java_version }} | |
| scala_version: ${{ matrix.spark.scala_version }} | |
| hadoop_version: ${{ matrix.spark.hadoop_version }} | |
| python_dev_tag: ${{ matrix.spark.python_dev_tag }} | |
| spark_dev_tag: ${{ matrix.spark.spark_dev_tag }} | |
| publish_to_registry: "true" | |
| registry: ${{ vars.REGISTRY || 'quay.io' }} | |
| git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} | |
| runs-on: ubuntu-latest | |
| secrets: inherit | |
| needs: [latest-github-release, build-version-compatibility-matrix, publish-base] | |
| ## https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 | |
| partial-rerun-on-failure: | |
| # Allow re-run the main branch only | |
| if: failure() && github.ref == 'refs/heads/main' && fromJSON(github.run_attempt) < 3 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| actions: write | |
| packages: write | |
| steps: | |
| - name: "Re-run failed jobs" | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ github.token }} | |
| GH_DEBUG: api | |
| run: | | |
| gh workflow run auto-rerun.yml \ | |
| -F run_id=${{ github.run_id }} \ | |
| -F workflow_name=${{ github.workflow }} \ | |
| -F workflow_sha=${{ github.workflow_sha }} | |
| working-directory: ./ | |
| shell: bash | |
| needs: [publish-base, publish-datascience, publish-spark] | |