publish.yml

name: publish

on:
  ### Periodically rebuild all the images to fix os security vulnerabilities
  schedule:
    # At 05:00 AM, only on Monday
    #- cron: "0 5 * * 1"
    # https://crontab.cronhub.io/
    # At 05:"0 AM, only on Monday
    - cron: "0 5 * * 1"
  # The release should be created manually (or with user token=pr approval/merge) in order to trigger the event
  ### https://github.com/orgs/community/discussions/25281
  ### Instead of using the event, we call the workflow from release-please workflow (more secure)
  #release:
  #  types: [published]

  workflow_dispatch:

# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

defaults:
  run:
    working-directory: ./docker-stacks
    
permissions:
  packages: write

jobs:

  latest-github-release:
    if: github.repository_owner == 'OKDP'
    runs-on: "ubuntu-latest"
    outputs:
      tag_name: ${{ steps.git-release-tag.outputs.tag_name }}
    steps:
      - name: Checkout Repo ⚡️
        uses: actions/checkout@v4

      - name: Get latest GitHub Release tag name 📥
        id: git-release-tag
        uses: InsonusK/get-latest-release@v1.0.1
        with:
          myToken: ${{ github.token }}
          exclude_types: "draft"
          view_top: 1
      
      - name: Info - Found latest release tag
        run: |
            echo "id: ${{ steps.git-release-tag.outputs.id }}"
            echo "name: ${{ steps.git-release-tag.outputs.name }}"
            echo "tag_name: ${{ steps.git-release-tag.outputs.tag_name }}"
            echo "created_at: ${{ steps.git-release-tag.outputs.created_at }}"
            echo "draft: ${{ steps.git-release-tag.outputs.draft }}"
            echo "prerelease: ${{ steps.git-release-tag.outputs.prerelease }}"
        shell: bash

  build-version-compatibility-matrix:
    runs-on: ubuntu-latest
    outputs:
      spark: ${{ steps.set-matrix.outputs.spark }}
      python: ${{ steps.set-matrix.outputs.python }}
    steps:
      - name: Checkout Repo ⚡️
        uses: actions/checkout@v4
      
      - name: Setup dev env patchs 📦
        uses: ./.github/actions/install-patchs-and-extension
      
      - name: Create dev environment 📦
        uses: ./docker-stacks/.github/actions/create-dev-env
      
      - name: Run unit tests (versions compatibility matrix)
        run:  pytest python/tests -v --color=yes
        working-directory: ./.build
        shell: bash

      - name: Get current branch 📦
        id: branch
        uses: tj-actions/branch-names@v8
    
      - name: Build version compatibility matrix  🛠
        id: set-matrix
        run: |
            python3 -m okdp.extension.matrix.version_compatibility_matrix \
            --versions-matrix-path ../.build/.versions.yml \
            --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT
            cat $GITHUB_OUTPUT
        shell: bash

  publish-base:
    if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != ''
    name: publish-base (python-${{ matrix.python.python_version }})
    strategy:
      matrix:
        python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }}
    uses: ./.github/workflows/build-base-images-template.yml
    with:
      python_version: ${{ matrix.python.python_version }}
      python_dev_tag: ${{ matrix.python.python_dev_tag }}
      publish_to_registry: "true"
      registry: ${{ vars.REGISTRY || 'quay.io' }}
      git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }}
      runs-on: ubuntu-latest
    secrets: inherit
    needs: [latest-github-release,build-version-compatibility-matrix]

  publish-datascience:
    if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != ''
    name: publish-datascience (python-${{ matrix.python.python_version }})
    strategy:
      matrix:
        python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }}
    uses: ./.github/workflows/build-datascience-images-template.yml
    with:
      python_dev_tag: ${{ matrix.python.python_dev_tag }}
      publish_to_registry: "true"
      registry: ${{ vars.REGISTRY || 'quay.io' }}
      git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }}
      runs-on: ubuntu-latest
    secrets: inherit
    needs: [latest-github-release, build-version-compatibility-matrix, publish-base]

  publish-spark:
    if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != ''
    strategy:
      matrix: 
        spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }}
    uses: ./.github/workflows/build-spark-images-template.yml
    with:
      spark_download_url: ${{ matrix.spark.spark_download_url }}
      python_version: ${{ matrix.spark.python_version }}
      spark_version: ${{ matrix.spark.spark_version }}
      java_version: ${{ matrix.spark.java_version }}
      scala_version: ${{ matrix.spark.scala_version }}
      hadoop_version: ${{ matrix.spark.hadoop_version }}
      python_dev_tag: ${{ matrix.spark.python_dev_tag }}
      spark_dev_tag: ${{ matrix.spark.spark_dev_tag }}
      publish_to_registry: "true"
      registry: ${{ vars.REGISTRY || 'quay.io' }}
      git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }}
      runs-on: ubuntu-latest
    secrets: inherit
    needs: [latest-github-release, build-version-compatibility-matrix, publish-base]

  ## https://github.com/orgs/community/discussions/67654#discussioncomment-8038649
  partial-rerun-on-failure:
    # Allow re-run the main branch only
    if:  failure() && github.ref == 'refs/heads/main' && fromJSON(github.run_attempt) < 3
    runs-on: ubuntu-latest
    permissions:
      contents: write
      actions: write
      packages: write
    steps:
      - name: "Re-run failed jobs"
        env:
          GH_REPO: ${{ github.repository }}
          GH_TOKEN: ${{ github.token }}
          GH_DEBUG: api
        run: |
          gh workflow run auto-rerun.yml \
             -F run_id=${{ github.run_id }} \
             -F workflow_name=${{ github.workflow }} \
             -F workflow_sha=${{ github.workflow_sha }}
        
        working-directory: ./
        shell: bash
      
    needs: [publish-base, publish-datascience, publish-spark]