Merge pull request #1 from idirze/build-pipeline

Jupyterlab images build pipeline

.build/.versions.yml

@@ -0,0 +1,47 @@
+################ Version Compatibility Matrix ######################
+##### SECTION USED TO GET COMPATIBLE VERSIONS ######################
+##### DONT PUT python_version IN AN ARRAY - JAVA COMPATIBILITY #####
+####### ENRICH WITH NEWER VERSIONS #################################
+compatibility-matrix:
+  # Maximum python version supported by spark-3.2.x: 3.9
+  # Java support: 8/11
+  - python_version: 3.9
+    spark_version: [3.2.1, 3.2.2, 3.2.3, 3.2.4]
+    java_version: [11]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3.2
+    spark_download_url: https://archive.apache.org/dist/spark/
+  # Maximum python version supported by spark-3.3.x: 3.10
+  # Java support: 8/11/17
+  - python_version: '3.10'
+    spark_version: [3.3.1, 3.3.2, 3.3.3, 3.3.4]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+    spark_download_url: https://archive.apache.org/dist/spark/
+  # Maximum python version supported by spark-3.4.x: 3.11
+  # Java support: 8/11/17
+  - python_version: 3.11
+    spark_version: [3.4.1, 3.4.2]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+    spark_download_url: https://archive.apache.org/dist/spark/
+  # https://spark.apache.org/releases/spark-release-3-5-0.html
+  # Minimum supported java version: 17/21
+  - python_version: 3.11
+    spark_version: [3.5.0]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+    spark_download_url: https://archive.apache.org/dist/spark/
+### Override the matrix above by providing the versions to build
+### 1- The build-matrix is empty: build with all possible combintations
+### 2- Override specific versions: build with all possible combinations which are compatible with that specific versions
+### 3- The versions not present on compatibility-matrix are ignored
+build-matrix:
+  python_version: ['3.9', '3.10', '3.11']
+  spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.0]
+  java_version: [11, 17]
+  scala_version: [2.12]
+

.github/actions/install-patchs-and-extension/action.yml

@@ -0,0 +1,20 @@
+name: Set up dev env requirements
+description: Set up dev env requirements for build and tests
+
+runs:
+  using: composite
+  steps:
+  - name: Setup dev env requirement 📦
+    run: |
+      cp -f requirements-dev.txt ../
+      cat ../python/requirements-extended.txt >> ../requirements-dev.txt
+    working-directory: ./docker-stacks
+    shell: bash
+
+  - name: Copy patchs 📦
+    run: |
+      cp -fr ../python/okdp/patch/* ./
+      cp -fr ../python/okdp ./
+    working-directory: ./docker-stacks
+    shell: bash
+

.github/workflows/auto-rerun.yml

@@ -0,0 +1,27 @@
+# Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649
+name: Automatic partital rerun on workflow failure
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        required: true
+      workflow_name:
+        required: true
+      workflow_sha:
+        required: true
+jobs:
+  rerun:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
+    steps:
+      - name: Re-run workflow ${{ inputs.workflow_name }}-${{ inputs.run_id }}
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+          GH_DEBUG: api
+        run: |
+           gh run watch ${{ inputs.run_id }} > /dev/null 2>&1
+           gh run rerun ${{ inputs.run_id }} --failed

.github/workflows/build-test-base.yml

@@ -0,0 +1,82 @@
+name: Build, test, and push jupyter base images
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        description: Python version
+        required: true
+        type: string
+      python_dev_tag:
+        description: Tag to use for latest base images (foundation, minimal, etc)
+        required: true
+        type: string
+      registry:
+        description: The list of tags space separated values
+        required: true
+        type: string
+      runs-on:
+        description: GitHub Actions Runner image
+        required: true
+        type: string
+    secrets:
+      registry_username:
+        required: true
+      registry_token:
+        required: true
+
+defaults:
+  run:
+    working-directory: ./docker-stacks
+
+jobs:
+
+  docker-stacks-foundation:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: ""
+      image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      build-args:
+        PYTHON_VERSION=${{ inputs.python_version }}
+      runs-on: ubuntu-latest
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+
+  base-notebook:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
+      image: base-notebook:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ubuntu-latest
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+    needs: [docker-stacks-foundation]
+
+  minimal-notebook:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: base-notebook:${{ inputs.python_dev_tag }}
+      image: minimal-notebook:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ubuntu-latest
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+    needs: [base-notebook]
+
+  scipy-notebook:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
+      image: scipy-notebook:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ubuntu-latest
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+    needs: [minimal-notebook]
+

.github/workflows/build-test-datascience.yml

@@ -0,0 +1,88 @@
+name: Build, test, and push jupyter datascience images
+
+on:
+  workflow_call:
+    inputs:
+      python_dev_tag:
+        description: Tag to use for latest base images (foundation, minimal, etc)
+        required: true
+        type: string
+      registry:
+        description: The list of tags space separated values
+        required: true
+        type: string
+      runs-on:
+        description: GitHub Actions Runner image
+        required: true
+        type: string
+    secrets:
+      registry_username:
+        required: true
+      registry_token:
+        required: true
+
+defaults:
+  run:
+    working-directory: ./docker-stacks
+
+jobs:
+  r:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
+      image: r-notebook:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ${{ inputs.runs-on }}
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+
+  datascience:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
+      image: datascience-notebook:${{ inputs.python_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ${{ inputs.runs-on }}
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+
+  # julia:
+  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   with:
+  #     parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
+  #     image: julia-notebook
+  #     registry: ${{ inputs.registry }}
+  #     runs-on: ${{ inputs.runs-on }}
+  #   secrets:
+  #     registry_username: ${{ secrets.registry_username }}
+  #     registry_token: ${{ secrets.registry_token }}
+
+  # tensorflow:
+  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   with:
+  #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
+  #     image: tensorflow-notebook
+  #     registry: ${{ inputs.registry }}
+  #     runs-on: ${{ inputs.runs-on }}
+  #   secrets:
+  #     registry_username: ${{ secrets.registry_username }}
+  #     registry_token: ${{ secrets.registry_token }}
+
+  # pytorch:
+  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   with:
+  #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
+  #     image: pytorch-notebook
+  #     registry: ${{ inputs.registry }}
+  #     runs-on: ${{ inputs.runs-on }}
+  #   secrets:
+  #     registry_username: ${{ secrets.registry_username }}
+  #     registry_token: ${{ secrets.registry_token }}
+
+
+
+
+
+

.github/workflows/build-test-spark.yml

@@ -0,0 +1,85 @@
+name: Build, test, and push jupyter Spark images
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        description: Python version
+        required: true
+        type: string
+      spark_download_url:
+        description: Spark dist download url
+        required: true
+        type: string
+      spark_version:
+        description: Spark version
+        required: true
+        type: string
+      java_version:
+        description: Java version
+        required: true
+        type: string
+      scala_version:
+        description: Scala version
+        required: true
+        type: string
+      hadoop_version:
+        description: Hadoop version
+        required: true
+        type: string
+      python_dev_tag:
+        description: Tag to use for latest base images (foundation, minimal, etc)
+        required: true
+        type: string
+      spark_dev_tag:
+        description: Tag to use for latest pyspark images (pyspark, all-spark, etc)
+        required: true
+        type: string
+      registry:
+        description: The list of tags space separated values
+        required: true
+        type: string
+      runs-on:
+        description: GitHub Actions Runner image
+        required: true
+        type: string
+    secrets:
+      registry_username:
+        required: true
+      registry_token:
+        required: true
+
+defaults:
+  run:
+    working-directory: ./docker-stacks
+
+jobs:
+  pyspark:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
+      image: pyspark-notebook:${{ inputs.spark_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ${{ inputs.runs-on }}
+      build-args:
+        spark_download_url=${{ inputs.spark_download_url }}
+        spark_version=${{ inputs.spark_version }}
+        openjdk_version=${{ inputs.java_version }}
+        scala_version=${{ inputs.scala_version }}
+        hadoop_version=${{ inputs.hadoop_version }}
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+
+  all-spark:
+    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    with:
+      parent-image: pyspark-notebook:${{ inputs.spark_dev_tag }}
+      image: all-spark-notebook:${{ inputs.spark_dev_tag }}
+      registry: ${{ inputs.registry }}
+      runs-on: ${{ inputs.runs-on }}
+    secrets:
+      registry_username: ${{ secrets.registry_username }}
+      registry_token: ${{ secrets.registry_token }}
+    needs: [pyspark]
+