-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from idirze/build-pipeline
Jupyterlab images build pipeline
- Loading branch information
Showing
275 changed files
with
14,838 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
################ Version Compatibility Matrix ###################### | ||
##### SECTION USED TO GET COMPATIBLE VERSIONS ###################### | ||
##### DONT PUT python_version IN AN ARRAY - JAVA COMPATIBILITY ##### | ||
####### ENRICH WITH NEWER VERSIONS ################################# | ||
compatibility-matrix: | ||
# Maximum python version supported by spark-3.2.x: 3.9 | ||
# Java support: 8/11 | ||
- python_version: 3.9 | ||
spark_version: [3.2.1, 3.2.2, 3.2.3, 3.2.4] | ||
java_version: [11] | ||
scala_version: [2.12, 2.13] | ||
hadoop_version: 3.2 | ||
spark_download_url: https://archive.apache.org/dist/spark/ | ||
# Maximum python version supported by spark-3.3.x: 3.10 | ||
# Java support: 8/11/17 | ||
- python_version: '3.10' | ||
spark_version: [3.3.1, 3.3.2, 3.3.3, 3.3.4] | ||
java_version: [17] | ||
scala_version: [2.12, 2.13] | ||
hadoop_version: 3 | ||
spark_download_url: https://archive.apache.org/dist/spark/ | ||
# Maximum python version supported by spark-3.4.x: 3.11 | ||
# Java support: 8/11/17 | ||
- python_version: 3.11 | ||
spark_version: [3.4.1, 3.4.2] | ||
java_version: [17] | ||
scala_version: [2.12, 2.13] | ||
hadoop_version: 3 | ||
spark_download_url: https://archive.apache.org/dist/spark/ | ||
# https://spark.apache.org/releases/spark-release-3-5-0.html | ||
# Minimum supported java version: 17/21 | ||
- python_version: 3.11 | ||
spark_version: [3.5.0] | ||
java_version: [17] | ||
scala_version: [2.12, 2.13] | ||
hadoop_version: 3 | ||
spark_download_url: https://archive.apache.org/dist/spark/ | ||
### Override the matrix above by providing the versions to build | ||
### 1- The build-matrix is empty: build with all possible combintations | ||
### 2- Override specific versions: build with all possible combinations which are compatible with that specific versions | ||
### 3- The versions not present on compatibility-matrix are ignored | ||
build-matrix: | ||
python_version: ['3.9', '3.10', '3.11'] | ||
spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.0] | ||
java_version: [11, 17] | ||
scala_version: [2.12] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
name: Set up dev env requirements | ||
description: Set up dev env requirements for build and tests | ||
|
||
runs: | ||
using: composite | ||
steps: | ||
- name: Setup dev env requirement 📦 | ||
run: | | ||
cp -f requirements-dev.txt ../ | ||
cat ../python/requirements-extended.txt >> ../requirements-dev.txt | ||
working-directory: ./docker-stacks | ||
shell: bash | ||
|
||
- name: Copy patchs 📦 | ||
run: | | ||
cp -fr ../python/okdp/patch/* ./ | ||
cp -fr ../python/okdp ./ | ||
working-directory: ./docker-stacks | ||
shell: bash | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
# Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 | ||
name: Automatic partital rerun on workflow failure | ||
on: | ||
workflow_dispatch: | ||
inputs: | ||
run_id: | ||
required: true | ||
workflow_name: | ||
required: true | ||
workflow_sha: | ||
required: true | ||
jobs: | ||
rerun: | ||
runs-on: ubuntu-latest | ||
permissions: | ||
contents: write | ||
actions: write | ||
packages: write | ||
steps: | ||
- name: Re-run workflow ${{ inputs.workflow_name }}-${{ inputs.run_id }} | ||
env: | ||
GH_REPO: ${{ github.repository }} | ||
GH_TOKEN: ${{ github.token }} | ||
GH_DEBUG: api | ||
run: | | ||
gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 | ||
gh run rerun ${{ inputs.run_id }} --failed |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
name: Build, test, and push jupyter base images | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
python_version: | ||
description: Python version | ||
required: true | ||
type: string | ||
python_dev_tag: | ||
description: Tag to use for latest base images (foundation, minimal, etc) | ||
required: true | ||
type: string | ||
registry: | ||
description: The list of tags space separated values | ||
required: true | ||
type: string | ||
runs-on: | ||
description: GitHub Actions Runner image | ||
required: true | ||
type: string | ||
secrets: | ||
registry_username: | ||
required: true | ||
registry_token: | ||
required: true | ||
|
||
defaults: | ||
run: | ||
working-directory: ./docker-stacks | ||
|
||
jobs: | ||
|
||
docker-stacks-foundation: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: "" | ||
image: docker-stacks-foundation:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
build-args: | ||
PYTHON_VERSION=${{ inputs.python_version }} | ||
runs-on: ubuntu-latest | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
|
||
base-notebook: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: docker-stacks-foundation:${{ inputs.python_dev_tag }} | ||
image: base-notebook:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ubuntu-latest | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
needs: [docker-stacks-foundation] | ||
|
||
minimal-notebook: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: base-notebook:${{ inputs.python_dev_tag }} | ||
image: minimal-notebook:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ubuntu-latest | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
needs: [base-notebook] | ||
|
||
scipy-notebook: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: minimal-notebook:${{ inputs.python_dev_tag }} | ||
image: scipy-notebook:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ubuntu-latest | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
needs: [minimal-notebook] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
name: Build, test, and push jupyter datascience images | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
python_dev_tag: | ||
description: Tag to use for latest base images (foundation, minimal, etc) | ||
required: true | ||
type: string | ||
registry: | ||
description: The list of tags space separated values | ||
required: true | ||
type: string | ||
runs-on: | ||
description: GitHub Actions Runner image | ||
required: true | ||
type: string | ||
secrets: | ||
registry_username: | ||
required: true | ||
registry_token: | ||
required: true | ||
|
||
defaults: | ||
run: | ||
working-directory: ./docker-stacks | ||
|
||
jobs: | ||
r: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: minimal-notebook:${{ inputs.python_dev_tag }} | ||
image: r-notebook:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ${{ inputs.runs-on }} | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
|
||
datascience: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: scipy-notebook:${{ inputs.python_dev_tag }} | ||
image: datascience-notebook:${{ inputs.python_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ${{ inputs.runs-on }} | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
|
||
# julia: | ||
# uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
# with: | ||
# parent-image: minimal-notebook:${{ inputs.python_dev_tag }} | ||
# image: julia-notebook | ||
# registry: ${{ inputs.registry }} | ||
# runs-on: ${{ inputs.runs-on }} | ||
# secrets: | ||
# registry_username: ${{ secrets.registry_username }} | ||
# registry_token: ${{ secrets.registry_token }} | ||
|
||
# tensorflow: | ||
# uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
# with: | ||
# parent-image: scipy-notebook:${{ inputs.python_dev_tag }} | ||
# image: tensorflow-notebook | ||
# registry: ${{ inputs.registry }} | ||
# runs-on: ${{ inputs.runs-on }} | ||
# secrets: | ||
# registry_username: ${{ secrets.registry_username }} | ||
# registry_token: ${{ secrets.registry_token }} | ||
|
||
# pytorch: | ||
# uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
# with: | ||
# parent-image: scipy-notebook:${{ inputs.python_dev_tag }} | ||
# image: pytorch-notebook | ||
# registry: ${{ inputs.registry }} | ||
# runs-on: ${{ inputs.runs-on }} | ||
# secrets: | ||
# registry_username: ${{ secrets.registry_username }} | ||
# registry_token: ${{ secrets.registry_token }} | ||
|
||
|
||
|
||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
name: Build, test, and push jupyter Spark images | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
python_version: | ||
description: Python version | ||
required: true | ||
type: string | ||
spark_download_url: | ||
description: Spark dist download url | ||
required: true | ||
type: string | ||
spark_version: | ||
description: Spark version | ||
required: true | ||
type: string | ||
java_version: | ||
description: Java version | ||
required: true | ||
type: string | ||
scala_version: | ||
description: Scala version | ||
required: true | ||
type: string | ||
hadoop_version: | ||
description: Hadoop version | ||
required: true | ||
type: string | ||
python_dev_tag: | ||
description: Tag to use for latest base images (foundation, minimal, etc) | ||
required: true | ||
type: string | ||
spark_dev_tag: | ||
description: Tag to use for latest pyspark images (pyspark, all-spark, etc) | ||
required: true | ||
type: string | ||
registry: | ||
description: The list of tags space separated values | ||
required: true | ||
type: string | ||
runs-on: | ||
description: GitHub Actions Runner image | ||
required: true | ||
type: string | ||
secrets: | ||
registry_username: | ||
required: true | ||
registry_token: | ||
required: true | ||
|
||
defaults: | ||
run: | ||
working-directory: ./docker-stacks | ||
|
||
jobs: | ||
pyspark: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: scipy-notebook:${{ inputs.python_dev_tag }} | ||
image: pyspark-notebook:${{ inputs.spark_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ${{ inputs.runs-on }} | ||
build-args: | ||
spark_download_url=${{ inputs.spark_download_url }} | ||
spark_version=${{ inputs.spark_version }} | ||
openjdk_version=${{ inputs.java_version }} | ||
scala_version=${{ inputs.scala_version }} | ||
hadoop_version=${{ inputs.hadoop_version }} | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
|
||
all-spark: | ||
uses: ./.github/workflows/docker-build-test-push-latest.yml | ||
with: | ||
parent-image: pyspark-notebook:${{ inputs.spark_dev_tag }} | ||
image: all-spark-notebook:${{ inputs.spark_dev_tag }} | ||
registry: ${{ inputs.registry }} | ||
runs-on: ${{ inputs.runs-on }} | ||
secrets: | ||
registry_username: ${{ secrets.registry_username }} | ||
registry_token: ${{ secrets.registry_token }} | ||
needs: [pyspark] | ||
|
Oops, something went wrong.