Skip to content

CI Overhaul

CI Overhaul #36

Workflow file for this run

name: Master
on:
push:
branches:
- master
paths-ignore:
- "docs/**"
pull_request:
paths-ignore:
- "docs/**"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
# This is the build system for the new example directory structure
list_examples:
runs-on: ubuntu-latest
name: "Create a list of example packages"
steps:
- uses: actions/checkout@v2
- name: "Provide the list"
id: create-example-list
run: |
echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \
| grep -v -e 'testing' -e 'examples' \
| grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \
| sort \
| jq --raw-input . \
| jq --slurp . \
| jq -c .)" >> "$GITHUB_OUTPUT"
outputs:
packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
serialize:
needs: [list_examples]
runs-on: "ubuntu-latest"
env:
FLYTE_SDK_RICH_TRACEBACKS: "0"
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
strategy:
fail-fast: false
matrix:
example: "${{ fromJson(needs.list_examples.outputs.packages) }}"
steps:
- uses: actions/checkout@v4
- name: 'Clear action cache'
uses: ./.github/actions/clear-action-cache
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
working-directory: examples/${{ matrix.example }}
run: |
pip install uv
uv venv
source .venv/bin/activate
uv pip install flytekit flytekitplugins-envd
if [ -f requirements.in ]; then uv pip install -r requirements.in; fi
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ secrets.FLYTE_BOT_USERNAME }}
password: ${{ secrets.FLYTE_BOT_PAT }}
- name: Pyflyte package
working-directory: examples/${{ matrix.example }}
run: |
source .venv/bin/activate
pyflyte \
--pkgs ${{ matrix.example }} package \
--image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
--image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
--output flyte-package.tgz \
--force
tar -xvf flyte-package.tgz
- name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: snacks-examples-${{ matrix.example }}
path: examples/${{ matrix.example }}/**/*.pb
# Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them.
register:
name: Register example to sandbox
runs-on: ubuntu-latest
needs: [ "serialize" ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- uses: unionai/[email protected]
- name: setup download artifact dir
run: |
mkdir download-artifact
- name: Download artifacts
uses: actions/download-artifact@v2
with:
path: ./download-artifact/
- name: setup sandbox
run: |
flytectl demo start
flytectl config init
- name: Register examples
uses: unionai/[email protected]
with:
flytesnacks: false
proto: ./download-artifact/**/*
project: flytesnacks
version: "latest"
domain: development