Update task_cache.py #3759

Workflow file for this run

	name: Master

	on:
	push:
	branches:
	- master
	paths-ignore:
	- "docs/**"
	pull_request:
	paths-ignore:
	- "docs/**"

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	lint:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: [ "3.11" ]
	steps:
	- uses: actions/checkout@v2
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v2
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }}
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r dev-requirements.txt
	- name: Pip info
	run: python -m pip list
	- name: Lint
	run: pre-commit run --all --show-diff-on-failure
	# This is the build system for the new example directory structure
	list_examples:
	runs-on: ubuntu-latest
	name: "Create a list of example packages"
	steps:
	- uses: actions/checkout@v2
	- name: "Provide the list"
	id: create-example-list
	run: \|
	# TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc)
	echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \
	\| grep -v -e 'testing' -e 'examples' \
	\| grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'mnist_classifier' \
	\| sort \
	\| jq --raw-input . \
	\| jq --slurp . \
	\| jq -c .)" >> "$GITHUB_OUTPUT"
	outputs:
	packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
	serialize:
	needs: [list_examples]
	runs-on: "ubuntu-latest"
	env:
	FLYTE_SDK_RICH_TRACEBACKS: "0"
	strategy:
	fail-fast: false
	matrix:
	example: "${{ fromJson(needs.list_examples.outputs.packages) }}"
	steps:
	- uses: actions/checkout@v4
	- uses: insightsengineering/disk-space-reclaimer@v1
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.10"
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	working-directory: examples/${{ matrix.example }}
	run: \|
	pip install uv
	uv venv
	source .venv/bin/activate
	if [ -f requirements.in ]; then uv pip install -r requirements.in; fi
	uv pip install "flytekit>=1.12.2" flytekitplugins-envd "numpy<2.0.0"
	pip freeze
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	- name: Login to GitHub Container Registry
	if: ${{ github.event_name != 'pull_request' }}
	uses: docker/login-action@v1
	with:
	registry: ghcr.io
	username: ${{ secrets.FLYTE_BOT_USERNAME }}
	password: ${{ secrets.FLYTE_BOT_PAT }}
	- name: Check if dockerfile exists
	working-directory: examples/${{ matrix.example }}
	id: dockerfile
	run: \|
	if [ -f Dockerfile ]
	then
	echo "exist=true" >> "$GITHUB_OUTPUT"
	else
	echo "exist=false" >> "$GITHUB_OUTPUT"
	fi
	- name: Build and push default image
	if: ${{ steps.dockerfile.outputs.exist == 'true' }}
	uses: docker/build-push-action@v5
	with:
	context: examples/${{ matrix.example }}
	push: ${{ github.event_name != 'pull_request' }}
	tags: ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }},ghcr.io/flyteorg/flytecookbook:latest
	cache-from: type=gha
	cache-to: type=gha,mode=max
	- name: Pyflyte package
	working-directory: examples/${{ matrix.example }}
	run: \|
	export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }}
	default_image=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }}
	source .venv/bin/activate
	pyflyte \
	--pkgs ${{ matrix.example }} package \
	--image $default_image \
	--image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
	--image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
	--output flyte-package.tgz \
	--force
	tar -xvf flyte-package.tgz
	- name: Upload artifacts
	uses: actions/upload-artifact@v2
	with:
	name: snacks-examples-${{ matrix.example }}
	path: examples/${{ matrix.example }}/*/.pb

	# Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them.
	register:
	name: Register example to sandbox
	runs-on: ubuntu-latest
	needs: [ "serialize" ]
	steps:
	- uses: actions/checkout@v2
	with:
	fetch-depth: "0"
	- uses: unionai/[email protected]
	- name: setup download artifact dir
	run: \|
	mkdir download-artifact
	- name: Download artifacts
	uses: actions/download-artifact@v2
	with:
	path: ./download-artifact/
	- name: setup sandbox
	run: \|
	flytectl demo start
	flytectl config init
	- name: Register examples
	run: \|
	flytectl register files ./download-artifact/*/ -p flytesnacks -d development --version latest

	bump_version:
	name: Bump Version
	if: ${{ github.event_name != 'pull_request' }}
	needs: [ serialize ] # Only to ensure it can successfully build
	uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
	secrets:
	FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

	prerelease:
	name: Create Prerelease
	needs: [ bump_version ]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	with:
	fetch-depth: "0"
	- name: Create Pre release
	id: prerelease
	run: \|
	RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
	--header 'Accept: application/vnd.github.v3+json' \
	--header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
	--data-raw '{
	"tag_name": "'${{ needs.bump-version.outputs.version }}'",
	"prerelease": true
	}' \| jq -r '.id')
	echo ::set-output name=release_id::$RELEASE_ID
	outputs:
	release_id: ${{ steps.prerelease.outputs.release_id }}

	# Download artifacts again and push them to the release only if this is not a pull request
	release_workflow:
	name: Publish artifacts to github release
	runs-on: ubuntu-latest
	needs: [ prerelease ]
	strategy:
	matrix:
	python-version: [ "3.11" ]
	steps:
	- uses: actions/checkout@v2
	with:
	fetch-depth: "0"
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: setup download artifact dir
	run: \|
	mkdir download-artifact
	- name: Download artifacts
	uses: actions/download-artifact@v2
	with:
	path: ./download-artifact/
	- name: Package Examples
	run: \|
	mkdir -p release-snacks
	cd download-artifact
	for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
	cd .. && sudo rm -rf download-artifact/
	cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
	cp flyte_tests.txt release-snacks/flyte_tests.txt
	- name: Release test manifest
	uses: goreleaser/goreleaser-action@v5
	with:
	version: latest
	args: release --rm-dist
	env:
	GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
	GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}

	make_release:
	name: Mark github pre-release as Release
	needs: [ release_workflow ]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	with:
	fetch-depth: "0"
	- name: Update Release
	id: release
	run: \|
	curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
	--header 'Accept: application/vnd.github.v3+json' \
	--header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
	--data-raw '{
	"tag_name": "'${{ needs.bump-version.outputs.version }}'",
	"prerelease": false
	}'

	e2e-tests:
	runs-on: ubuntu-latest
	env:
	FLYTESNACKS_PRIORITIES: "P0"
	FLYTESNACKS_VERSION: ""
	timeout-minutes: 30
	steps:
	- uses: insightsengineering/disk-space-reclaimer@v1
	- name: Set latest Flytesnacks release
	if: ${{ env.FLYTESNACKS_VERSION == '' }}
	run: \|
	FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest \| jq -r .tag_name)"
	echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
	- name: Checkout
	uses: actions/checkout@v3
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.11"
	- uses: unionai/[email protected]
	- name: Setup sandbox
	run: \|
	mkdir -p ~/.flyte/sandbox
	cat << EOF > ~/.flyte/sandbox/config.yaml
	task_resources:
	defaults:
	cpu: "0"
	memory: "0"
	limits:
	cpu: "0"
	memory: "0"
	EOF
	flytectl demo start --imagePullPolicy Never
	- name: Install Python dependencies
	run: \|
	pip install uv
	uv venv
	source .venv/bin/activate
	uv pip install "flytekit>=1.12.2" flytekitplugins-deck-standard flytekitplugins-envd torch tabulate
	pip freeze
	- name: Checkout flytesnacks
	uses: actions/checkout@v3
	with:
	repository: flyteorg/flytesnacks
	path: flytesnacks
	- name: Verify existence of the tests
	run: \|
	source .venv/bin/activate
	python flyte_tests_validate.py
	- name: Register specific tests
	run: \|
	source .venv/bin/activate
	while read -r line;
	do
	pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
	register \
	--project flytesnacks \
	--domain development \
	--image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
	--version ${{ env.FLYTESNACKS_VERSION }} \
	flytesnacks/$line;
	done < flyte_tests.txt
	- name: Setup tmate session
	if: ${{ failure() }}
	uses: mxschmitt/action-tmate@v3

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update task_cache.py #3759

Workflow file

Update task_cache.py #3759

Jobs

Run details

Workflow file for this run