Makefile

# Copyright 2020-2021 kubeflow.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Acknowledgements:
#  - The help target was derived from https://stackoverflow.com/a/35730328/5601796

VENV ?= .venv
KFP_TEKTON_RELEASE ?= v1.9.2
export VIRTUAL_ENV := $(abspath ${VENV})
export PATH := ${VIRTUAL_ENV}/bin:${PATH}
DOCKER_REGISTRY ?= aipipeline
GITHUB_ACTION ?= false

.PHONY: help
help: ## Display the Make targets
	@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'

.PHONY: venv
venv: $(VENV)/bin/activate ## Create and activate virtual environment
$(VENV)/bin/activate: sdk/python/setup.py
# create/update the VENV when there was a change to setup.py
# check if kfp-tekton is already installed (Travis/CI did during install step)
# use pip from the specified VENV as opposed to any pip available in the shell
	@echo "VENV=$(VENV)"
	@test -d $(VENV) || python3 -m venv $(VENV)
	@$(VENV)/bin/pip show kfp-tekton >/dev/null 2>&1 || $(VENV)/bin/pip install -e sdk/python
	@if [ "$(GITHUB_ACTION)" = "false" ]; then touch $(VENV)/bin/activate; fi

.PHONY: install
install: venv ## Install the kfp_tekton compiler in a virtual environment
	@echo "Run 'source $(VENV)/bin/activate' to activate the virtual environment."

.PHONY: validate-generated-test-yamls
validate-generated-test-yamls:
	@echo "=================================================================="
	@echo "Reporting files with same non-inlined and inlined generated yamls in testdata:"
	@find sdk/python/tests/compiler/testdata \
		\( -type f -name "*yaml" -name "*noninlined.yaml" \) | sort -z >/tmp/validate-generated-test-yamls_total
	@find sdk/python/tests/compiler/testdata \
		\( -type f -name "*yaml" -name "*noninlined.yaml" \) | \
		sed -n -e 's/\(.*\)_noninlined.yaml/\1/p' | \
		xargs -n1 -I '{}' diff -q '{}.yaml' '{}_noninlined.yaml' | cut -f4 -d' ' | \
		sort -z >/tmp/validate-generated-test-yamls_valid
	@echo "=================================================================="
	@echo "Noninlined and inlined testdata yamls, having same content."
	@diff -a /tmp/validate-generated-test-yamls_total /tmp/validate-generated-test-yamls_valid
	@echo "$@: OK"

.PHONY: validate-testdata
validate-testdata:
	@cd tekton-catalog/pipeline-loops/ && make validate-testdata-python-sdk
	@echo "$@: OK"

.PHONY: validate-pipelineloop-examples
validate-pipelineloop-examples:
	@cd tekton-catalog/pipeline-loops/ && make validate-examples
	@echo "$@: OK"

.PHONY: unit_test
unit_test: venv ## Run compiler unit tests
	@echo "=================================================================="
	@echo "Optional environment variables to configure $@, examples:"
	@sed -n -e 's/# *\(make $@ .*\)/  \1/p' sdk/python/tests/compiler/compiler_tests.py
	@echo "=================================================================="
	@pip show pytest > /dev/null 2>&1 || pip install pytest
	@sdk/python/tests/run_tests.sh
	@echo "$@: OK"

.PHONY: ci_unit_test
ci_unit_test: unit_test

.PHONY: e2e_test
e2e_test: venv ## Run compiler end-to-end tests (requires kubectl and tkn CLI)
	@echo "=================================================================="
	@echo "Optional environment variables to configure $@, examples:"
	@sed -n -e 's/# *\(make $@ .*\)/  \1/p' sdk/python/tests/compiler/compiler_tests_e2e.py
	@echo "=================================================================="
	@which kubectl > /dev/null || (echo "Missing kubectl CLI" && exit 1)
	@test -z "${KUBECONFIG}" && echo "KUBECONFIG not set" || echo "KUBECONFIG=${KUBECONFIG}"
	@kubectl version --short || (echo "Failed to access kubernetes cluster" && exit 1)
	@which tkn > /dev/null || (echo "Missing tkn CLI" && exit 1)
	@sdk/python/tests/run_e2e_tests.sh
	@echo "$@: OK"

.PHONY: test
test: unit_test e2e_test ## Run compiler unit tests and end-to-end tests
	@echo "$@: OK"

.PHONY: report
report: ## Report compilation status of KFP testdata DSL scripts
	@sdk/python/tests/test_kfp_samples.sh
	@echo "$@: OK"

.PHONY: lint
lint: venv ## Check Python code style compliance
	@which flake8 > /dev/null || pip install flake8
	@flake8 sdk/python --show-source --statistics \
		--select=E9,E2,E3,E5,F63,F7,F82,F4,F841,W291,W292 \
		--per-file-ignores sdk/python/tests/compiler/testdata/*:F841,F821 \
 		--max-line-length=140
	@echo "$@: OK"

.PHONY: check_license
check_license: ## Check for license header in source files
	@find ./sdk/python -type f \( -name '*.py' -o -name '*.yaml' \) -exec \
		grep -H -E -o -c 'Copyright 20.* kubeflow.org' {} \; | \
		grep -E ':0$$' | sed 's/..$$//' | \
		grep . && echo "The files listed above are missing the license header" && exit 1 || \
		echo "$@: OK"

.PHONY: check_mdtoc
check_mdtoc: ## Check Markdown files for valid the Table of Contents
	@find guides samples sdk *.md -type f -name '*.md' -exec \
		grep -l -i 'Table of Contents' {} \; | sort | \
		while read -r md_file; do \
			grep -oE '^ *[-+*] \[[^]]+\]\(#[^)]+\)' "$${md_file}" |  sed -e 's/[-+*] /- /g' > md_file_toc; \
			./tools/mdtoc.sh "$${md_file}" > generated_toc; \
			diff -w md_file_toc generated_toc || echo "$${md_file}"; \
			rm -f md_file_toc generated_toc; \
		done | grep . && echo "Run './tools/mdtoc.sh <md-file>' to update the 'Table of Contents' in the Markdown files reported above." && exit 1 || \
		echo "$@: OK"

.PHONY: check_doc_links
check_doc_links: ## Check Markdown files for valid links
	@pip3 show requests > /dev/null || pip install requests
	@python3 tools/python/verify_doc_links.py
	@echo "$@: OK"

.PHONY: verify
verify: check_license check_mdtoc check_doc_links lint unit_test report ## Run all verification targets: check_license, check_mdtoc, lint, unit_test, report
	@echo "$@: OK"

.PHONY: distribution
distribution: venv ## Create a distribution and upload to test.PyPi.org
	@echo "NOTE: Using test.PyPi.org -- edit Makefile to target real PyPi index"
	@twine --version > /dev/null 2>&1 || pip install twine
	@cd sdk/python && \
		rm -rf dist/ && \
		python3 setup.py sdist && \
		twine check dist/* && \
		twine upload --repository testpypi dist/*

.PHONY: build
build: ## Create GO vendor directories with all dependencies
	go mod vendor
	# Extract go licenses into a single file. This assume licext is install globally through
	# npm install -g license-extractor
	# See https://github.com/arei/license-extractor
	licext --mode merge --source vendor/ --target third_party/license.txt --overwrite
	# Delete vendor directory
	rm -rf vendor

.PHONY: build-release-template
build-release-template: ## Build KFP Tekton release deployment templates
	@mkdir -p install/$(KFP_TEKTON_RELEASE)
	@kustomize build manifests/kustomize/env/kfp-template -o install/$(KFP_TEKTON_RELEASE)/kfp-tekton.yaml

.PHONY: build-backend
build-backend: build-apiserver build-agent build-workflow build-cacheserver ## Verify apiserver, agent, and workflow build
	@echo "$@: OK"

.PHONY: build-apiserver
build-apiserver: ## Build apiserver
	go build -o apiserver ./backend/src/apiserver

.PHONY: build-agent
build-agent: ## Build agent
	go build -o agent ./backend/src/agent/persistence

.PHONY: build-workflow
build-workflow: ## Build workflow
	go build -o workflow ./backend/src/crd/controller/scheduledworkflow/*.go

.PHONY: build-cacheserver
build-cacheserver: ## Build cache
	go build -o cache ./backend/src/cache/*.go

.PHONY: build-backend-images
build-backend-images: \
	build-api-server-image \
	build-persistenceagent-image \
	build-metadata-writer-image \
	build-scheduledworkflow-image \
	build-cacheserver-image \
	## Build backend docker images
	@echo "$@: OK"

.PHONY: build-v2-custom-controller-images
build-v2-custom-controller-images: \
	build-tekton-exithandler-controller-image \
	build-tekton-exithandler-webhook-image \
	build-tekton-kfptask-controller-image \
	build-tekton-kfptask-webhook-image \
	## Build V2 Tekton custom task controller images 
	@echo "$@: OK"

.PHONY: build-api-server-image
build-api-server-image: ## Build api-server docker image
	docker build -t ${DOCKER_REGISTRY}/api-server -f backend/Dockerfile .

.PHONY: build-persistenceagent-image
build-persistenceagent-image: ## Build persistenceagent docker image
	docker build -t ${DOCKER_REGISTRY}/persistenceagent -f backend/Dockerfile.persistenceagent .

.PHONY: build-metadata-writer-image
build-metadata-writer-image: ## Build metadata-writer docker image
	docker build -t ${DOCKER_REGISTRY}/metadata-writer -f backend/metadata_writer/Dockerfile .

.PHONY: build-scheduledworkflow-image
build-scheduledworkflow-image: ## Build scheduledworkflow docker image
	docker build -t ${DOCKER_REGISTRY}/scheduledworkflow -f backend/Dockerfile.scheduledworkflow .

.PHONY: build-cacheserver-image
build-cacheserver-image: ## Build cacheserver docker image
	docker build -t ${DOCKER_REGISTRY}/cache-server -f backend/Dockerfile.cacheserver .

.PHONY: build-tekton-exithandler-controller-image
build-tekton-exithandler-controller-image: ## Build tekton-exithandler-controller docker image
	@cd tekton-catalog/tekton-exithandler/ && docker build -t ${DOCKER_REGISTRY}/tekton-exithandler-controller -f Dockerfile.tekton-exithandler.controller .

.PHONY: build-tekton-exithandler-webhook-image
build-tekton-exithandler-webhook-image: ## Build tekton-exithandler-webhook docker image
	@cd tekton-catalog/tekton-exithandler/ && docker build -t ${DOCKER_REGISTRY}/tekton-exithandler-webhook -f Dockerfile.tekton-exithandler.webhook .

.PHONY: build-tekton-kfptask-controller-image
build-tekton-kfptask-controller-image: ## Build tekton-kfptask-controller docker image
	@cd tekton-catalog/tekton-kfptask/ && docker build -t ${DOCKER_REGISTRY}/tekton-kfptask-controller -f Dockerfile.tekton-kfptask.controller .

.PHONY: build-tekton-kfptask-webhook-image
build-tekton-kfptask-webhook-image: ## Build tekton-kfptask-webhook docker image
	@cd tekton-catalog/tekton-kfptask/ && docker build -t ${DOCKER_REGISTRY}/tekton-kfptask-webhook -f Dockerfile.tekton-kfptask.webhook .

.PHONY: build-tekton-driver-image
build-tekton-driver-image: ## Build tekton-driver docker image
	@cd tekton-catalog/tekton-driver/ && docker build -t ${DOCKER_REGISTRY}/tekton-driver -f Dockerfile.tektondriver .

.PHONY: run-go-unittests
run-go-unittests: \
	run-apiserver-unittests \
	run-common-unittests \
	run-crd-unittests \
	run-persistenceagent-unittests \
	run-cacheserver-unittests \
	run-tekton-exithandler-unittests \
	run-tekton-kfptask-unittests \
	## Verify go backend unit tests
	@echo "$@: OK"

run-apiserver-unittests: # apiserver golang unit tests
	go test -v -cover ./backend/src/apiserver/...

run-common-unittests: # common golang unit tests
	go test -v -cover ./backend/src/common/...

run-crd-unittests: # crd golang unit tests
	go test -v -cover ./backend/src/crd/...

run-persistenceagent-unittests: # persistence agent golang unit tests
	go test -v -cover ./backend/src/agent/...

run-cacheserver-unittests: # cache golang unit tests
	go test -v -cover ./backend/src/cache/...

run-tekton-exithandler-unittests: # tekton-exithandler golang unit tests
	@cd tekton-catalog/tekton-exithandler/ && go test -v -cover ./...

run-tekton-kfptask-unittests: # tekton-kfptask golang unit tests
	@cd tekton-catalog/tekton-kfptask/ && go test -v -cover ./...

run-tekton-driver-unittests: # tekton-driver golang unit tests
	@cd tekton-catalog/tekton-driver/ && go test -v -cover ./...