-
Notifications
You must be signed in to change notification settings - Fork 17
/
Makefile
134 lines (106 loc) · 6.55 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# Put targets here if there is a risk that a target name might conflict with a filename.
# this list is probably overkill right now.
# See: https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
.PHONY: test test-unit test-e2e images run format verify
ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results)
TEST_TAGS := $(if $(TEST_TAGS),$(TEST_TAGS),"")
SUITE_ID := $(if $(SUITE_ID),$(SUITE_ID),"nosuite")
PROVIDER := $(if $(PROVIDER),$(PROVIDER),"openai")
MODEL := $(if $(MODEL),$(MODEL),"gpt-3.5-turbo")
images: requirements.txt ## Build container images
scripts/build-container.sh
install-tools: install-woke ## Install required utilities/tools
# OLS 1085: Service build failure issue caused by newest PDM version
# (right now we need to stick to PDM specified in pyproject.toml file)
@command -v pdm > /dev/null || { echo >&2 "pdm is not installed. Installing..."; pip install pdm; }
pdm --version
# this is quick fix for OLS-758: "Verify" CI job is broken after new Mypy 1.10.1 was released 2 days ago
# CI job configuration would need to be updated in follow-up task
# pip uninstall -y mypy 2> /dev/null || true
# display setuptools version
pip show setuptools
# install all dependencies, including devel ones
pdm install --dev
# check that correct mypy version is installed
# mypy --version
pdm run mypy --version
# check that correct Black version is installed
pdm run black --version
# check that correct Ruff version is installed
pdm run ruff --version
# check that correct Pydocstyle version is installed
pdm run pydocstyle --version
install-woke: ## Install woke, required for Inclusive Naming scan
@command -v ./woke > /dev/null || { echo >&2 "woke is not installed. Installing..."; curl -sSfL https://git.io/getwoke | bash -s -- -b ./; }
pdm-lock-check: ## Check that the pdm.lock file is in a good shape
pdm lock --check
install-deps: install-tools pdm-lock-check ## Install all required dependencies needed to run the service, according to pdm.lock
pdm sync
install-deps-test: install-tools pdm-lock-check ## Install all required dev dependencies needed to test the service, according to pdm.lock
pdm sync --dev
update-deps: ## Check pyproject.toml for changes, update the lock file if needed, then sync.
pdm install
pdm install --dev
run: ## Run the service locally
python runner.py
test: test-unit test-integration test-e2e ## Run all tests
benchmarks: ## Run benchmarks
@echo "Running benchmarks..."
python -m pytest tests/benchmarks --benchmark-histogram
test-unit: ## Run the unit tests
@echo "Running unit tests..."
@echo "Reports will be written to ${ARTIFACT_DIR}"
COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" python -m pytest tests/unit --cov=ols --cov=runner --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml"
python scripts/transform_coverage_report.py "${ARTIFACT_DIR}/coverage_unit.json" "${ARTIFACT_DIR}/coverage_unit.out"
scripts/codecov.sh "${ARTIFACT_DIR}/coverage_unit.out"
test-integration: ## Run integration tests tests
@echo "Running integration tests..."
@echo "Reports will be written to ${ARTIFACT_DIR}"
COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.integration" python -m pytest -m 'not redis' tests/integration --cov=ols --cov=runner --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_integration.json" --junit-xml="${ARTIFACT_DIR}/junit_integration.xml" --cov-fail-under=60
python scripts/transform_coverage_report.py "${ARTIFACT_DIR}/coverage_integration.json" "${ARTIFACT_DIR}/coverage_integration.out"
scripts/codecov.sh "${ARTIFACT_DIR}/coverage_integration.out"
check-coverage: test-unit test-integration ## Unit tests and integration tests overall code coverage check
coverage combine --keep "${ARTIFACT_DIR}/.coverage.unit" "${ARTIFACT_DIR}/.coverage.integration"
# the threshold should be very high there, in theory it should reach 100%
coverage report -m --fail-under=94
test-e2e: ## Run e2e tests - requires running OLS server
@echo "Running e2e tests..."
@echo "Reports will be written to ${ARTIFACT_DIR}"
python -m pytest tests/e2e -s --durations=0 -o junit_suite_name="${SUITE_ID}" -m "${TEST_TAGS}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
--eval_provider ${PROVIDER} --eval_model ${MODEL} --eval_out_dir ${ARTIFACT_DIR} --rp_name=ols-e2e-tests
coverage-report: unit-tests-coverage-report integration-tests-coverage-report ## Export coverage reports into interactive HTML
unit-tests-coverage-report: test-unit ## Export unit test coverage report into interactive HTML
coverage html --data-file="${ARTIFACT_DIR}/.coverage.unit" -d htmlcov-unit
integration-tests-coverage-report: test-integration ## Export integration test coverage report into interactive HTML
coverage html --data-file="${ARTIFACT_DIR}/.coverage.integration" -d htmlcov-integration
check-types: ## Checks type hints in sources
pdm run mypy --explicit-package-bases --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs ols/
security-check: ## Check the project for security issues
bandit -c pyproject.toml -r .
format: ## Format the code into unified format
pdm run black .
pdm run ruff check . --fix --per-file-ignores=tests/*:S101 --per-file-ignores=scripts/*:S101
verify: install-woke install-deps-test ## Verify the code using various linters
pdm run black . --check
pdm run ruff check . --per-file-ignores=tests/*:S101 --per-file-ignores=scripts/*:S101
./woke . --exit-1-on-failure
schema: ## Generate OpenAPI schema file
python scripts/generate_openapi_schema.py docs/openapi.json
requirements.txt: pyproject.toml pdm.lock ## Generate requirements.txt file containing hashes for all non-devel packages
pdm export --prod --format requirements --output requirements.txt
verify-packages-completeness: requirements.txt ## Verify that requirements.txt file contains complete list of packages
pip download -d /tmp/ --use-pep517 --verbose -r requirements.txt
get-rag: ## Download a copy of the RAG embedding model and vector database
podman create --replace --name tmp-rag-container $$(cat build.args | awk 'BEGIN{FS="="}{print $$2}') true
rm -rf vector_db embeddings_model
podman cp tmp-rag-container:/rag/vector_db vector_db
podman cp tmp-rag-container:/rag/embeddings_model embeddings_model
podman rm tmp-rag-container
help: ## Show this help screen
@echo 'Usage: make <OPTIONS> ... <TARGETS>'
@echo ''
@echo 'Available targets are:'
@echo ''
@grep -E '^[ a-zA-Z0-9_.-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
@echo ''