From 44d5a0c623fddaae441ec7297728f481ca9597d4 Mon Sep 17 00:00:00 2001 From: diversemix Date: Wed, 23 Oct 2024 17:24:19 +0100 Subject: [PATCH 1/6] initial structure --- .github/workflows/ci-cd.yml | 41 ++++++++++++++++++++++++ makefile-docker.defs | 11 +++++++ pyproject.toml | 4 +-- tests/use-cases/test_mcf_uc.py | 58 ++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 tests/use-cases/test_mcf_uc.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 15339de8..38a2c2e0 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -84,6 +84,47 @@ jobs: if: always() && ! cancelled() run: docker compose logs + use-case-tests: + if: | + ! cancelled() && always() && + (needs.check-auto-tagging-will-work.result == 'skipped' || needs.check-auto-tagging-will-work.result == 'success') && + ! startsWith(github.ref, 'refs/tags') + needs: + - check-auto-tagging-will-work + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Use .env.example + run: cp .env.example .env + + - name: Get python Container + run: docker pull python:3.10 + + - name: Install latest Vespa CLI + env: + VESPA_CLI_VERSION: 8.250.43 + run: | + mkdir -p scripts/vespa-cli + curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | \ + tar -zxf - -C scripts/vespa-cli --strip-component=1 + echo "scripts/vespa-cli/bin" >> $GITHUB_PATH + + - name: Build + run: | + docker compose build + docker images + + - name: Build docker-compose stack & Setup vespa for search + run: make start + + - name: Run backend search tests for vespa + run: make test_use_cases + + - name: Log Dump + if: always() && ! cancelled() + run: docker compose logs + search-tests: if: | ! cancelled() && always() && diff --git a/makefile-docker.defs b/makefile-docker.defs index 2aa780f5..15307ebb 100644 --- a/makefile-docker.defs +++ b/makefile-docker.defs @@ -110,6 +110,17 @@ test_search: -vvv tests/search \ -m 'search' ${ARGS} +.ONESHELL: +test_use_cases: + docker compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + run --rm --name search_test \ + -v "${PWD}/data:/data" \ + backend pytest \ + -vvv tests/use-cases \ + -m 'use_case' ${ARGS} + test_cors: docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm backend pytest -vvv -m 'cors' ${ARGS} diff --git a/pyproject.toml b/pyproject.toml index e0ba3706..aecea330 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "navigator_backend" -version = "1.18.0" +version = "1.18.1" description = "" authors = ["CPR-dev-team "] packages = [{ include = "app" }, { include = "tests" }] @@ -58,7 +58,7 @@ build-backend = "poetry.core.masonry.api" [tool.pytest.ini_options] addopts = "-p no:cacheprovider" -markers = ["cors", "search", "unit"] +markers = ["cors", "search", "unit", "use_case"] asyncio_mode = "strict" [tool.pydocstyle] diff --git a/tests/use-cases/test_mcf_uc.py b/tests/use-cases/test_mcf_uc.py new file mode 100644 index 00000000..01e33d96 --- /dev/null +++ b/tests/use-cases/test_mcf_uc.py @@ -0,0 +1,58 @@ +from unittest.mock import patch + +import pytest + +from app.api.api_v1.routers import search +from app.repository.lookups import get_country_slug_from_country_code +from tests.search.vespa.setup_search_tests import ( + VESPA_FIXTURE_COUNT, + _make_search_request, + _populate_db_families, +) + + +@pytest.mark.use_case +@patch( + "app.api.api_v1.routers.search.AppTokenFactory.verify_corpora_in_db", + return_value=True, +) +@pytest.mark.parametrize("label,query", [("search", "the"), ("browse", "")]) +def test_keyword_country_filters__geographies( + mock_corpora_exist_in_db, + label, + query, + test_vespa, + data_client, + data_db, + monkeypatch, + valid_token, +): + monkeypatch.setattr(search, "_VESPA_CONNECTION", test_vespa) + _populate_db_families(data_db) + base_params = {"query_string": query} + + # Get all documents and iterate over their country codes to confirm that each are + # the specific one that is returned in the query (as they each have a unique + # country code) + all_body = _make_search_request(data_client, valid_token, params=base_params) + families = [f for f in all_body["families"]] + assert len(families) == VESPA_FIXTURE_COUNT + + for family in families: + for country_code in family["family_geographies"]: + country_slug = get_country_slug_from_country_code(data_db, country_code) + + params = { + **base_params, + **{"keyword_filters": {"countries": [country_slug]}}, + } + body_with_filters = _make_search_request( + data_client, valid_token, params=params + ) + filtered_family_slugs = [ + f["family_slug"] for f in body_with_filters["families"] + ] + assert len(filtered_family_slugs) == 1 + assert family["family_slug"] in filtered_family_slugs + + assert mock_corpora_exist_in_db.assert_called From 41476c4d9c3cabaf49a97e869db89e23e0696e7a Mon Sep 17 00:00:00 2001 From: diversemix Date: Wed, 23 Oct 2024 17:28:54 +0100 Subject: [PATCH 2/6] update lock file --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 791d3a4f..76562209 100644 --- a/poetry.lock +++ b/poetry.lock @@ -834,7 +834,7 @@ vision = ["Pillow (>=9.4.0)"] [[package]] name = "db-client" -version = "3.8.19" +version = "3.8.18" description = "All things to do with the datamodel and its storage. Including alembic migrations and datamodel code." optional = false python-versions = "^3.9" @@ -854,8 +854,8 @@ SQLAlchemy-Utils = "^0.38.2" [package.source] type = "git" url = "https://github.com/climatepolicyradar/navigator-db-client.git" -reference = "v3.8.19" -resolved_reference = "603666581c5c4aa8436f7c419167a8ae5ab49fb2" +reference = "v3.8.18" +resolved_reference = "b221b4d7a23a111b8b0ce2a19329f2d3a73b0e7c" [[package]] name = "deprecation" @@ -4239,4 +4239,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "711884413de31b1e08edd8d61dc17af9c3254b52bd8988bcd1d33edf3278a63a" +content-hash = "6bcc2beaba87225d85fe50521e88cc2f944d5274c66d8b43b49722fa71c62f39" From 4fabfd22aa94a682b0c91732916b65151c6d9908 Mon Sep 17 00:00:00 2001 From: diversemix Date: Wed, 23 Oct 2024 17:31:18 +0100 Subject: [PATCH 3/6] update needs in ci --- .github/workflows/ci-cd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 38a2c2e0..2fc31121 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -183,6 +183,7 @@ jobs: ! cancelled() && always() && (needs.code-quality.result == 'success' && needs.non-search-tests.result == 'success' && + needs.use-case-tests.result == 'success' && needs.search-tests.result == 'success' && needs.integration-tests.result == 'success') && ! startsWith(github.ref, 'refs/tags') @@ -190,6 +191,7 @@ jobs: needs: - code-quality - non-search-tests + - use-case-tests - search-tests - integration-tests steps: From c7eb5465e944e48f9d723f3f3d347968a2dcb772 Mon Sep 17 00:00:00 2001 From: diversemix Date: Wed, 23 Oct 2024 17:38:42 +0100 Subject: [PATCH 4/6] rename task in ci --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2fc31121..986b5f82 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -118,7 +118,7 @@ jobs: - name: Build docker-compose stack & Setup vespa for search run: make start - - name: Run backend search tests for vespa + - name: Run backend Use Case tests run: make test_use_cases - name: Log Dump From 130d7fc9a3470a080946b84f2e7f8d83545344ec Mon Sep 17 00:00:00 2001 From: diversemix Date: Tue, 29 Oct 2024 17:22:41 +0000 Subject: [PATCH 5/6] Also mark test as a search test --- tests/use-cases/test_mcf_uc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/use-cases/test_mcf_uc.py b/tests/use-cases/test_mcf_uc.py index 01e33d96..2bb4bf52 100644 --- a/tests/use-cases/test_mcf_uc.py +++ b/tests/use-cases/test_mcf_uc.py @@ -12,6 +12,7 @@ @pytest.mark.use_case +@pytest.mark.search @patch( "app.api.api_v1.routers.search.AppTokenFactory.verify_corpora_in_db", return_value=True, From 117e94e614fb89a54456e7d93c5942cb28403e2d Mon Sep 17 00:00:00 2001 From: diversemix Date: Tue, 29 Oct 2024 17:25:05 +0000 Subject: [PATCH 6/6] update make command --- makefile-docker.defs | 2 +- tests/use-cases/test_mcf_uc.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/makefile-docker.defs b/makefile-docker.defs index 32894007..48736a86 100644 --- a/makefile-docker.defs +++ b/makefile-docker.defs @@ -131,7 +131,7 @@ test_unit: docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm backend pytest -vvv tests/unit ${ARGS} test_non_search: - docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm backend pytest -vvv -m 'not search' ${ARGS} + docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm backend pytest -vvv -m 'not search and not use_case' ${ARGS} test: docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm backend pytest -vvv ${ARGS} diff --git a/tests/use-cases/test_mcf_uc.py b/tests/use-cases/test_mcf_uc.py index 2bb4bf52..01e33d96 100644 --- a/tests/use-cases/test_mcf_uc.py +++ b/tests/use-cases/test_mcf_uc.py @@ -12,7 +12,6 @@ @pytest.mark.use_case -@pytest.mark.search @patch( "app.api.api_v1.routers.search.AppTokenFactory.verify_corpora_in_db", return_value=True,