From 6ce2761288ce5bce23cc9757f3f647b56bf77b10 Mon Sep 17 00:00:00 2001 From: Katy Baulch <46493669+katybaulch@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:25:05 +0000 Subject: [PATCH] Add trunk, fix autofixable lint & format errors & check for FIXMEs on PR (#96) --- .github/workflows/ci-cd.yml | 42 ++++-------- .gitignore | 13 +++- .pre-commit-config.yaml | 34 ---------- .trunk/configs/.hadolint.yaml | 4 ++ .trunk/configs/.isort.cfg | 2 + .trunk/configs/.markdownlint.yaml | 11 +++ .trunk/configs/.shellcheckrc | 7 ++ .trunk/configs/.yamllint.yaml | 10 +++ .trunk/configs/bandit.yaml | 1 + .trunk/configs/cspell.yaml | 4 ++ .trunk/trunk.yaml | 67 +++++++++++++++++++ DESIGN.md | 49 +++++++------- DEVELOPERS.md | 21 +++--- Dockerfile | 4 +- GETTING_STARTED.md | 2 +- Makefile | 19 ++++-- app/api/api_v1/routers/__init__.py | 8 +-- app/api/api_v1/routers/analytics.py | 4 +- app/api/api_v1/routers/auth.py | 5 +- app/api/api_v1/routers/collection.py | 1 + app/api/api_v1/routers/config.py | 3 +- app/api/api_v1/routers/document.py | 7 +- app/api/api_v1/routers/event.py | 1 + app/api/api_v1/routers/family.py | 5 +- app/clients/aws/client.py | 2 +- app/clients/aws/s3bucket.py | 3 +- app/logging_config.py | 7 +- app/main.py | 29 ++++---- app/model/analytics.py | 3 +- app/model/collection.py | 1 + app/model/config.py | 1 - app/model/document.py | 7 +- app/model/event.py | 2 +- app/model/general.py | 1 - app/repository/__init__.py | 12 ++-- app/repository/app_user.py | 2 +- app/repository/collection.py | 15 +++-- app/repository/config.py | 12 ++-- app/repository/document.py | 12 ++-- app/repository/document_file.py | 4 +- app/repository/event.py | 12 ++-- app/repository/family.py | 20 +++--- app/repository/geography.py | 2 +- app/repository/helpers.py | 8 +-- app/repository/metadata.py | 9 ++- app/repository/organisation.py | 3 +- app/repository/protocols.py | 1 + app/service/analytics.py | 8 +-- app/service/app_user.py | 1 + app/service/authentication.py | 2 +- app/service/authorisation.py | 1 + app/service/category.py | 1 + app/service/collection.py | 1 + app/service/config.py | 8 +-- app/service/family.py | 1 + app/service/geography.py | 1 + app/service/metadata.py | 4 +- app/service/organisation.py | 1 + app/service/token.py | 12 ++-- docker-compose.yml | 5 +- integration_tests/analytics/test_get.py | 5 +- integration_tests/collection/test_create.py | 5 +- integration_tests/collection/test_delete.py | 5 +- integration_tests/collection/test_get.py | 2 +- integration_tests/collection/test_search.py | 4 +- integration_tests/collection/test_update.py | 7 +- integration_tests/conftest.py | 22 +++--- integration_tests/document/test_create.py | 10 ++- integration_tests/document/test_delete.py | 11 +-- integration_tests/document/test_get.py | 2 +- integration_tests/document/test_search.py | 4 +- integration_tests/event/test_create.py | 5 +- integration_tests/event/test_delete.py | 7 +- integration_tests/event/test_get.py | 2 +- integration_tests/event/test_search.py | 4 +- integration_tests/event/test_update.py | 8 +-- integration_tests/family/test_create.py | 8 ++- integration_tests/family/test_delete.py | 10 +-- integration_tests/family/test_get.py | 2 +- integration_tests/family/test_search.py | 6 +- integration_tests/family/test_update.py | 6 +- integration_tests/login/test_login.py | 2 +- .../mocks/bad_collection_repo.py | 3 +- integration_tests/mocks/bad_document_repo.py | 3 +- integration_tests/mocks/bad_event_repo.py | 3 +- integration_tests/mocks/bad_family_repo.py | 3 +- .../mocks/rollback_collection_repo.py | 1 + .../mocks/rollback_document_repo.py | 1 + .../mocks/rollback_event_repo.py | 2 +- .../mocks/rollback_family_repo.py | 1 + integration_tests/setup_db.py | 5 +- integration_tests/test_config.py | 3 +- pyproject.toml | 32 ++++++--- scripts/generate_new_user_sql.sh | 7 +- scripts/testing/get-all-documents | 4 +- scripts/testing/get-all-families | 4 +- scripts/testing/get-config | 4 +- scripts/testing/get-family | 4 +- scripts/testing/search-families | 4 +- scripts/testing/update-family | 8 +-- scripts/update-db-model.sh | 4 +- unit_tests/conftest.py | 38 +++++------ unit_tests/helpers/analytics.py | 1 + unit_tests/helpers/collection.py | 1 + unit_tests/helpers/document.py | 3 +- unit_tests/helpers/event.py | 5 +- unit_tests/helpers/family.py | 2 + unit_tests/mocks/repos/__init__.py | 1 + unit_tests/mocks/repos/app_user_repo.py | 6 +- unit_tests/mocks/repos/config_repo.py | 2 +- unit_tests/mocks/repos/geography_repo.py | 1 + unit_tests/mocks/repos/metadata_repo.py | 1 + unit_tests/mocks/repos/organisation_repo.py | 1 + .../mocks/services/analytics_service.py | 2 +- unit_tests/mocks/services/config_service.py | 2 +- unit_tests/routers/test_analytics.py | 5 +- unit_tests/routers/test_collection.py | 9 ++- unit_tests/routers/test_document.py | 4 +- unit_tests/routers/test_event.py | 9 +-- unit_tests/routers/test_family.py | 5 +- unit_tests/service/test_analytics_service.py | 9 ++- .../service/test_authorisation_service.py | 3 +- unit_tests/service/test_family_service.py | 1 + unit_tests/service/test_geography_service.py | 3 +- unit_tests/service/test_id_service.py | 1 + unit_tests/service/test_metadata_service.py | 2 +- unit_tests/service/test_token_service.py | 4 +- 127 files changed, 505 insertions(+), 387 deletions(-) delete mode 100644 .pre-commit-config.yaml create mode 100644 .trunk/configs/.hadolint.yaml create mode 100644 .trunk/configs/.isort.cfg create mode 100644 .trunk/configs/.markdownlint.yaml create mode 100644 .trunk/configs/.shellcheckrc create mode 100644 .trunk/configs/.yamllint.yaml create mode 100644 .trunk/configs/bandit.yaml create mode 100644 .trunk/configs/cspell.yaml create mode 100644 .trunk/trunk.yaml diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 9f050c63..e92508f3 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -2,47 +2,33 @@ name: CI/CD on: push: - tags: ['v*'] + tags: ["v*"] branches: - main pull_request: branches: - main +permissions: read-all + # https://github.com/marketplace/actions/docker-layer-caching jobs: code-quality: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install dependencies - run: | - python -m pip install "poetry==1.6.1" && poetry install && poetry run pre-commit install - - - name: Run pre-commit checks - run: | - poetry run pre-commit run --all-files - - - name: Check code contains no FIXME's - run: | - git grep -r --no-color ${case_sensitive} --line-number -e "FIXME" :^.github - if [[ $? -eq 0 ]]; then - # if we found any FIXME entries in checked in files, fail on main - exit 1 - else - exit 0 - fi - shell: bash {0} + permissions: + # For trunk to post annotations + checks: write + # For repo checkout + contents: read + uses: climatepolicyradar/reusable-workflows/.github/workflows/python-precommit-validator.yml@main test: runs-on: ubuntu-latest steps: + - name: Install latest Docker Compose + uses: ndeloof/install-compose-action@v0.0.1 + with: + legacy: false + - uses: actions/checkout@v4 - name: Configure test env variables diff --git a/.gitignore b/.gitignore index 7d34703b..5d328aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,15 @@ load_blank.txt load_default.txt # PIP (because we want to use Poetry) -requirements.txt \ No newline at end of file +requirements.txt + +# Trunk miscellaneous +*out +*logs +*actions +*notifications +*tools +plugins +user_trunk.yaml +user.yaml +tmp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index ebcaa710..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 - hooks: - - id: check-json - - id: detect-aws-credentials - args: [--allow-missing-credentials] - - repo: https://github.com/ambv/black - rev: 23.1.0 - hooks: - - id: black - language_version: python3 - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.0.246" - hooks: - - id: ruff - - repo: local - hooks: - - id: pyright - name: pyright (admin-backend) - entry: pyright - language: node - types: [python] - additional_dependencies: ["pyright@1.1.294"] - - repo: local - hooks: - - id: markdownlint-cli2-fix-rules-docker - name: markdownlint (admin-backend) - description: "Checks and fixes the style of Markdown files." - entry: davidanson/markdownlint-cli2-rules markdownlint-cli2-fix - language: docker_image - types: [markdown] - minimum_pre_commit_version: 0.15.0 -# TODO more checks? e.g. bandit, safety, snyk, ... diff --git a/.trunk/configs/.hadolint.yaml b/.trunk/configs/.hadolint.yaml new file mode 100644 index 00000000..98bf0cd2 --- /dev/null +++ b/.trunk/configs/.hadolint.yaml @@ -0,0 +1,4 @@ +# Following source doesn't work in most setups +ignored: + - SC1090 + - SC1091 diff --git a/.trunk/configs/.isort.cfg b/.trunk/configs/.isort.cfg new file mode 100644 index 00000000..b9fb3f3e --- /dev/null +++ b/.trunk/configs/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile=black diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml new file mode 100644 index 00000000..229faeca --- /dev/null +++ b/.trunk/configs/.markdownlint.yaml @@ -0,0 +1,11 @@ +# Autoformatter friendly markdownlint config (all formatting rules disabled) +default: true +blank_lines: false +bullet: false +html: false +indentation: false +line_length: false +spaces: false +url: false +whitespace: false +tables: false diff --git a/.trunk/configs/.shellcheckrc b/.trunk/configs/.shellcheckrc new file mode 100644 index 00000000..8c7b1ada --- /dev/null +++ b/.trunk/configs/.shellcheckrc @@ -0,0 +1,7 @@ +enable=all +source-path=SCRIPTDIR +disable=SC2154 + +# If you're having issues with shellcheck following source, disable the errors via: +# disable=SC1090 +# disable=SC1091 diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 00000000..4d444662 --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,10 @@ +rules: + quoted-strings: + required: only-when-needed + extra-allowed: ["{|}"] + empty-values: + forbid-in-block-mappings: true + forbid-in-flow-mappings: true + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/configs/bandit.yaml b/.trunk/configs/bandit.yaml new file mode 100644 index 00000000..3d2e7fbc --- /dev/null +++ b/.trunk/configs/bandit.yaml @@ -0,0 +1 @@ +exclude_dirs: ["unit_tests", "integration_tests"] diff --git a/.trunk/configs/cspell.yaml b/.trunk/configs/cspell.yaml new file mode 100644 index 00000000..7f995845 --- /dev/null +++ b/.trunk/configs/cspell.yaml @@ -0,0 +1,4 @@ +version: "0.2" +# Suggestions can sometimes take longer on CI machines, +# leading to inconsistent results. +suggestionsTimeout: 5000 # ms diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml new file mode 100644 index 00000000..4c20c10a --- /dev/null +++ b/.trunk/trunk.yaml @@ -0,0 +1,67 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml +version: 0.1 +cli: + version: 1.20.1 + +# Trunk provides extensibility via plugins. +# (https://docs.trunk.io/plugins) +plugins: + sources: + - id: trunk + ref: v1.4.4 + uri: https://github.com/trunk-io/plugins + +# Many linters and tools depend on runtimes - configure them here. +# (https://docs.trunk.io/runtimes) +runtimes: + enabled: + - go@1.21.0 + - node@18.12.1 + - python@3.10.8 + +# This is the section where you manage your linters. +# (https://docs.trunk.io/check/configuration) +lint: + definitions: + - name: bandit + direct_configs: [bandit.yaml] + commands: + - name: lint + run: bandit --exit-zero -c bandit.yaml --format json --output ${tmpfile} ${target} + + enabled: + - actionlint@1.6.27 + - bandit@1.7.8 + - black@23.1.0 + - checkov@3.2.34 + - git-diff-check + - hadolint@2.12.0 + - isort@5.13.2 + - markdownlint@0.39.0 + - osv-scanner@1.6.2 + - pre-commit-hooks@4.5.0: + commands: + - end-of-file-fixer + - check-json + - detect-aws-credentials + - prettier@3.2.5 + - pyright@1.1.294 + - ruff@0.3.2 + - shellcheck@0.10.0 + - shfmt@3.6.0 + - taplo@0.8.1 + - terrascan@1.19.1 + - trivy@0.49.1 + - trufflehog@3.69.0 + - yamllint@1.35.1 + +actions: + disabled: + - trunk-announce + - trunk-check-pre-push + enabled: + - trunk-check-pre-commit + - trunk-fmt-pre-commit + - trunk-upgrade-available diff --git a/DESIGN.md b/DESIGN.md index a462b2bc..09e19270 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -7,46 +7,47 @@ The approach taken is influenced by: - [The Twelve Factor App](https://12factor.net/) - [SOLID principles](https://www.baeldung.com/solid-principles) - [Domain Driven Design (DDD)](https://martinfowler.com/tags/domain%20driven%20design.html) -particularly [folder structure](https://dev.to/stevescruz/domain-driven-design-ddd-file-structure-4pja) + particularly [folder structure](https://dev.to/stevescruz/domain-driven-design-ddd-file-structure-4pja) ## Overview There are three main layers to the application: - **Routing Layer** - The responsibility here is to manage the network payloads -and any authentication middleware. All business logic is handed off to... + and any authentication middleware. All business logic is handed off to... - **Service Layer** - Contains all validation and business logic for the -application. This in turn uses the... + application. This in turn uses the... - **Repository Layer** - With the sole responsibility of managing how data is -stored and retrieved to/from the database. -**Note**, this split into responsibilities for separate entities, should the -need arise to create a transaction (for example creating two separate entities -atomically) - then this is the responsibility of the service layer to manage the -transaction. - - Router - │ - │ - ▼ - Service - │ - │ - ┌───────┴───────┐ - │ │ - ▼ ▼ -Repository ──────► Client - (ext) + stored and retrieved to/from the database. + **Note**, this split into responsibilities for separate entities, should the + need arise to create a transaction (for example creating two separate entities + atomically) - then this is the responsibility of the service layer to manage the + transaction. + + Router + │ + │ + ▼ + Service + │ + │ + ┌───────┴───────┐ + │ │ + ▼ ▼ + + Repository ──────► Client + (ext) ## Testing Strategy ### Unit tests - Routing Layer - this is tested my mocking out the required services by each -individual route. The tests should alter how the service behaves to test out the -routing layer responds. + individual route. The tests should alter how the service behaves to test out the + routing layer responds. - Service Layer - the required repositories are mocked out so that the tests can -check the service returns/raises what is expected. + check the service returns/raises what is expected. - Repository Layer - there are no unit tests. diff --git a/DEVELOPERS.md b/DEVELOPERS.md index 4be6b86c..d6f4d9a8 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -11,19 +11,20 @@ At the moment we are using PyRight pegged at version 1.1.294. ## VS Code - Although we use PyRight for linting (see [.pre-commit-config.yaml](./.pre-commit-config.yaml)) -[PyRight recommends that you use PyLance when using VS Code](https://microsoft.github.io/pyright/#/installation?id=vs-code). + [PyRight recommends that you use PyLance when using VS Code](https://microsoft.github.io/pyright/#/installation?id=vs-code). ### Extensions -| Extension | Author | Description | Recommended/Optional | -|-----------|--------|-------------|-------------------------------| -| [Black Formatter](https://github.com/) | Microsoft | | Optional | -| [Code Spell Checker](https://github.com/) | Street Side Software | | Recommended | -| [Makefile Tools](https://github.com/) | Microsoft | | Recommended | -| [markdownlint](https://github.com/) | David Anson | | Recommended | -| [Markdown All in One](https://github.com/) | Yu Zhang | | Optional | -| [PyLance](https://github.com/) | Microsoft | | Recommended | -| [YAML](https://github.com/) | Red Hat | | Optional | +| Extension | Author | Recommended/Optional | +| ------------------------------------------ | -------------------- | -------------------- | +| [Black Formatter](https://github.com/) | Microsoft | Optional | +| [Code Spell Checker](https://github.com/) | Street Side Software | Recommended | +| [Makefile Tools](https://github.com/) | Microsoft | Recommended | +| [markdownlint](https://github.com/) | David Anson | Recommended | +| [Markdown All in One](https://github.com/) | Yu Zhang | Optional | +| [PyLance](https://github.com/) | Microsoft | Recommended | +| [YAML](https://github.com/) | Red Hat | Optional | +| [Ruff](https://github.com/) | Charlie R Marsh | Recommended | ### Recommended User Settings JSON diff --git a/Dockerfile b/Dockerfile index 02cbfdf9..e575acf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ WORKDIR /usr/src ENV PYTHONPATH=/usr/src # Requirements -RUN pip install poetry +RUN pip install --no-cache-dir poetry==1.6.1 COPY poetry.lock pyproject.toml ./ -RUN poetry config virtualenvs.create false && poetry install --no-cache +RUN poetry config virtualenvs.create false && poetry install --no-directory --no-root # Now code COPY ./app ./app diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index a3dd0d8f..f3829088 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -87,7 +87,7 @@ Currently the deployment is manual, this required the following steps: - Create a new tagged release [here](https://github.com/climatepolicyradar/navigator-admin-backend/releases) - Wait for the `semver` Action to run in github - this creates and pushes the -image into ECR + image into ECR - Log into the AWS console in the environment you wish to deploy. - In AppRunner - find the running images and hit the `Deploy` button. diff --git a/Makefile b/Makefile index 5ffa0cf5..28257ce5 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,21 @@ bootstrap: pip3 install poetry poetry install -git_hooks: - # Install & run git pre-commit hooks - poetry run pre-commit install --install-hooks - pre-commit run --all-files +install_trunk: + $(eval trunk_installed=$(shell trunk --version > /dev/null 2>&1 ; echo $$? )) +ifneq ($(trunk_installed),0) + $(eval OS_NAME=$(shell uname -s | tr A-Z a-z)) +ifeq ($(OS_NAME),linux) + curl https://get.trunk.io -fsSL | bash +endif +ifeq ($(OS_NAME),darwin) + brew install trunk-io +endif +endif + +git_hooks: install_trunk + trunk fmt + trunk check build: docker build -t navigator-admin-backend . diff --git a/app/api/api_v1/routers/__init__.py b/app/api/api_v1/routers/__init__.py index 977baea0..de5e661e 100644 --- a/app/api/api_v1/routers/__init__.py +++ b/app/api/api_v1/routers/__init__.py @@ -1,7 +1,7 @@ -from .family import families_router -from .collection import collections_router +from .analytics import analytics_router from .auth import auth_router -from .document import document_router +from .collection import collections_router from .config import config_router -from .analytics import analytics_router +from .document import document_router from .event import event_router +from .family import families_router diff --git a/app/api/api_v1/routers/analytics.py b/app/api/api_v1/routers/analytics.py index c3f512f5..fa24a7a8 100644 --- a/app/api/api_v1/routers/analytics.py +++ b/app/api/api_v1/routers/analytics.py @@ -1,9 +1,11 @@ """Endpoints for managing the Analytics service.""" + import logging + from fastapi import APIRouter, HTTPException, status -from app.errors import RepositoryError import app.service.analytics as analytics_service +from app.errors import RepositoryError from app.model.analytics import SummaryDTO analytics_router = r = APIRouter() diff --git a/app/api/api_v1/routers/auth.py b/app/api/api_v1/routers/auth.py index 70867f2b..45f82343 100644 --- a/app/api/api_v1/routers/auth.py +++ b/app/api/api_v1/routers/auth.py @@ -2,6 +2,9 @@ from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm + +import app.service.authorisation as auth_service +import app.service.token as token_service from app.errors import ( AuthenticationError, AuthorisationError, @@ -9,8 +12,6 @@ TokenError, ) from app.service.authentication import authenticate_user -import app.service.authorisation as auth_service -import app.service.token as token_service auth_router = r = APIRouter() diff --git a/app/api/api_v1/routers/collection.py b/app/api/api_v1/routers/collection.py index 9f8bbea1..6fddcadc 100644 --- a/app/api/api_v1/routers/collection.py +++ b/app/api/api_v1/routers/collection.py @@ -1,4 +1,5 @@ """Endpoints for managing the Collection entity.""" + import logging from fastapi import APIRouter, HTTPException, Request, status diff --git a/app/api/api_v1/routers/config.py b/app/api/api_v1/routers/config.py index da588354..6957f2f5 100644 --- a/app/api/api_v1/routers/config.py +++ b/app/api/api_v1/routers/config.py @@ -1,9 +1,10 @@ import logging from fastapi import APIRouter, HTTPException, Request, status -from app.model.config import ConfigReadDTO + import app.service.config as config_service from app.errors import RepositoryError +from app.model.config import ConfigReadDTO config_router = r = APIRouter() diff --git a/app/api/api_v1/routers/document.py b/app/api/api_v1/routers/document.py index 25c06a9d..c699fe78 100644 --- a/app/api/api_v1/routers/document.py +++ b/app/api/api_v1/routers/document.py @@ -1,4 +1,5 @@ """Endpoints for managing the Document entity.""" + import logging from fastapi import APIRouter, HTTPException, Request, status @@ -10,11 +11,7 @@ validate_query_params, ) from app.errors import RepositoryError, ValidationError -from app.model.document import ( - DocumentCreateDTO, - DocumentReadDTO, - DocumentWriteDTO, -) +from app.model.document import DocumentCreateDTO, DocumentReadDTO, DocumentWriteDTO document_router = r = APIRouter() diff --git a/app/api/api_v1/routers/event.py b/app/api/api_v1/routers/event.py index d1aefebc..2abeb5e5 100644 --- a/app/api/api_v1/routers/event.py +++ b/app/api/api_v1/routers/event.py @@ -1,4 +1,5 @@ """Endpoints for managing Family Event entities.""" + import logging from fastapi import APIRouter, HTTPException, Request, status diff --git a/app/api/api_v1/routers/family.py b/app/api/api_v1/routers/family.py index ff20a045..9ef4a59f 100644 --- a/app/api/api_v1/routers/family.py +++ b/app/api/api_v1/routers/family.py @@ -1,11 +1,12 @@ """ Endpoints for managing the Family entity. -It was considered to create a "service" layer that would use the repo -directly. However, this API has little / no logic in so the service +It was considered to create a "service" layer that would use the repo +directly. However, this API has little / no logic in so the service layer would just pass through directly to the repo. So the approach implemented directly accesses the "repository" layer. """ + import logging from fastapi import APIRouter, HTTPException, Request, status diff --git a/app/clients/aws/client.py b/app/clients/aws/client.py index 72b6f888..afe918aa 100644 --- a/app/clients/aws/client.py +++ b/app/clients/aws/client.py @@ -1,11 +1,11 @@ import os from typing import Any + import boto3 import botocore.client from app.model.aws_config import AWSConfig - _AWS_REGION = os.getenv("AWS_REGION", "eu-west-2") _AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") _AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "") diff --git a/app/clients/aws/s3bucket.py b/app/clients/aws/s3bucket.py index 4523c5cd..00ae6352 100644 --- a/app/clients/aws/s3bucket.py +++ b/app/clients/aws/s3bucket.py @@ -1,8 +1,9 @@ import re from urllib.parse import quote_plus, urlsplit -from app.clients.aws.client import AWSClient + from botocore.exceptions import ClientError +from app.clients.aws.client import AWSClient from app.errors import RepositoryError diff --git a/app/logging_config.py b/app/logging_config.py index d5ec8ebc..14fa4fe8 100644 --- a/app/logging_config.py +++ b/app/logging_config.py @@ -1,8 +1,9 @@ -import os -from fastapi import FastAPI -import json_logging import logging import logging.config +import os + +import json_logging +from fastapi import FastAPI LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() DEFAULT_LOGGING = { diff --git a/app/main.py b/app/main.py index 0f6c05f7..958ce6f0 100644 --- a/app/main.py +++ b/app/main.py @@ -4,28 +4,29 @@ Note: If you want to add a new endpoint, please make sure you update AuthEndpoint and the AUTH_TABLE in app/clients/db/models/app/authorisation.py. """ -from fastapi_pagination import add_pagination + from contextlib import asynccontextmanager -from app.api.api_v1.routers.auth import check_user_auth -from app.logging_config import DEFAULT_LOGGING, setup_json_logging + +import uvicorn +from db_client import run_migrations +from fastapi import Depends, FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi_health import health +from fastapi_pagination import add_pagination + from app.api.api_v1.routers import ( - families_router, + analytics_router, auth_router, collections_router, - document_router, config_router, - analytics_router, + document_router, event_router, + families_router, ) -from fastapi import FastAPI, Depends -from fastapi_health import health -from fastapi.middleware.cors import CORSMiddleware -import uvicorn - -from app.service.health import is_database_online +from app.api.api_v1.routers.auth import check_user_auth from app.clients.db.session import engine -from db_client import run_migrations - +from app.logging_config import DEFAULT_LOGGING, setup_json_logging +from app.service.health import is_database_online _ALLOW_ORIGIN_REGEX = ( r"http://localhost:3000|" diff --git a/app/model/analytics.py b/app/model/analytics.py index c6a5e8a4..5127531b 100644 --- a/app/model/analytics.py +++ b/app/model/analytics.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel from typing import Optional +from pydantic import BaseModel + class SummaryDTO(BaseModel): """Representation of an Analytics Summary.""" diff --git a/app/model/collection.py b/app/model/collection.py index 10bce2de..14b6c31b 100644 --- a/app/model/collection.py +++ b/app/model/collection.py @@ -1,4 +1,5 @@ from datetime import datetime + from pydantic import BaseModel diff --git a/app/model/config.py b/app/model/config.py index 68543043..ab920fdf 100644 --- a/app/model/config.py +++ b/app/model/config.py @@ -2,7 +2,6 @@ from pydantic import BaseModel - TaxonomyData = Mapping[str, Mapping[str, Union[bool, str, Sequence[str]]]] diff --git a/app/model/document.py b/app/model/document.py index b521a48e..808c5cbe 100644 --- a/app/model/document.py +++ b/app/model/document.py @@ -1,11 +1,8 @@ from datetime import datetime from typing import Optional -from pydantic import BaseModel, AnyHttpUrl - -from db_client.models.law_policy.family import ( - DocumentStatus, -) +from db_client.models.law_policy.family import DocumentStatus +from pydantic import AnyHttpUrl, BaseModel class DocumentReadDTO(BaseModel): diff --git a/app/model/event.py b/app/model/event.py index bc6bbe58..1408245d 100644 --- a/app/model/event.py +++ b/app/model/event.py @@ -1,8 +1,8 @@ from datetime import datetime from typing import Optional -from pydantic import BaseModel from db_client.models.law_policy.family import EventStatus +from pydantic import BaseModel class EventReadDTO(BaseModel): diff --git a/app/model/general.py b/app/model/general.py index d78aa9dd..3282f4f5 100644 --- a/app/model/general.py +++ b/app/model/general.py @@ -1,4 +1,3 @@ from typing import Any - Json = dict[str, Any] diff --git a/app/repository/__init__.py b/app/repository/__init__.py index 77757fff..3d3071ed 100644 --- a/app/repository/__init__.py +++ b/app/repository/__init__.py @@ -1,13 +1,13 @@ +import app.clients.aws.s3bucket as s3bucket_repo +import app.repository.app_user as app_user_repo +import app.repository.collection as collection_repo +import app.repository.config as config_repo +import app.repository.document as document_repo +import app.repository.event as event_repo import app.repository.family as family_repo import app.repository.geography as geography_repo import app.repository.metadata as metadata_repo import app.repository.organisation as organisation_repo -import app.repository.collection as collection_repo -import app.repository.document as document_repo -import app.repository.app_user as app_user_repo -import app.clients.aws.s3bucket as s3bucket_repo -import app.repository.config as config_repo -import app.repository.event as event_repo from app.repository.protocols import FamilyRepo family_repo: FamilyRepo diff --git a/app/repository/app_user.py b/app/repository/app_user.py index cffb32d4..8a5485ad 100644 --- a/app/repository/app_user.py +++ b/app/repository/app_user.py @@ -1,7 +1,7 @@ from typing import Optional, Tuple, cast -from sqlalchemy.orm import Session from db_client.models.app.users import AppUser, Organisation, OrganisationUser +from sqlalchemy.orm import Session MaybeAppUser = Optional[AppUser] diff --git a/app/repository/collection.py b/app/repository/collection.py index ec9f3f91..52e5021e 100644 --- a/app/repository/collection.py +++ b/app/repository/collection.py @@ -4,13 +4,6 @@ from datetime import datetime from typing import Optional, Tuple, Union, cast -from sqlalchemy import Column, and_, desc, or_ -from sqlalchemy import delete as db_delete -from sqlalchemy import update as db_update -from sqlalchemy.exc import NoResultFound, OperationalError -from sqlalchemy.orm import Query, Session -from sqlalchemy_utils import escape_like - from db_client.models.app.counters import CountedEntity from db_client.models.app.users import Organisation from db_client.models.law_policy import Collection @@ -19,6 +12,14 @@ CollectionOrganisation, ) from db_client.models.law_policy.family import Family +from sqlalchemy import Column, and_ +from sqlalchemy import delete as db_delete +from sqlalchemy import desc, or_ +from sqlalchemy import update as db_update +from sqlalchemy.exc import NoResultFound, OperationalError +from sqlalchemy.orm import Query, Session +from sqlalchemy_utils import escape_like + from app.errors import RepositoryError from app.model.collection import ( CollectionCreateDTO, diff --git a/app/repository/config.py b/app/repository/config.py index 7401289a..822f4632 100644 --- a/app/repository/config.py +++ b/app/repository/config.py @@ -1,7 +1,8 @@ import logging from typing import Any, Optional -from sqlalchemy.orm import Session + from db_client.models.app.users import Organisation +from db_client.models.base import AnyModel from db_client.models.document.physical_document import Language from db_client.models.law_policy.family import ( FamilyDocumentRole, @@ -10,13 +11,10 @@ Variant, ) from db_client.models.law_policy.geography import Geography -from db_client.models.law_policy.metadata import ( - MetadataOrganisation, - MetadataTaxonomy, -) -from db_client.models.base import AnyModel -from app.model.config import ConfigReadDTO, DocumentConfig, EventConfig, TaxonomyData +from db_client.models.law_policy.metadata import MetadataOrganisation, MetadataTaxonomy +from sqlalchemy.orm import Session +from app.model.config import ConfigReadDTO, DocumentConfig, EventConfig, TaxonomyData _LOGGER = logging.getLogger(__name__) diff --git a/app/repository/document.py b/app/repository/document.py index 9fe08900..39dfb949 100644 --- a/app/repository/document.py +++ b/app/repository/document.py @@ -8,15 +8,11 @@ PhysicalDocument, PhysicalDocumentLanguage, ) -from db_client.models.law_policy import ( - FamilyDocument, -) -from db_client.models.law_policy.family import ( - DocumentStatus, - Slug, -) -from sqlalchemy import Column, and_, func +from db_client.models.law_policy import FamilyDocument +from db_client.models.law_policy.family import DocumentStatus, Slug +from sqlalchemy import Column, and_ from sqlalchemy import delete as db_delete +from sqlalchemy import func from sqlalchemy import insert as db_insert from sqlalchemy import update as db_update from sqlalchemy.exc import NoResultFound, OperationalError diff --git a/app/repository/document_file.py b/app/repository/document_file.py index 18822103..8535f97f 100644 --- a/app/repository/document_file.py +++ b/app/repository/document_file.py @@ -1,8 +1,8 @@ import os from typing import Tuple -from app.clients.aws.client import AWSClient -import app.clients.aws.s3bucket as s3_bucket +import app.clients.aws.s3bucket as s3_bucket +from app.clients.aws.client import AWSClient _CDN_URL: str = os.getenv("CDN_URL", "https://cdn.climatepolicyradar.org") _BUCKET_NAME = os.getenv("S3_DOCUMENT_BUCKET", "test-document-bucket") diff --git a/app/repository/event.py b/app/repository/event.py index b6246443..2790f0e8 100644 --- a/app/repository/event.py +++ b/app/repository/event.py @@ -4,20 +4,16 @@ from datetime import datetime from typing import Optional, Tuple, Union, cast -from sqlalchemy import Column, and_, or_ +from db_client.models.app.counters import CountedEntity +from db_client.models.law_policy import EventStatus, Family, FamilyDocument, FamilyEvent +from sqlalchemy import Column, and_ from sqlalchemy import delete as db_delete +from sqlalchemy import or_ from sqlalchemy import update as db_update from sqlalchemy.exc import NoResultFound, OperationalError from sqlalchemy.orm import Query, Session from sqlalchemy_utils import escape_like -from db_client.models.app.counters import CountedEntity -from db_client.models.law_policy import ( - EventStatus, - Family, - FamilyDocument, - FamilyEvent, -) from app.errors import RepositoryError, ValidationError from app.model.event import EventCreateDTO, EventReadDTO, EventWriteDTO from app.repository import family as family_repo diff --git a/app/repository/family.py b/app/repository/family.py index 7ed9727d..a7d2f240 100644 --- a/app/repository/family.py +++ b/app/repository/family.py @@ -4,13 +4,6 @@ from datetime import datetime from typing import Optional, Tuple, Union, cast -from sqlalchemy import Column, and_, desc, or_ -from sqlalchemy import delete as db_delete -from sqlalchemy import update as db_update -from sqlalchemy.exc import NoResultFound, OperationalError -from sqlalchemy.orm import Query, Session -from sqlalchemy_utils import escape_like - from db_client.models.app.counters import CountedEntity from db_client.models.app.users import Organisation from db_client.models.law_policy.collection import CollectionFamily @@ -23,10 +16,15 @@ Slug, ) from db_client.models.law_policy.geography import Geography -from db_client.models.law_policy.metadata import ( - FamilyMetadata, - MetadataOrganisation, -) +from db_client.models.law_policy.metadata import FamilyMetadata, MetadataOrganisation +from sqlalchemy import Column, and_ +from sqlalchemy import delete as db_delete +from sqlalchemy import desc, or_ +from sqlalchemy import update as db_update +from sqlalchemy.exc import NoResultFound, OperationalError +from sqlalchemy.orm import Query, Session +from sqlalchemy_utils import escape_like + from app.errors import RepositoryError from app.model.family import FamilyCreateDTO, FamilyReadDTO, FamilyWriteDTO from app.repository.helpers import generate_import_id, generate_slug diff --git a/app/repository/geography.py b/app/repository/geography.py index be57540c..73473118 100644 --- a/app/repository/geography.py +++ b/app/repository/geography.py @@ -1,7 +1,7 @@ from typing import Optional -from sqlalchemy.orm import Session from db_client.models.law_policy.geography import Geography +from sqlalchemy.orm import Session def get_id_from_value(db: Session, geo_string: str) -> Optional[int]: diff --git a/app/repository/helpers.py b/app/repository/helpers.py index f49a558d..f3311bbb 100644 --- a/app/repository/helpers.py +++ b/app/repository/helpers.py @@ -2,12 +2,12 @@ from typing import Union, cast from uuid import uuid4 -from slugify import slugify -from sqlalchemy.orm import Session + from db_client.models.app.counters import CountedEntity, EntityCounter from db_client.models.app.users import Organisation - from db_client.models.law_policy.family import Slug +from slugify import slugify +from sqlalchemy.orm import Session def generate_slug( @@ -58,7 +58,7 @@ def generate_import_id( :return str: the generated import_id """ - if type(org) == str: + if isinstance(org, str): org_name = org else: org_name = ( diff --git a/app/repository/metadata.py b/app/repository/metadata.py index b2d7eb0c..d429e5ff 100644 --- a/app/repository/metadata.py +++ b/app/repository/metadata.py @@ -1,11 +1,10 @@ from typing import Optional -from db_client.models.law_policy.metadata import ( - MetadataOrganisation, - MetadataTaxonomy, -) -from app.model.general import Json + +from db_client.models.law_policy.metadata import MetadataOrganisation, MetadataTaxonomy from sqlalchemy.orm import Session +from app.model.general import Json + def get_schema_for_org(db: Session, org_id: int) -> Optional[Json]: metadata = ( diff --git a/app/repository/organisation.py b/app/repository/organisation.py index e531e7f7..5f7f0731 100644 --- a/app/repository/organisation.py +++ b/app/repository/organisation.py @@ -1,6 +1,7 @@ from typing import Optional -from sqlalchemy.orm import Session + from db_client.models.app.users import Organisation +from sqlalchemy.orm import Session def get_id_from_name(db: Session, org_name: str) -> Optional[int]: diff --git a/app/repository/protocols.py b/app/repository/protocols.py index 92f424d0..ee8195c1 100644 --- a/app/repository/protocols.py +++ b/app/repository/protocols.py @@ -1,4 +1,5 @@ from typing import Optional, Protocol, Union + from sqlalchemy.orm import Session from app.model.family import FamilyCreateDTO, FamilyReadDTO, FamilyWriteDTO diff --git a/app/service/analytics.py b/app/service/analytics.py index ef4377a9..7cc1c586 100644 --- a/app/service/analytics.py +++ b/app/service/analytics.py @@ -4,18 +4,18 @@ This layer uses the document, family, and collection repos to handle querying the count of available entities. """ + import logging from pydantic import ConfigDict, validate_call from sqlalchemy import exc -from app.errors import RepositoryError -from app.model.analytics import SummaryDTO import app.service.collection as collection_service import app.service.document as document_service -import app.service.family as family_service import app.service.event as event_service - +import app.service.family as family_service +from app.errors import RepositoryError +from app.model.analytics import SummaryDTO _LOGGER = logging.getLogger(__name__) diff --git a/app/service/app_user.py b/app/service/app_user.py index f3118444..c126ca49 100644 --- a/app/service/app_user.py +++ b/app/service/app_user.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session + from app.errors import ValidationError from app.repository import app_user_repo diff --git a/app/service/authentication.py b/app/service/authentication.py index f7fae02d..12718d6a 100644 --- a/app/service/authentication.py +++ b/app/service/authentication.py @@ -5,9 +5,9 @@ from sqlalchemy.exc import NoResultFound import app.clients.db.session as db_session +import app.service.token as token_service from app.errors import AuthenticationError, RepositoryError from app.repository import app_user_repo -import app.service.token as token_service pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") diff --git a/app/service/authorisation.py b/app/service/authorisation.py index abc45976..7bc0a5ce 100644 --- a/app/service/authorisation.py +++ b/app/service/authorisation.py @@ -5,6 +5,7 @@ AuthEndpoint, AuthOperation, ) + from app.errors import AuthorisationError from app.model.jwt_user import JWTUser diff --git a/app/service/category.py b/app/service/category.py index 627b40d9..08c793c1 100644 --- a/app/service/category.py +++ b/app/service/category.py @@ -1,4 +1,5 @@ from db_client.models.law_policy.family import FamilyCategory + from app.errors import ValidationError diff --git a/app/service/collection.py b/app/service/collection.py index 590d6dd3..70a59b0a 100644 --- a/app/service/collection.py +++ b/app/service/collection.py @@ -4,6 +4,7 @@ This layer uses the collection repo to handle storage management and other services for validation etc. """ + import logging from typing import Optional, Union diff --git a/app/service/config.py b/app/service/config.py index 9f84216a..ce6fb24a 100644 --- a/app/service/config.py +++ b/app/service/config.py @@ -1,11 +1,11 @@ import logging -from app.errors import RepositoryError -from app.model.config import ConfigReadDTO -import app.repository.config as config_repo from sqlalchemy import exc -import app.clients.db.session as db_session +import app.clients.db.session as db_session +import app.repository.config as config_repo +from app.errors import RepositoryError +from app.model.config import ConfigReadDTO _LOGGER = logging.getLogger(__name__) diff --git a/app/service/family.py b/app/service/family.py index e8c70a55..d9620c8f 100644 --- a/app/service/family.py +++ b/app/service/family.py @@ -3,6 +3,7 @@ This file hands off to the family repo, adding the dependency of the db (future) """ + import logging from typing import Optional, Union diff --git a/app/service/geography.py b/app/service/geography.py index aad66a0f..934c9a34 100644 --- a/app/service/geography.py +++ b/app/service/geography.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session + from app.errors import ValidationError from app.repository import geography_repo diff --git a/app/service/metadata.py b/app/service/metadata.py index afa9bbed..331432f2 100644 --- a/app/service/metadata.py +++ b/app/service/metadata.py @@ -1,7 +1,9 @@ import logging + +from sqlalchemy.orm import Session + from app.errors import ValidationError from app.model.general import Json -from sqlalchemy.orm import Session from app.repository import metadata_repo _LOGGER = logging.getLogger(__name__) diff --git a/app/service/organisation.py b/app/service/organisation.py index 0518c789..e579382c 100644 --- a/app/service/organisation.py +++ b/app/service/organisation.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session + from app.errors import ValidationError from app.repository import organisation_repo diff --git a/app/service/token.py b/app/service/token.py index 2d546112..7b5c7d7a 100644 --- a/app/service/token.py +++ b/app/service/token.py @@ -1,12 +1,12 @@ -import jwt -import os import logging +import os from datetime import datetime, timedelta from typing import Any, Optional -from app.errors import TokenError -from app.model.jwt_user import JWTUser +import jwt +from app.errors import TokenError +from app.model.jwt_user import JWTUser SECRET_KEY = os.environ["SECRET_KEY"] ALGORITHM = "HS256" @@ -31,12 +31,12 @@ def encode( if "@" not in email or "." not in email: raise TokenError(f"Parameter email should be an email, not {email}") - if type(authorisation) != dict: + if not isinstance(authorisation, dict): raise TokenError( f"Parameter authorisation should be a dict, not {authorisation}" ) - if type(is_superuser) != bool: + if not isinstance(is_superuser, bool): raise TokenError(f"Parameter is_superuser should be a bool, not {is_superuser}") to_encode = { diff --git a/docker-compose.yml b/docker-compose.yml index dcaa8854..7c494ee8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: "3.7" services: admin_backend_db: image: postgres:14 @@ -10,7 +10,7 @@ services: volumes: - admin-data:/var/lib/postgresql/data:cached healthcheck: - test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER}" ] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] interval: 5s timeout: 3s retries: 30 @@ -33,6 +33,5 @@ services: admin_backend_db: condition: service_healthy - volumes: admin-data: diff --git a/integration_tests/analytics/test_get.py b/integration_tests/analytics/test_get.py index 45626134..251dbac9 100644 --- a/integration_tests/analytics/test_get.py +++ b/integration_tests/analytics/test_get.py @@ -1,5 +1,5 @@ -from fastapi.testclient import TestClient from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session from integration_tests.setup_db import ( @@ -8,7 +8,6 @@ setup_db, ) - # --- GET ALL @@ -41,7 +40,7 @@ def test_get_analytics_summary(client: TestClient, test_db: Session, user_header assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is dict + assert isinstance(data, dict) assert list(data.keys()) == EXPECTED_ANALYTICS_SUMMARY_KEYS assert dict(sorted(data.items())) == dict( diff --git a/integration_tests/collection/test_create.py b/integration_tests/collection/test_create.py index bc999e1c..8e4d5a14 100644 --- a/integration_tests/collection/test_create.py +++ b/integration_tests/collection/test_create.py @@ -1,7 +1,8 @@ -from fastapi.testclient import TestClient +from db_client.models.law_policy.collection import Collection from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from db_client.models.law_policy.collection import Collection + from integration_tests.setup_db import setup_db from unit_tests.helpers.collection import create_collection_create_dto diff --git a/integration_tests/collection/test_delete.py b/integration_tests/collection/test_delete.py index d08edbe8..86b5c347 100644 --- a/integration_tests/collection/test_delete.py +++ b/integration_tests/collection/test_delete.py @@ -1,7 +1,8 @@ -from fastapi.testclient import TestClient +from db_client.models.law_policy.collection import Collection from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from db_client.models.law_policy.collection import Collection + from integration_tests.setup_db import EXPECTED_NUM_COLLECTIONS, setup_db diff --git a/integration_tests/collection/test_get.py b/integration_tests/collection/test_get.py index 4b9a15d5..53170664 100644 --- a/integration_tests/collection/test_get.py +++ b/integration_tests/collection/test_get.py @@ -15,7 +15,7 @@ def test_get_all_collections(client: TestClient, test_db: Session, user_header_t ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) == 3 ids_found = set([f["import_id"] for f in data]) expected_ids = set(["C.0.0.1", "C.0.0.2", "C.0.0.3"]) diff --git a/integration_tests/collection/test_search.py b/integration_tests/collection/test_search.py index f14003d8..6e3a068d 100644 --- a/integration_tests/collection/test_search.py +++ b/integration_tests/collection/test_search.py @@ -15,7 +15,7 @@ def test_search_collection(client: TestClient, test_db: Session, user_header_tok ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 3 @@ -72,7 +72,7 @@ def test_search_collections_with_max_results( ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 1 diff --git a/integration_tests/collection/test_update.py b/integration_tests/collection/test_update.py index e8ff214d..0f163418 100644 --- a/integration_tests/collection/test_update.py +++ b/integration_tests/collection/test_update.py @@ -1,11 +1,12 @@ -from fastapi.testclient import TestClient -from fastapi import status -from sqlalchemy.orm import Session from db_client.models.law_policy.collection import ( Collection, CollectionFamily, CollectionOrganisation, ) +from fastapi import status +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + from integration_tests.setup_db import EXPECTED_COLLECTIONS, setup_db from unit_tests.helpers.collection import create_collection_write_dto diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 8a996a22..f459c4a7 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -2,21 +2,17 @@ from typing import Dict import pytest +from db_client.models.base import Base from fastapi.testclient import TestClient -from app.config import SQLALCHEMY_DATABASE_URI -from sqlalchemy_utils import create_database, database_exists, drop_database from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from sqlalchemy_utils import create_database, database_exists, drop_database import app.clients.db.session as db_session -from db_client.models.base import Base import app.service.token as token_service +from app.config import SQLALCHEMY_DATABASE_URI from app.main import app -from app.repository import family_repo, collection_repo, document_repo, event_repo -from integration_tests.mocks.bad_family_repo import ( - mock_bad_family_repo, - mock_family_count_none, -) +from app.repository import collection_repo, document_repo, event_repo, family_repo from integration_tests.mocks.bad_collection_repo import ( mock_bad_collection_repo, mock_collection_count_none, @@ -29,14 +25,16 @@ mock_bad_event_repo, mock_event_count_none, ) +from integration_tests.mocks.bad_family_repo import ( + mock_bad_family_repo, + mock_family_count_none, +) from integration_tests.mocks.rollback_collection_repo import ( mock_rollback_collection_repo, ) -from integration_tests.mocks.rollback_document_repo import ( - mock_rollback_document_repo, -) -from integration_tests.mocks.rollback_family_repo import mock_rollback_family_repo +from integration_tests.mocks.rollback_document_repo import mock_rollback_document_repo from integration_tests.mocks.rollback_event_repo import mock_rollback_event_repo +from integration_tests.mocks.rollback_family_repo import mock_rollback_family_repo def get_test_db_url() -> str: diff --git a/integration_tests/document/test_create.py b/integration_tests/document/test_create.py index 8ade1723..b8d1667f 100644 --- a/integration_tests/document/test_create.py +++ b/integration_tests/document/test_create.py @@ -1,13 +1,11 @@ +from db_client.models.document import PhysicalDocument +from db_client.models.document.physical_document import PhysicalDocumentLanguage +from db_client.models.law_policy import FamilyDocument +from db_client.models.law_policy.family import DocumentStatus, Slug from fastapi import status from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from db_client.models.document import PhysicalDocument -from db_client.models.document.physical_document import ( - PhysicalDocumentLanguage, -) -from db_client.models.law_policy import FamilyDocument -from db_client.models.law_policy.family import DocumentStatus, Slug from integration_tests.setup_db import setup_db from unit_tests.helpers.document import create_document_create_dto diff --git a/integration_tests/document/test_delete.py b/integration_tests/document/test_delete.py index 332fa5e9..5fd9c98b 100644 --- a/integration_tests/document/test_delete.py +++ b/integration_tests/document/test_delete.py @@ -1,11 +1,12 @@ -from fastapi.testclient import TestClient -from fastapi import status -from sqlalchemy.orm import Session -from db_client.models.law_policy import FamilyDocument from db_client.models.document import PhysicalDocument +from db_client.models.law_policy import FamilyDocument from db_client.models.law_policy.family import DocumentStatus -from integration_tests.setup_db import setup_db +from fastapi import status +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + import app.repository.document as document_repo +from integration_tests.setup_db import setup_db def test_delete_document(client: TestClient, test_db: Session, admin_user_header_token): diff --git a/integration_tests/document/test_get.py b/integration_tests/document/test_get.py index f989845b..88bef414 100644 --- a/integration_tests/document/test_get.py +++ b/integration_tests/document/test_get.py @@ -15,7 +15,7 @@ def test_get_all_documents(client: TestClient, test_db: Session, user_header_tok ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) == 2 ids_found = set([f["import_id"] for f in data]) expected_ids = set(["D.0.0.1", "D.0.0.2"]) diff --git a/integration_tests/document/test_search.py b/integration_tests/document/test_search.py index a482a3f1..ec3b4897 100644 --- a/integration_tests/document/test_search.py +++ b/integration_tests/document/test_search.py @@ -15,7 +15,7 @@ def test_search_document(client: TestClient, test_db: Session, user_header_token ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 2 @@ -72,7 +72,7 @@ def test_search_document_with_max_results( ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 1 diff --git a/integration_tests/event/test_create.py b/integration_tests/event/test_create.py index ed41c529..f719f9b4 100644 --- a/integration_tests/event/test_create.py +++ b/integration_tests/event/test_create.py @@ -1,10 +1,9 @@ +from db_client.models.law_policy import Family, FamilyEvent +from fastapi import status from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient -from fastapi import status from sqlalchemy.orm import Session -from db_client.models.law_policy import FamilyEvent, Family - from integration_tests.setup_db import setup_db from unit_tests.helpers.event import create_event_create_dto diff --git a/integration_tests/event/test_delete.py b/integration_tests/event/test_delete.py index 7d2f0ce9..eab5dc5b 100644 --- a/integration_tests/event/test_delete.py +++ b/integration_tests/event/test_delete.py @@ -1,9 +1,10 @@ -from fastapi.testclient import TestClient +from db_client.models.law_policy import FamilyEvent from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from db_client.models.law_policy import FamilyEvent -from integration_tests.setup_db import setup_db + import app.repository.event as event_repo +from integration_tests.setup_db import setup_db def test_delete_event(client: TestClient, test_db: Session, admin_user_header_token): diff --git a/integration_tests/event/test_get.py b/integration_tests/event/test_get.py index e1c5af04..e3914fa7 100644 --- a/integration_tests/event/test_get.py +++ b/integration_tests/event/test_get.py @@ -15,7 +15,7 @@ def test_get_all_events(client: TestClient, test_db: Session, user_header_token) ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) == 3 ids_found = set([f["import_id"] for f in data]) expected_ids = set(["E.0.0.1", "E.0.0.2", "E.0.0.3"]) diff --git a/integration_tests/event/test_search.py b/integration_tests/event/test_search.py index 36d32a88..46b937fd 100644 --- a/integration_tests/event/test_search.py +++ b/integration_tests/event/test_search.py @@ -15,7 +15,7 @@ def test_search_event(client: TestClient, test_db: Session, user_header_token): ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 2 @@ -71,7 +71,7 @@ def test_search_document_with_max_results( ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 1 diff --git a/integration_tests/event/test_update.py b/integration_tests/event/test_update.py index c81a2fc7..5c71b291 100644 --- a/integration_tests/event/test_update.py +++ b/integration_tests/event/test_update.py @@ -1,11 +1,11 @@ from datetime import datetime, timezone + +from db_client.models.law_policy import FamilyEvent +from db_client.models.law_policy.family import EventStatus +from fastapi import status from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient -from fastapi import status from sqlalchemy.orm import Session -from db_client.models.law_policy.family import EventStatus - -from db_client.models.law_policy import FamilyEvent from integration_tests.setup_db import EXPECTED_EVENTS, setup_db from unit_tests.helpers.event import create_event_write_dto diff --git a/integration_tests/family/test_create.py b/integration_tests/family/test_create.py index 577d1ec8..0279f052 100644 --- a/integration_tests/family/test_create.py +++ b/integration_tests/family/test_create.py @@ -1,10 +1,12 @@ from typing import Optional -from fastapi.testclient import TestClient -from fastapi import status -from sqlalchemy.orm import Session + from db_client.models.law_policy.collection import CollectionFamily from db_client.models.law_policy.family import Family, Slug from db_client.models.law_policy.metadata import FamilyMetadata +from fastapi import status +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + from integration_tests.setup_db import setup_db from unit_tests.helpers.family import create_family_create_dto diff --git a/integration_tests/family/test_delete.py b/integration_tests/family/test_delete.py index 5baec51b..7e9f110d 100644 --- a/integration_tests/family/test_delete.py +++ b/integration_tests/family/test_delete.py @@ -1,13 +1,13 @@ -from fastapi.testclient import TestClient -from fastapi import status -from sqlalchemy.orm import Session - from db_client.models.law_policy import ( + DocumentStatus, Family, FamilyDocument, - DocumentStatus, FamilyStatus, ) +from fastapi import status +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + from integration_tests.setup_db import setup_db diff --git a/integration_tests/family/test_get.py b/integration_tests/family/test_get.py index 1d8a96ff..7e5e227d 100644 --- a/integration_tests/family/test_get.py +++ b/integration_tests/family/test_get.py @@ -15,7 +15,7 @@ def test_get_all_families(client: TestClient, test_db: Session, user_header_toke ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) == 3 ids_found = set([f["import_id"] for f in data]) expected_ids = set(["A.0.0.1", "A.0.0.2", "A.0.0.3"]) diff --git a/integration_tests/family/test_search.py b/integration_tests/family/test_search.py index f5eca59c..652c0ad1 100644 --- a/integration_tests/family/test_search.py +++ b/integration_tests/family/test_search.py @@ -15,7 +15,7 @@ def test_search_family_using_q(client: TestClient, test_db: Session, user_header ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 2 @@ -34,7 +34,7 @@ def test_search_family_with_specific_param( ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 1 @@ -53,7 +53,7 @@ def test_search_family_with_max_results( ) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) ids_found = set([f["import_id"] for f in data]) assert len(ids_found) == 1 diff --git a/integration_tests/family/test_update.py b/integration_tests/family/test_update.py index 05ea9b96..4c638762 100644 --- a/integration_tests/family/test_update.py +++ b/integration_tests/family/test_update.py @@ -1,12 +1,12 @@ from typing import Optional +from db_client.models.law_policy.collection import CollectionFamily +from db_client.models.law_policy.family import Family, FamilyCategory, Slug +from db_client.models.law_policy.metadata import FamilyMetadata from fastapi import status from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from db_client.models.law_policy.collection import CollectionFamily -from db_client.models.law_policy.family import Family, FamilyCategory, Slug -from db_client.models.law_policy.metadata import FamilyMetadata from integration_tests.setup_db import EXPECTED_FAMILIES, setup_db from unit_tests.helpers.family import create_family_write_dto diff --git a/integration_tests/login/test_login.py b/integration_tests/login/test_login.py index 5ef2052f..0eaccd8a 100644 --- a/integration_tests/login/test_login.py +++ b/integration_tests/login/test_login.py @@ -1,5 +1,5 @@ -from fastapi.testclient import TestClient from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session from integration_tests.setup_db import setup_db diff --git a/integration_tests/mocks/bad_collection_repo.py b/integration_tests/mocks/bad_collection_repo.py index a845b6f3..5972b4c7 100644 --- a/integration_tests/mocks/bad_collection_repo.py +++ b/integration_tests/mocks/bad_collection_repo.py @@ -1,7 +1,8 @@ from typing import Optional + from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.collection import CollectionReadDTO diff --git a/integration_tests/mocks/bad_document_repo.py b/integration_tests/mocks/bad_document_repo.py index af2f7b7b..c4519eed 100644 --- a/integration_tests/mocks/bad_document_repo.py +++ b/integration_tests/mocks/bad_document_repo.py @@ -1,7 +1,8 @@ from typing import Optional + from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.document import DocumentCreateDTO, DocumentReadDTO diff --git a/integration_tests/mocks/bad_event_repo.py b/integration_tests/mocks/bad_event_repo.py index 29c1317d..1829637d 100644 --- a/integration_tests/mocks/bad_event_repo.py +++ b/integration_tests/mocks/bad_event_repo.py @@ -1,7 +1,8 @@ from typing import Optional + from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.event import EventCreateDTO, EventReadDTO diff --git a/integration_tests/mocks/bad_family_repo.py b/integration_tests/mocks/bad_family_repo.py index 38f010df..562680e4 100644 --- a/integration_tests/mocks/bad_family_repo.py +++ b/integration_tests/mocks/bad_family_repo.py @@ -1,7 +1,8 @@ from typing import Optional + from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.family import FamilyReadDTO diff --git a/integration_tests/mocks/rollback_collection_repo.py b/integration_tests/mocks/rollback_collection_repo.py index 292afd42..843f047d 100644 --- a/integration_tests/mocks/rollback_collection_repo.py +++ b/integration_tests/mocks/rollback_collection_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch from sqlalchemy.exc import NoResultFound diff --git a/integration_tests/mocks/rollback_document_repo.py b/integration_tests/mocks/rollback_document_repo.py index f7c042f0..63fa070e 100644 --- a/integration_tests/mocks/rollback_document_repo.py +++ b/integration_tests/mocks/rollback_document_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch from sqlalchemy.exc import NoResultFound diff --git a/integration_tests/mocks/rollback_event_repo.py b/integration_tests/mocks/rollback_event_repo.py index 2a490ded..4cc826c0 100644 --- a/integration_tests/mocks/rollback_event_repo.py +++ b/integration_tests/mocks/rollback_event_repo.py @@ -1,6 +1,6 @@ from typing import Optional -from pytest import MonkeyPatch +from pytest import MonkeyPatch from sqlalchemy.exc import NoResultFound from app.model.event import EventCreateDTO, EventReadDTO, EventWriteDTO diff --git a/integration_tests/mocks/rollback_family_repo.py b/integration_tests/mocks/rollback_family_repo.py index 97095c39..ed78422e 100644 --- a/integration_tests/mocks/rollback_family_repo.py +++ b/integration_tests/mocks/rollback_family_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch from sqlalchemy.exc import NoResultFound diff --git a/integration_tests/setup_db.py b/integration_tests/setup_db.py index 928f6a81..24bbe9eb 100644 --- a/integration_tests/setup_db.py +++ b/integration_tests/setup_db.py @@ -1,8 +1,5 @@ from typing import cast -from sqlalchemy import text -from sqlalchemy.orm import Session - from db_client.models.app.users import AppUser, Organisation, OrganisationUser from db_client.models.document.physical_document import ( LanguageSource, @@ -27,6 +24,8 @@ MetadataOrganisation, MetadataTaxonomy, ) +from sqlalchemy import text +from sqlalchemy.orm import Session EXPECTED_NUM_FAMILIES = 3 EXPECTED_FAMILIES = [ diff --git a/integration_tests/test_config.py b/integration_tests/test_config.py index 8c97a3b3..e43f7d36 100644 --- a/integration_tests/test_config.py +++ b/integration_tests/test_config.py @@ -1,6 +1,7 @@ -from fastapi.testclient import TestClient from fastapi import status +from fastapi.testclient import TestClient from sqlalchemy.orm import Session + from integration_tests.setup_db import setup_db diff --git a/pyproject.toml b/pyproject.toml index af952d72..317a2501 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,11 @@ name = "admin_backend" version = "0.0.1" description = "" authors = ["CPR-dev-team "] +packages = [ + { include = "app" }, + { include = "integration_tests" }, + { include = "unit_tests" }, +] [tool.poetry.dependencies] python = "^3.9" @@ -28,7 +33,7 @@ boto3 = "^1.28.46" moto = "^4.2.2" types-sqlalchemy = "^1.4.53.38" urllib3 = "^1.26.17" -db-client = {git = "https://github.com/climatepolicyradar/navigator-db-client.git", tag = "v2.0.3"} +db-client = { git = "https://github.com/climatepolicyradar/navigator-db-client.git", tag = "v2.0.3" } [tool.poetry.dev-dependencies] pre-commit = "^2.17.0" @@ -51,9 +56,7 @@ env_files = """ .env.test .env """ -markers = [ - "unit", -] +markers = ["unit"] asyncio_mode = "strict" [tool.pydocstyle] @@ -67,9 +70,8 @@ exclude = "^/alembic/versions/" [tool.pyright] include = ["app", "unit_tests", "integration_tests"] -exclude = [ - "**/__pycache__", -] +exclude = ["**/__pycache__"] +pythonVersion = "3.9" [tool.ruff] select = ["E", "F", "D"] @@ -87,7 +89,21 @@ select = ["E", "F", "D"] # D407 - Missing dashed underline after section # D413 - Missing blank line after last section # D415 - First line should end with a period, question mark, or exclamation point -ignore = ["D100", "D103", "D104", "D107", "D202", "D203", "D212", "D400", "D401", "D406", "D407", "D413", "D415"] +ignore = [ + "D100", + "D103", + "D104", + "D107", + "D202", + "D203", + "D212", + "D400", + "D401", + "D406", + "D407", + "D413", + "D415", +] line-length = 88 # Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`. diff --git a/scripts/generate_new_user_sql.sh b/scripts/generate_new_user_sql.sh index e2f1c680..cb0a7112 100755 --- a/scripts/generate_new_user_sql.sh +++ b/scripts/generate_new_user_sql.sh @@ -1,9 +1,9 @@ #!/bin/bash new_user() { - P1=$(python -c "from app.core.security import get_password_hash; print(get_password_hash('$2'))") - echo "insert into app_user values('$1', 'Guest User', '${P1}', FALSE);" - echo "insert into organisation_admin(appuser_email, organisation_id, job_title, is_active, is_admin) values ('$1', $3, '', TRUE, FALSE);" + P1=$(python -c "from app.core.security import get_password_hash; print(get_password_hash('$2'))") + echo "insert into app_user values('$1', 'Guest User', '${P1}', FALSE);" + echo "insert into organisation_admin(appuser_email, organisation_id, job_title, is_active, is_admin) values ('$1', $3, '', TRUE, FALSE);" } # Add a list of new users here: @@ -13,4 +13,3 @@ new_user() { # - organisation id new_user "cclw@climatepolicyradar.org" "mypassword" 1 - diff --git a/scripts/testing/get-all-documents b/scripts/testing/get-all-documents index 905381bc..2eb1dc84 100755 --- a/scripts/testing/get-all-documents +++ b/scripts/testing/get-all-documents @@ -4,5 +4,5 @@ source config TOKEN=$(get_token) curl -s \ - -H "Authorization: Bearer ${TOKEN}" \ - ${TEST_URL}/api/v1/documents + -H "Authorization: Bearer ${TOKEN}" \ + "${TEST_URL}"/api/v1/documents diff --git a/scripts/testing/get-all-families b/scripts/testing/get-all-families index d28922a3..85e50d65 100755 --- a/scripts/testing/get-all-families +++ b/scripts/testing/get-all-families @@ -4,5 +4,5 @@ source config TOKEN=$(get_token) curl -s \ - -H "Authorization: Bearer ${TOKEN}" \ - ${TEST_URL}/api/v1/families | jq + -H "Authorization: Bearer ${TOKEN}" \ + "${TEST_URL}"/api/v1/families | jq diff --git a/scripts/testing/get-config b/scripts/testing/get-config index 30fed6dd..0bb1de59 100755 --- a/scripts/testing/get-config +++ b/scripts/testing/get-config @@ -4,5 +4,5 @@ source config TOKEN=$(get_token) curl -s \ - -H "Authorization: Bearer ${TOKEN}" \ - ${TEST_URL}/api/v1/config| jq + -H "Authorization: Bearer ${TOKEN}" \ + "${TEST_URL}"/api/v1/config | jq diff --git a/scripts/testing/get-family b/scripts/testing/get-family index bc77d69d..3b023e24 100755 --- a/scripts/testing/get-family +++ b/scripts/testing/get-family @@ -4,5 +4,5 @@ source config TOKEN=$(get_token) curl -s \ - -H "Authorization: Bearer ${TOKEN}" \ - ${TEST_URL}/api/v1/families/CCLW.family.1002.0| jq + -H "Authorization: Bearer ${TOKEN}" \ + "${TEST_URL}"/api/v1/families/CCLW.family.1002.0 | jq diff --git a/scripts/testing/search-families b/scripts/testing/search-families index e23acb95..6b96833d 100755 --- a/scripts/testing/search-families +++ b/scripts/testing/search-families @@ -5,5 +5,5 @@ source config TOKEN=$(get_token) curl -s \ - -H "Authorization: Bearer ${TOKEN}" \ - ${TEST_URL}/api/v1/families/?q=$*| jq + -H "Authorization: Bearer ${TOKEN}" \ + "${TEST_URL}"/api/v1/families/?q="$*" | jq diff --git a/scripts/testing/update-family b/scripts/testing/update-family index 7dd9a59f..753119cb 100755 --- a/scripts/testing/update-family +++ b/scripts/testing/update-family @@ -4,7 +4,7 @@ source config TOKEN=$(get_token) -curl -s -X PUT --data @family \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - ${TEST_URL}/api/v1/families| jq +curl -s -X PUT --data @family \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + "${TEST_URL}"/api/v1/families | jq diff --git a/scripts/update-db-model.sh b/scripts/update-db-model.sh index 1e37cade..f1cf97ec 100755 --- a/scripts/update-db-model.sh +++ b/scripts/update-db-model.sh @@ -1,7 +1,7 @@ #!/bin/bash # -cd $(git rev-parse --show-toplevel) +cd $(git rev-parse --show-toplevel) || exit NAV=../navigator-backend # Squirrel away this file to restore later @@ -11,7 +11,7 @@ cp app/clients/db/models/app/authorisation.py /tmp rm -rf app/clients/db/models # copy new db model -cp -r ${NAV}/app/db/models app/clients/db/models +cp -r "${NAV}"/app/db/models app/clients/db/models # restore the file - TODO: we shoud really move this methinks cp /tmp/authorisation.py app/clients/db/models/app/authorisation.py diff --git a/unit_tests/conftest.py b/unit_tests/conftest.py index 5278d8c0..8e75bf01 100644 --- a/unit_tests/conftest.py +++ b/unit_tests/conftest.py @@ -5,50 +5,48 @@ """ from typing import Dict -from app.clients.aws.client import get_s3_client -from app.main import app + import pytest from fastapi.testclient import TestClient from moto import mock_s3 -import app.service.family as family_service +import app.service.analytics as analytics_service +import app.service.app_user as app_user_service import app.service.collection as collection_service -import app.service.document as document_service import app.service.config as config_service -import app.service.token as token_service -import app.service.analytics as analytics_service +import app.service.document as document_service import app.service.event as event_service -import app.service.app_user as app_user_service - +import app.service.family as family_service +import app.service.token as token_service +from app.clients.aws.client import get_s3_client +from app.main import app from app.repository import ( - family_repo, - geography_repo, - metadata_repo, - organisation_repo, - collection_repo, app_user_repo, + collection_repo, config_repo, document_repo, event_repo, + family_repo, + geography_repo, + metadata_repo, + organisation_repo, ) from unit_tests.mocks.repos import create_mock_family_repo from unit_tests.mocks.repos.app_user_repo import mock_app_user_repo from unit_tests.mocks.repos.collection_repo import mock_collection_repo +from unit_tests.mocks.repos.config_repo import mock_config_repo from unit_tests.mocks.repos.document_repo import mock_document_repo - +from unit_tests.mocks.repos.event_repo import mock_event_repo from unit_tests.mocks.repos.geography_repo import mock_geography_repo from unit_tests.mocks.repos.metadata_repo import mock_metadata_repo from unit_tests.mocks.repos.organisation_repo import mock_organisation_repo -from unit_tests.mocks.repos.config_repo import mock_config_repo -from unit_tests.mocks.repos.event_repo import mock_event_repo - +from unit_tests.mocks.services.analytics_service import mock_analytics_service from unit_tests.mocks.services.app_user_service import mock_app_user_service -from unit_tests.mocks.services.family_service import mock_family_service from unit_tests.mocks.services.collection_service import mock_collection_service -from unit_tests.mocks.services.document_service import mock_document_service from unit_tests.mocks.services.config_service import mock_config_service -from unit_tests.mocks.services.analytics_service import mock_analytics_service +from unit_tests.mocks.services.document_service import mock_document_service from unit_tests.mocks.services.event_service import mock_event_service +from unit_tests.mocks.services.family_service import mock_family_service @pytest.fixture diff --git a/unit_tests/helpers/analytics.py b/unit_tests/helpers/analytics.py index 754a027b..41c0da5f 100644 --- a/unit_tests/helpers/analytics.py +++ b/unit_tests/helpers/analytics.py @@ -1,4 +1,5 @@ from typing import Optional + from app.model.analytics import SummaryDTO # ANALYTICS SUMMARY diff --git a/unit_tests/helpers/collection.py b/unit_tests/helpers/collection.py index c91a2e5f..a8457671 100644 --- a/unit_tests/helpers/collection.py +++ b/unit_tests/helpers/collection.py @@ -1,4 +1,5 @@ from datetime import datetime + from app.model.collection import ( CollectionCreateDTO, CollectionReadDTO, diff --git a/unit_tests/helpers/document.py b/unit_tests/helpers/document.py index 8c7fe4fb..857591c1 100644 --- a/unit_tests/helpers/document.py +++ b/unit_tests/helpers/document.py @@ -1,8 +1,9 @@ from datetime import datetime from typing import Optional, cast -from pydantic import AnyHttpUrl from db_client.models.law_policy.family import DocumentStatus +from pydantic import AnyHttpUrl + from app.model.document import DocumentCreateDTO, DocumentReadDTO, DocumentWriteDTO diff --git a/unit_tests/helpers/event.py b/unit_tests/helpers/event.py index 2b325fff..1bf91d63 100644 --- a/unit_tests/helpers/event.py +++ b/unit_tests/helpers/event.py @@ -1,7 +1,8 @@ +from datetime import datetime, timezone + from db_client.models.law_policy.family import EventStatus -from app.model.event import EventCreateDTO, EventReadDTO, EventWriteDTO -from datetime import datetime, timezone +from app.model.event import EventCreateDTO, EventReadDTO, EventWriteDTO def create_event_read_dto( diff --git a/unit_tests/helpers/family.py b/unit_tests/helpers/family.py index 010437a0..2f704c75 100644 --- a/unit_tests/helpers/family.py +++ b/unit_tests/helpers/family.py @@ -1,6 +1,8 @@ from datetime import datetime from typing import Optional + from db_client.models.law_policy.family import FamilyCategory + from app.model.family import FamilyCreateDTO, FamilyReadDTO, FamilyWriteDTO diff --git a/unit_tests/mocks/repos/__init__.py b/unit_tests/mocks/repos/__init__.py index 8f284fa1..f622eff3 100644 --- a/unit_tests/mocks/repos/__init__.py +++ b/unit_tests/mocks/repos/__init__.py @@ -1,6 +1,7 @@ from pytest import MonkeyPatch from app.repository.protocols import FamilyRepo + from . import family_repo as mock_repo mock_repo: FamilyRepo diff --git a/unit_tests/mocks/repos/app_user_repo.py b/unit_tests/mocks/repos/app_user_repo.py index 710db769..b12dac47 100644 --- a/unit_tests/mocks/repos/app_user_repo.py +++ b/unit_tests/mocks/repos/app_user_repo.py @@ -1,10 +1,10 @@ from typing import Tuple -from pytest import MonkeyPatch + from db_client.models.app.users import AppUser, Organisation, OrganisationUser +from pytest import MonkeyPatch -from app.repository.app_user import MaybeAppUser import app.service.authentication as auth_service - +from app.repository.app_user import MaybeAppUser PLAIN_PASSWORD = "test-password" HASH_PASSWORD = auth_service.get_password_hash(PLAIN_PASSWORD) diff --git a/unit_tests/mocks/repos/config_repo.py b/unit_tests/mocks/repos/config_repo.py index f59a51ee..84c12594 100644 --- a/unit_tests/mocks/repos/config_repo.py +++ b/unit_tests/mocks/repos/config_repo.py @@ -1,6 +1,6 @@ from typing import Optional -from pytest import MonkeyPatch +from pytest import MonkeyPatch from sqlalchemy import exc from app.model.config import ConfigReadDTO, DocumentConfig, EventConfig diff --git a/unit_tests/mocks/repos/geography_repo.py b/unit_tests/mocks/repos/geography_repo.py index 38781e50..14e9dbf8 100644 --- a/unit_tests/mocks/repos/geography_repo.py +++ b/unit_tests/mocks/repos/geography_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch diff --git a/unit_tests/mocks/repos/metadata_repo.py b/unit_tests/mocks/repos/metadata_repo.py index 0007db98..b48dbb5e 100644 --- a/unit_tests/mocks/repos/metadata_repo.py +++ b/unit_tests/mocks/repos/metadata_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch from app.model.general import Json diff --git a/unit_tests/mocks/repos/organisation_repo.py b/unit_tests/mocks/repos/organisation_repo.py index 44a819e0..8aa84d4c 100644 --- a/unit_tests/mocks/repos/organisation_repo.py +++ b/unit_tests/mocks/repos/organisation_repo.py @@ -1,4 +1,5 @@ from typing import Optional + from pytest import MonkeyPatch diff --git a/unit_tests/mocks/services/analytics_service.py b/unit_tests/mocks/services/analytics_service.py index bdd8d8bd..14f0bd10 100644 --- a/unit_tests/mocks/services/analytics_service.py +++ b/unit_tests/mocks/services/analytics_service.py @@ -1,6 +1,6 @@ from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.analytics import SummaryDTO from unit_tests.helpers.analytics import create_summary_dto diff --git a/unit_tests/mocks/services/config_service.py b/unit_tests/mocks/services/config_service.py index 9b3a3589..14f77ad5 100644 --- a/unit_tests/mocks/services/config_service.py +++ b/unit_tests/mocks/services/config_service.py @@ -1,6 +1,6 @@ from pytest import MonkeyPatch -from app.errors import RepositoryError +from app.errors import RepositoryError from app.model.config import ConfigReadDTO, DocumentConfig, EventConfig diff --git a/unit_tests/routers/test_analytics.py b/unit_tests/routers/test_analytics.py index ae48e490..0aafda0f 100644 --- a/unit_tests/routers/test_analytics.py +++ b/unit_tests/routers/test_analytics.py @@ -3,6 +3,7 @@ This uses a service mock and ensures each endpoint calls into the service. """ + from fastapi import status from fastapi.testclient import TestClient @@ -11,7 +12,7 @@ def test_get_all_when_ok(client: TestClient, analytics_service_mock, user_header response = client.get("/api/v1/analytics/summary", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is dict + assert isinstance(data, dict) assert len(data) == 4 assert analytics_service_mock.summary.call_count == 1 @@ -20,7 +21,7 @@ def test_get_when_ok(client: TestClient, analytics_service_mock, user_header_tok response = client.get("/api/v1/analytics/summary", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is dict + assert isinstance(data, dict) assert len(data) == 4 assert analytics_service_mock.summary.call_count == 1 diff --git a/unit_tests/routers/test_collection.py b/unit_tests/routers/test_collection.py index c6e8e044..f7e6b2ad 100644 --- a/unit_tests/routers/test_collection.py +++ b/unit_tests/routers/test_collection.py @@ -3,15 +3,14 @@ This uses a service mock and ensures each endpoint calls into the service. """ + import logging import pytest from fastapi import status from fastapi.testclient import TestClient -from unit_tests.helpers.collection import ( - create_collection_write_dto, -) +from unit_tests.helpers.collection import create_collection_write_dto def test_get_all_when_ok( @@ -20,7 +19,7 @@ def test_get_all_when_ok( response = client.get("/api/v1/collections", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "test" assert collection_service_mock.all.call_count == 1 @@ -49,7 +48,7 @@ def test_search_when_ok(client: TestClient, collection_service_mock, user_header response = client.get("/api/v1/collections/?q=anything", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "search1" assert collection_service_mock.search.call_count == 1 diff --git a/unit_tests/routers/test_document.py b/unit_tests/routers/test_document.py index 6c63f5d7..5656f9fc 100644 --- a/unit_tests/routers/test_document.py +++ b/unit_tests/routers/test_document.py @@ -15,7 +15,7 @@ def test_get_all_when_ok(client: TestClient, user_header_token, document_service response = client.get("/api/v1/documents", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "test" assert document_service.all.call_count == 1 @@ -44,7 +44,7 @@ def test_search_when_ok(client: TestClient, document_service_mock, user_header_t response = client.get("/api/v1/documents/?q=anything", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "search1" assert document_service_mock.search.call_count == 1 diff --git a/unit_tests/routers/test_event.py b/unit_tests/routers/test_event.py index 6aff1cd2..350ec073 100644 --- a/unit_tests/routers/test_event.py +++ b/unit_tests/routers/test_event.py @@ -6,17 +6,14 @@ from fastapi.testclient import TestClient import app.service.event as event_service -from unit_tests.helpers.event import ( - create_event_create_dto, - create_event_write_dto, -) +from unit_tests.helpers.event import create_event_create_dto, create_event_write_dto def test_get_all_when_ok(client: TestClient, user_header_token, event_service_mock): response = client.get("/api/v1/events", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "test" assert event_service.all.call_count == 1 @@ -43,7 +40,7 @@ def test_search_when_ok(client: TestClient, event_service_mock, user_header_toke response = client.get("/api/v1/events/?q=anything", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "search1" assert event_service_mock.search.call_count == 1 diff --git a/unit_tests/routers/test_family.py b/unit_tests/routers/test_family.py index 5813875b..a16d7122 100644 --- a/unit_tests/routers/test_family.py +++ b/unit_tests/routers/test_family.py @@ -3,6 +3,7 @@ This uses a service mock and ensures each endpoint calls into the service. """ + import logging import pytest @@ -16,7 +17,7 @@ def test_get_all_when_ok(client: TestClient, family_service_mock, user_header_to response = client.get("/api/v1/families", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "test" assert family_service_mock.all.call_count == 1 @@ -43,7 +44,7 @@ def test_search_when_ok(client: TestClient, family_service_mock, user_header_tok response = client.get("/api/v1/families/?q=anything", headers=user_header_token) assert response.status_code == status.HTTP_200_OK data = response.json() - assert type(data) is list + assert isinstance(data, list) assert len(data) > 0 assert data[0]["import_id"] == "search1" assert family_service_mock.search.call_count == 1 diff --git a/unit_tests/service/test_analytics_service.py b/unit_tests/service/test_analytics_service.py index 18e6f4c5..05f98107 100644 --- a/unit_tests/service/test_analytics_service.py +++ b/unit_tests/service/test_analytics_service.py @@ -2,16 +2,15 @@ import pytest +import app.service.analytics as analytics_service from app.errors import RepositoryError from app.model.analytics import SummaryDTO -import app.service.analytics as analytics_service - from unit_tests.helpers.analytics import ( - create_summary_dto, - EXPECTED_NUM_DOCUMENTS, - EXPECTED_NUM_FAMILIES, EXPECTED_NUM_COLLECTIONS, + EXPECTED_NUM_DOCUMENTS, EXPECTED_NUM_EVENTS, + EXPECTED_NUM_FAMILIES, + create_summary_dto, ) diff --git a/unit_tests/service/test_authorisation_service.py b/unit_tests/service/test_authorisation_service.py index 63ba40be..4c3150ca 100644 --- a/unit_tests/service/test_authorisation_service.py +++ b/unit_tests/service/test_authorisation_service.py @@ -1,7 +1,8 @@ import pytest -from app.errors import AuthenticationError, RepositoryError + import app.service.authentication as auth_service import app.service.token as token_service +from app.errors import AuthenticationError, RepositoryError from unit_tests.mocks.repos.app_user_repo import ( HASH_PASSWORD, PLAIN_PASSWORD, diff --git a/unit_tests/service/test_family_service.py b/unit_tests/service/test_family_service.py index f2760654..62d297b5 100644 --- a/unit_tests/service/test_family_service.py +++ b/unit_tests/service/test_family_service.py @@ -3,6 +3,7 @@ Uses a family repo mock and ensures that the repo is called. """ + from typing import Optional import pytest diff --git a/unit_tests/service/test_geography_service.py b/unit_tests/service/test_geography_service.py index 41eaba90..f59f5d5c 100644 --- a/unit_tests/service/test_geography_service.py +++ b/unit_tests/service/test_geography_service.py @@ -1,6 +1,7 @@ import pytest -from app.errors import ValidationError + import app.service.geography as geography_service +from app.errors import ValidationError def test_geo_service_validate_raises_when_invalid( diff --git a/unit_tests/service/test_id_service.py b/unit_tests/service/test_id_service.py index 27342711..cb63f92b 100644 --- a/unit_tests/service/test_id_service.py +++ b/unit_tests/service/test_id_service.py @@ -1,4 +1,5 @@ import pytest + from app.errors import ValidationError from app.service import id diff --git a/unit_tests/service/test_metadata_service.py b/unit_tests/service/test_metadata_service.py index 5f4002e5..9f0cb78e 100644 --- a/unit_tests/service/test_metadata_service.py +++ b/unit_tests/service/test_metadata_service.py @@ -1,10 +1,10 @@ import json import pytest + from app.errors import ValidationError from app.service.metadata import _validate - SCHEMA_VALUES_NO_BLANKS = json.loads( """ { diff --git a/unit_tests/service/test_token_service.py b/unit_tests/service/test_token_service.py index c46fa148..82f2daca 100644 --- a/unit_tests/service/test_token_service.py +++ b/unit_tests/service/test_token_service.py @@ -1,8 +1,10 @@ from typing import cast + import jwt import pytest -from app.errors import TokenError + import app.service.token as token_service +from app.errors import TokenError @pytest.mark.parametrize(