Remove taxonomies from config (#136)

* Only show families belonging to user org unless super

* Remove debug

* Placate pyright

* Bump to 2.6.1

* Replace print with logging

* Show all corpora if superuser

* Use non-superuser user for tests

* Remove old taxonomy tests and check all corpora returned for superuser

* Bump to 2.6.2

* Take common query out of conditional

* Remove taxonomies

* Add venv support for configuring pyright

.trunk/configure-pyright.sh

@@ -0,0 +1,52 @@
+#! /usr/bin/env bash
+set -e
+
+# Get the name of the expected venv for this repo from the pyproject.toml file.
+venv_name=$(grep -m 1 venv pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3) || true
+
+# Check if pyrightconfig already exists.
+if [[ ! -f pyrightconfig.json ]]; then
+
+	if [[ -z ${venv_name} ]]; then
+		echo "Using Poetry"
+		poetry_env_info=$(poetry env info --path) >/dev/null 2>&1
+		venv_path=${poetry_env_info%/*} # ABCDE
+		venv_name=${poetry_env_info#*/} # 12345
+
+		# Generate the pyrightconfig.json file.
+		json_string=$(jq -n \
+			--arg v "${venv_name}" \
+			--arg vp "${venv_path}" \
+			'{venv: $v, venvPath: $vp} ')
+
+		echo "${json_string}" >pyrightconfig.json
+	else
+		echo "Using Pyenv"
+		# Check if pyenv-pyright plugin is installed
+		if ! command -v pyenv &>/dev/null; then
+			echo "pyenv not installed. Please install pyenv..."
+			exit 1
+		fi
+
+		pyenv_root=$(pyenv root)
+		dir_path="${pyenv_root}"/plugins/pyenv-pyright
+		if [[ ! -d ${dir_path} ]]; then
+			# trunk-ignore(shellcheck/SC2312)
+			if [[ -n $(ls -A "${dir_path}") ]]; then
+				git clone https://github.com/alefpereira/pyenv-pyright.git "${dir_path}"
+			fi
+		fi
+
+		# Generate the pyrightconfig.json file.
+		pyenv pyright "${venv_name}"
+		pyenv local "${venv_name}"
+	fi
+fi
+
+# Check whether required keys are present in pyrightconfig.json.
+if ! jq -r --arg venv_name "${venv_name}" '. | select((.venv != $venv_name or .venv == "") and (.venvPath == null or .venvPath == ""))' pyrightconfig.json >/dev/null 2>&1; then
+	echo "Failed to configure pyright to use environment '${venv_name}' as interpreter. Please check pyrightconfig.json..."
+	exit 1
+fi
+echo "All done!"
+exit 0

.trunk/trunk.yaml

@@ -74,11 +74,11 @@ actions:
     - trunk-announce
   enabled:
     - trunk-check-pre-commit
-    - configure-pyright-with-pyenv
+    - configure-pyright
     - trunk-fmt-pre-commit
     - trunk-upgrade-available
   definitions:
-    - id: configure-pyright-with-pyenv
-      run: source .trunk/configure-pyright-with-pyenv.sh
+    - id: configure-pyright
+      run: source .trunk/configure-pyright.sh
       triggers:
         - git_hooks: [pre-commit]

app/api/api_v1/query_params.py

@@ -1,10 +1,13 @@
+import logging
 from typing import Union, cast
 
 from fastapi import HTTPException, status
 
+_LOGGER = logging.getLogger(__name__)
+
 
 def get_query_params_as_dict(query_params) -> dict[str, Union[str, int]]:
-    print(query_params)
+    _LOGGER.debug("Query params: %s", query_params)
     return {k: query_params[k] for k in query_params.keys()}
 
 

app/model/config.py

@@ -34,9 +34,6 @@ class ConfigReadDTO(BaseModel):
     """Definition of the new Config which just includes taxonomy."""
 
     geographies: Sequence[dict]
-    taxonomies: Mapping[
-        str, TaxonomyData
-    ]  # TODO: Will be Mapping[str, Sequence[CorpusData]] after PDCT-1052 finished
     corpora: Sequence[CorpusData]
     languages: Mapping[str, str]
     document: DocumentConfig

app/repository/config.py

@@ -22,6 +22,7 @@
     EventConfig,
     TaxonomyData,
 )
+from app.repository import app_user
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -89,7 +90,9 @@ def _to_corpus_data(row, event_types) -> CorpusData:
     )
 
 
-def get_corpora_for_org(db: Session, org_id: int) -> Sequence[CorpusData]:
+def get_corpora(
+    db: Session, user_email: str, is_superuser: bool
+) -> Sequence[CorpusData]:
     corpora = (
         db.query(
             Corpus.import_id.label("corpus_import_id"),
@@ -104,9 +107,12 @@ def get_corpora_for_org(db: Session, org_id: int) -> Sequence[CorpusData]:
             Corpus.corpus_type_name == CorpusType.name,
         )
         .join(Organisation, Organisation.id == Corpus.organisation_id)
-        .filter(Organisation.id == org_id)
-        .all()
     )
+    if is_superuser:
+        corpora = corpora.all()
+    else:
+        org_id = app_user.get_org_id(db, user_email)
+        corpora = corpora.filter(Organisation.id == org_id).all()
 
     event_types = db.query(FamilyEventType).all()
     entry = TaxonomyEntry(
@@ -117,7 +123,7 @@ def get_corpora_for_org(db: Session, org_id: int) -> Sequence[CorpusData]:
     return [_to_corpus_data(row, entry) for row in corpora]
 
 
-def get(db: Session, org_id: int) -> ConfigReadDTO:
+def get(db: Session, user_email: str) -> ConfigReadDTO:
     """
     Returns the configuration for the admin service.
 
@@ -134,7 +140,8 @@ def get(db: Session, org_id: int) -> ConfigReadDTO:
         if tax is not None:
             taxonomies[org.name] = tax
 
-    corpora = get_corpora_for_org(db, org_id)
+    is_superuser = app_user.is_superuser(db, user_email)
+    corpora = get_corpora(db, user_email, is_superuser)
 
     languages = {lang.language_code: lang.name for lang in db.query(Language).all()}
 
@@ -169,7 +176,6 @@ def get(db: Session, org_id: int) -> ConfigReadDTO:
     )
     return ConfigReadDTO(
         geographies=geographies,
-        taxonomies=taxonomies,
         corpora=corpora,
         languages=languages,
         document=doc_config,

app/service/config.py

@@ -7,7 +7,6 @@
 import app.repository.config as config_repo
 from app.errors import RepositoryError
 from app.model.config import ConfigReadDTO
-from app.service import app_user
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -20,13 +19,9 @@ def get(user_email: str) -> ConfigReadDTO:
     :raises RepositoryError: If there is an issue getting the config
     :return ConfigReadDTO: The config for the application
     """
-
     try:
         with db_session.get_db() as db:
-            # Get the organisation from the user's email
-            org_id = app_user.get_organisation(db, user_email)
-
-            return config_repo.get(db, org_id)
+            return config_repo.get(db, user_email)
 
     except exc.SQLAlchemyError:
         _LOGGER.exception("Error while getting config")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "admin_backend"
-version = "2.6.1"
+version = "2.6.2"
 description = ""
 authors = ["CPR-dev-team <[email protected]>"]
 packages = [{ include = "app" }, { include = "tests" }]

tests/integration_tests/config/test_config.py

@@ -23,81 +23,26 @@ def test_get_config_has_expected_keys(
     keys = data.keys()
 
     assert "geographies" in keys
-    assert "taxonomies" in keys
     assert "corpora" in keys
     assert "languages" in keys
     assert "document" in keys
     assert "event" in keys
 
 
-def test_get_config_has_correct_organisations(
-    client: TestClient, data_db: Session, user_header_token
+def test_get_config_has_correct_number_corpora_super(
+    client: TestClient, data_db: Session, superuser_header_token
 ):
     setup_db(data_db)
 
     response = client.get(
         "/api/v1/config",
-        headers=user_header_token,
+        headers=superuser_header_token,
     )
     assert response.status_code == status.HTTP_200_OK
     data = response.json()
 
     # Now sanity check the data
-    #
-    # Organisations.
-    LEN_ORG_CONFIG = 2
-
-    assert "CCLW" in data["taxonomies"].keys()
-    cclw_org = data["taxonomies"]["CCLW"]
-    assert len(cclw_org) == LEN_ORG_CONFIG
-
-    assert "UNFCCC" in data["taxonomies"]
-    unfccc_org = data["taxonomies"]["UNFCCC"]
-    assert len(unfccc_org) == LEN_ORG_CONFIG
-
-
-# TODO: Remove as part of PDCT-1052
-def test_get_config_cclw_old_taxonomy_correct(
-    client: TestClient, data_db: Session, user_header_token
-):
-    setup_db(data_db)
-
-    response = client.get(
-        "/api/v1/config",
-        headers=user_header_token,
-    )
-    assert response.status_code == status.HTTP_200_OK
-    data = response.json()
-
-    # Now sanity check the old taxonomy data
-    assert "CCLW" in data["taxonomies"].keys()
-    cclw_taxonomy = data["taxonomies"]["CCLW"]
-    assert set(cclw_taxonomy) == EXPECTED_CCLW_TAXONOMY
-    cclw_taxonomy_colours = cclw_taxonomy["color"]["allowed_values"]
-    assert set(cclw_taxonomy_colours) ^ set(EXPECTED_CCLW_COLOURS) == set()
-
-
-# TODO: Remove as part of PDCT-1052
-def test_get_config_unfccc_old_taxonomy_correct(
-    client: TestClient, data_db: Session, user_header_token
-):
-    setup_db(data_db)
-
-    response = client.get(
-        "/api/v1/config",
-        headers=user_header_token,
-    )
-    assert response.status_code == status.HTTP_200_OK
-    data = response.json()
-
-    # Now sanity check the old taxonomy data
-    assert "UNFCCC" in data["taxonomies"]
-    unfccc_taxonomy = data["taxonomies"]["UNFCCC"]
-    assert set(unfccc_taxonomy) == EXPECTED_UNFCCC_TAXONOMY
-    assert set(unfccc_taxonomy["author_type"]["allowed_values"]) == {
-        "Party",
-        "Non-Party",
-    }
+    assert len(data["corpora"]) == 2
 
 
 def test_get_config_has_correct_number_corpora_cclw(
@@ -189,6 +134,47 @@ def test_get_config_unfccc_corpora_correct(
     assert set(unfccc_corporas[0]["taxonomy"]) ^ expected_unfccc_taxonomy == set()
 
 
+def test_get_config_corpora_correct(
+    client: TestClient, data_db: Session, superuser_header_token
+):
+    setup_db(data_db)
+
+    response = client.get(
+        "/api/v1/config",
+        headers=superuser_header_token,
+    )
+    assert response.status_code == status.HTTP_200_OK
+    data = response.json()
+
+    # Now sanity check the new corpora data
+    corpora = data["corpora"]
+
+    assert corpora[0]["corpus_import_id"] == "CCLW.corpus.i00000001.n0000"
+    assert corpora[0]["corpus_type"] == "Laws and Policies"
+    assert corpora[0]["corpus_type_description"] == "Laws and policies"
+    assert corpora[0]["description"] == "CCLW national policies"
+    assert corpora[0]["title"] == "CCLW national policies"
+
+    cclw_taxonomy = corpora[0]["taxonomy"]
+    expected_cclw_taxonomy = {"color", "size"}
+    expected_cclw_taxonomy.add("event_types")
+    assert set(cclw_taxonomy) ^ expected_cclw_taxonomy == set()
+
+    expected_cclw_colours = ["green", "red", "pink", "blue"]
+    cclw_taxonomy_colours = cclw_taxonomy["color"]["allowed_values"]
+    assert set(cclw_taxonomy_colours) ^ set(expected_cclw_colours) == set()
+
+    assert corpora[1]["corpus_import_id"] == "UNFCCC.corpus.i00000001.n0000"
+    assert corpora[1]["corpus_type"] == "Intl. agreements"
+    assert corpora[1]["corpus_type_description"] == "Intl. agreements"
+    assert corpora[1]["description"] == "UNFCCC Submissions"
+    assert corpora[1]["title"] == "UNFCCC Submissions"
+
+    expected_unfccc_taxonomy = {"author", "author_type"}
+    expected_unfccc_taxonomy.add("event_types")
+    assert set(corpora[1]["taxonomy"]) ^ expected_unfccc_taxonomy == set()
+
+
 def test_config_languages(client: TestClient, data_db: Session, user_header_token):
     setup_db(data_db)
 

tests/integration_tests/conftest.py

@@ -218,7 +218,7 @@ def superuser_header_token() -> Dict[str, str]:
 
 @pytest.fixture
 def user_header_token() -> Dict[str, str]:
-    a_token = token_service.encode("test@cpr.org", False, {"is_admin": False})
+    a_token = token_service.encode("cclw@cpr.org", False, {"is_admin": False})
     headers = {"Authorization": f"Bearer {a_token}"}
     return headers
 

tests/integration_tests/login/test_login.py

@@ -8,7 +8,7 @@
 def test_login_ok(client: TestClient, data_db: Session):
     setup_db(data_db)
 
-    form_data = {"username": "test@cpr.org", "password": "scruffycode"}
+    form_data = {"username": "cclw@cpr.org", "password": "scruffycode"}
 
     response = client.post(
         "/api/tokens",