Add _event schema as alembic migration to fix admin tests (#69)

* Add event metadata field

* Bump to 3.8.20

* Remove old code

* Update _event sub taxonomy key

* Bump to 3.8.21

* _event taxonomy as alembic migration to fix admin tests

* Bump to 3.8.22

* Fix expected taxonomy test

* Consolidate queries

* Fix tests

Author: katybaulch

db_client/alembic/versions/0057_add__event_sub_taxonomy.py

@@ -0,0 +1,48 @@
+"""Add _event sub taxonomy.
+
+Revision ID: 0057
+Revises: 0056
+Create Date: 2024-10-29 14:01:33.721363
+
+"""
+
+import os
+
+import sqlalchemy as sa
+from alembic import op
+
+from db_client.utils import get_library_path
+
+revision = "0057"
+down_revision = "0056"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Get the path to the SQL file
+    sql_file_path = os.path.join(
+        get_library_path(),
+        "alembic",
+        "versions",
+        "data",
+        "0057",
+        "add-_event-schema.sql",
+    )
+
+    # Read the SQL file
+    with open(sql_file_path, "r") as file:
+        sql_commands = file.read()
+
+    # Execute the SQL commands
+    op.execute(sa.text(sql_commands))
+
+
+def downgrade():
+    op.execute(
+        sa.text(
+            "UPDATE corpus_type "
+            "SET valid_metadata = valid_metadata - '_event' "
+            "WHERE valid_metadata ? '_event';"
+        )
+    )

db_client/alembic/versions/data/0057/add-_event-schema.sql

@@ -0,0 +1,62 @@
+WITH updated_values AS (
+    SELECT
+        name,
+        CASE
+            WHEN name IN ('Intl. agreements', 'Laws and Policies') THEN jsonb_build_array('Passed/Approved')
+            WHEN name = 'GEF' THEN jsonb_build_array('Concept Approved')
+            ELSE jsonb_build_array('Project Approved')
+        END AS new_allowed_values
+    FROM corpus_type
+)
+UPDATE corpus_type
+SET valid_metadata = jsonb_set(
+    valid_metadata,
+    '{_event}',
+    COALESCE(
+        valid_metadata->'_event',
+        jsonb_build_object()
+    ) || jsonb_build_object(
+        'event_type',
+        COALESCE(
+            valid_metadata->'_event'->'event_type',
+            jsonb_build_object(
+                'allow_any', false,
+                'allow_blanks', true,
+                'allowed_values', (
+                    SELECT jsonb_agg(value)
+                    FROM jsonb_array_elements_text(
+                        (
+                            SELECT jsonb_path_query_first(
+                                ct.valid_metadata,
+                                '$.event_type.allowed_values'
+                            )
+                            FROM corpus_type ct
+                            WHERE ct.name = corpus_type.name
+                            LIMIT 1
+                        )
+                    ) AS value
+                )
+            )
+        ),
+        'datetime_event_name',
+        COALESCE(
+            valid_metadata->'_event'->'datetime_event_name',
+            jsonb_build_object(
+                'allow_any', false,
+                'allow_blanks', false,
+                'allowed_values', updated_values.new_allowed_values
+            )
+        )
+    )
+)
+FROM updated_values
+WHERE corpus_type.name = updated_values.name
+AND (
+    NOT (valid_metadata ? '_event')
+    OR NOT (valid_metadata->'_event' ? 'event_type')
+    OR NOT (valid_metadata->'_event' ? 'datetime_event_name')
+)
+AND corpus_type.name IN (
+    SELECT DISTINCT corpus_type_name
+    FROM corpus
+);

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "db-client"
-version = "3.8.21"
+version = "3.8.22"
 description = "All things to do with the datamodel and its storage. Including alembic migrations and datamodel code."
 authors = ["CPR-dev-team <[email protected]>"]
 license = "Apache-2.0"

tests/functions/test_get_taxonomy_by_corpus_type_name.py

@@ -34,6 +34,36 @@
             "Updated",
         ],
     },
+    "_event": {
+        "event_type": {
+            "allow_any": False,
+            "allow_blanks": True,
+            "allowed_values": [
+                "Amended",
+                "Appealed",
+                "Closed",
+                "Declaration Of Climate Emergency",
+                "Dismissed",
+                "Entered Into Force",
+                "Filing",
+                "Granted",
+                "Implementation Details",
+                "International Agreement",
+                "Net Zero Pledge",
+                "Other",
+                "Passed/Approved",
+                "Repealed/Replaced",
+                "Set",
+                "Settled",
+                "Updated",
+            ],
+        },
+        "datetime_event_name": {
+            "allow_any": False,
+            "allow_blanks": False,
+            "allowed_values": ["Passed/Approved"],
+        },
+    },
     "_document": {
         "role": {
             "allow_any": False,

tests/schema/test_inital_data.py

@@ -14,6 +14,7 @@
     "instrument",
     "event_type",
     "_document",
+    "_event",
 }
 EXPECTED_CCLW_TOPICS = 4
 EXPECTED_CCLW_HAZARDS = 81
@@ -22,10 +23,17 @@
 EXPECTED_CCLW_FRAMEWORKS = 3
 EXPECTED_CCLW_INSTRUMENTS = 25
 
-EXPECTED_UNFCCC_TAXONOMY = {"author", "author_type", "event_type", "_document"}
+EXPECTED_UNFCCC_TAXONOMY = {
+    "author",
+    "author_type",
+    "event_type",
+    "_document",
+    "_event",
+}
 EXPECTED_UNFCCC_AUTHOR_TYPES = 2
 
 EXPECTED_DOCUMENT_TAXONOMY_KEYS = 2
+EXPECTED_EVENT_TAXONOMY_KEYS = 2
 
 EXPECTED_ORGANISATIONS = 2
 EXPECTED_EVENT_TYPES = 17
@@ -128,6 +136,7 @@ def test_corpora_values_correct(
                 ("author_type", EXPECTED_UNFCCC_AUTHOR_TYPES),
                 ("event_type", EXPECTED_EVENT_TYPES),
                 ("_document", EXPECTED_DOCUMENT_TAXONOMY_KEYS),
+                ("_event", EXPECTED_EVENT_TAXONOMY_KEYS),
             ],
         ),
         (
@@ -142,6 +151,7 @@ def test_corpora_values_correct(
                 ("instrument", EXPECTED_CCLW_INSTRUMENTS),
                 ("event_type", EXPECTED_EVENT_TYPES),
                 ("_document", EXPECTED_DOCUMENT_TAXONOMY_KEYS),
+                ("_event", EXPECTED_EVENT_TAXONOMY_KEYS),
             ],
         ),
     ],
@@ -187,6 +197,13 @@ def test_taxonomy_value_counts_correct(
                         ("type", EXPECTED_DOCUMENT_TYPE),
                     ],
                 ),
+                (
+                    "_event",
+                    [
+                        ("datetime_event_name", 1),
+                        ("event_type", EXPECTED_EVENT_TYPES),
+                    ],
+                ),
             ],
         ),
         (
@@ -199,6 +216,13 @@ def test_taxonomy_value_counts_correct(
                         ("type", EXPECTED_DOCUMENT_TYPE),
                     ],
                 ),
+                (
+                    "_event",
+                    [
+                        ("datetime_event_name", 1),
+                        ("event_type", EXPECTED_EVENT_TYPES),
+                    ],
+                ),
             ],
         ),
     ],
@@ -251,27 +275,25 @@ def test_entity_specific_taxonomy_value_counts_correct(
         # This will give us _document taxonomy keys such as role, type etc
         entity_specific_keys = schema.keys()
         expected_entity_specific_keys = [
-            _entity_specific_key
-            for _, _entity_specific_keys in expected_taxonomy_items
-            for _entity_specific_key, __ in _entity_specific_keys
+            key for key, _ in entity_specific_taxonomy_items
         ]
 
         test_c = set(entity_specific_keys).difference(
             set(expected_entity_specific_keys)
         )
         if test_c != set():
             logging.error(
-                f"Actual entity specific key(s) {test_c} not present "
-                f"in list of expected entity specific keys {expected_entity_specific_keys}"
+                f"Actual {entity_specific_key} specific key(s) {test_c} not present "
+                f"in list of expected {entity_specific_key} specific keys {expected_entity_specific_keys}"
             )
 
         test_d = set(expected_entity_specific_keys).difference(
             set(entity_specific_keys)
         )
         if test_d != set():
             logging.error(
-                f"Expected entity specific key(s) {test_d} not present "
-                f"in list of actual entity specific keys {entity_specific_keys}"
+                f"Expected {entity_specific_key} specific key(s) {test_d} not present "
+                f"in list of actual {entity_specific_key} specific keys {entity_specific_keys}"
             )
 
         assert set(entity_specific_keys) ^ set(expected_entity_specific_keys) == set()