Merge pull request #241 from GispoCoding/217-type-of-plan-regulation-…

…group

217 type of plan regulation group

database/base.py

@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from geoalchemy2 import Geometry
 from shapely.geometry import MultiLineString, MultiPoint, MultiPolygon
@@ -73,7 +73,8 @@ class CodeBase(VersionedBase):
 
     __abstract__ = True
     __table_args__ = {"schema": "codes"}
-    code_list_uri = ""
+    code_list_uri = ""  # the URI to use for looking for codes online
+    local_codes: List[Dict] = []  # local codes to add to the code list
 
     value: Mapped[unique_str]
     short_name: Mapped[str] = mapped_column(server_default="", index=True)

database/codes.py

@@ -95,3 +95,21 @@ class AdministrativeRegion(CodeBase):
 
     __tablename__ = "administrative_region"
     code_list_uri = "http://uri.suomi.fi/codelist/jhs/maakunta_1_20240101"
+
+
+class TypeOfPlanRegulationGroup(CodeBase):
+    """
+    Kaavamääräysryhmän tyyppi
+
+    This is our own code list. It does not exist in koodistot.suomi.fi.
+    """
+
+    __tablename__ = "type_of_plan_regulation_group"
+    code_list_uri = ""
+    local_codes = [
+        {"value": "generalRegulations", "name": {"fin": "Yleismääräykset"}},
+        {"value": "landUseRegulations", "name": {"fin": "Aluevaraus"}},
+        {"value": "otherAreaRegulations", "name": {"fin": "Osa-alue"}},
+        {"value": "lineRegulations", "name": {"fin": "Viiva"}},
+        {"value": "otherPointRegulations", "name": {"fin": "Muu piste"}},
+    ]

database/koodistot_loader/koodistot_loader.py

@@ -8,7 +8,6 @@
 import codes
 import requests
 from sqlalchemy import create_engine
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session, sessionmaker
 
 """
@@ -26,8 +25,12 @@ class Response(TypedDict):
 
 
 class Event(TypedDict):
-    pass
-    # event_type: int  # EventType
+    """
+    Supports creating codes both online and from local code classes.
+    """
+
+    suomifi_codes: Optional[bool]
+    local_codes: Optional[bool]
 
 
 class DatabaseHelper:
@@ -87,52 +90,79 @@ class KoodistotLoader:
     HEADERS = {"User-Agent": "HAME - Ryhti compatible Maakuntakaava database"}
     api_base = "https://koodistot.suomi.fi/codelist-api/api/v1/coderegistries"
 
-    def __init__(self, connection_string: str, api_url: Optional[str] = None) -> None:
+    def __init__(
+        self,
+        connection_string: str,
+        api_url: Optional[str] = None,
+        load_suomifi_codes: Optional[bool] = True,
+        load_local_codes: Optional[bool] = True,
+    ) -> None:
         if api_url:
             self.api_base = api_url
         engine = create_engine(connection_string)
         self.Session = sessionmaker(bind=engine)
 
-        # Only load koodistot that have external URI defined
+        # Only load koodistot that have data source defined
         self.koodistot: List[Type[codes.CodeBase]] = [
             value
             for name, value in inspect.getmembers(codes, inspect.isclass)
-            if issubclass(value, codes.CodeBase) and value.code_list_uri
+            if issubclass(value, codes.CodeBase)
+            and (
+                (load_suomifi_codes and value.code_list_uri)
+                or (load_local_codes and value.local_codes)
+            )
         ]
+        if load_suomifi_codes:
+            LOGGER.info("Loading codes from suomi.fi")
+        if load_local_codes:
+            LOGGER.info("Loading local codes")
         LOGGER.info("Loader initialized with code classes:")
         LOGGER.info(self.koodistot)
 
-    def get_objects(self) -> Dict[Type[codes.CodeBase], List[dict]]:
+    def get_code_registry_data(self, koodisto: Type[codes.CodeBase]) -> List[Dict]:
+        """
+        Get code registry codes for given koodisto, or empty list if not present.
+        """
+        if not koodisto.code_list_uri:
+            return []
+        code_registry, name = koodisto.code_list_uri.rsplit("/", 2)[-2:None]
+        LOGGER.info(koodisto.code_list_uri)
+        LOGGER.info(code_registry)
+        LOGGER.info(name)
+        url = get_code_list_url(self.api_base, code_registry, name)
+        LOGGER.info(f"Loading codes from {url}")
+        r = requests.get(url, headers=self.HEADERS)
+        r.raise_for_status()
+        try:
+            return r.json()["results"]
+        except (KeyError, requests.exceptions.JSONDecodeError):
+            LOGGER.warning(f"{koodisto} response did not contain data")
+            return []
+
+    def get_objects(self) -> Dict[Type[codes.CodeBase], List[Dict]]:
         """
         Gets all koodistot data, divided by table.
         """
         data = dict()
         for koodisto in self.koodistot:
-            code_registry, name = koodisto.code_list_uri.rsplit("/", 2)[-2:None]
-            LOGGER.info(koodisto.code_list_uri)
-            LOGGER.info(code_registry)
-            LOGGER.info(name)
-            url = get_code_list_url(self.api_base, code_registry, name)
-            LOGGER.info(f"Loading codes from {url}")
-            r = requests.get(url, headers=self.HEADERS)
-            r.raise_for_status()
-            try:
-                data[koodisto] = r.json()["results"]
-            except (KeyError, requests.exceptions.JSONDecodeError):
-                LOGGER.warning(f"{koodisto} response did not contain data")
-                data[koodisto] = []
+            # Fetch external codes
+            data[koodisto] = self.get_code_registry_data(koodisto)
+            # Add local codes with status to distinguish them from other codes
+            local_codes = [dict(code, status="LOCAL") for code in koodisto.local_codes]
+            data[koodisto] += local_codes
         return data
 
-    def get_object(self, element: Dict) -> Optional[dict]:
+    def get_object(self, element: Dict) -> Optional[Dict]:
         """
         Returns database-ready dict of object to import, or None if the data
         was invalid.
         """
+        # local codes are already in database-ready format
+        if element["status"] == "LOCAL":
+            return element
         code_dict = dict()
-        # SQLAlchemy merge() doesn't know how to handle unique constraints that are not
-        # pk. Therefore, we will have to specify the primary key here (not generated in
-        # db) so we will not get an IntegrityError. Use uuids from koodistot.suomi.fi.
-        # https://sqlalchemy.narkive.com/mCDgZiDa/why-does-session-merge-only-look-at-primary-key-and-not-all-unique-keys
+        # Use uuids from koodistot.suomi.fi. This way, we can save all the children
+        # easily by referring to their parents.
         code_dict["id"] = element["id"]
         code_dict["value"] = element["codeValue"]
         short_name = element.get("shortName", None)
@@ -158,33 +188,37 @@ def get_object(self, element: Dict) -> Optional[dict]:
             code_dict["parent_id"] = parent["id"]
         return code_dict
 
-    def create_object(
-        self, code_class: Type[codes.CodeBase], incoming: Dict[str, Any]
+    def update_or_create_object(
+        self,
+        code_class: Type[codes.CodeBase],
+        incoming: Dict[str, Any],
+        session: Session,
     ) -> codes.CodeBase:
         """
-        Create code_class instance with incoming field values.
+        Find object based on its unique fields, or create new object. Update fields
+        that are present in the incoming dict.
         """
-        column_keys = set(code_class.__table__.columns.keys())
-        vals = {
+        columns = code_class.__table__.columns
+        unique_keys = set(column.key for column in columns if column.unique)
+        unique_values = {
+            key: incoming[key] for key in set(incoming.keys()).intersection(unique_keys)
+        }
+        instance = session.query(code_class).filter_by(**unique_values).first()
+        column_keys = set(columns.keys())
+        values = {
             key: incoming[key] for key in set(incoming.keys()).intersection(column_keys)
         }
-        return code_class(**vals)
-
-    def save_object(
-        self, code_class: Type[codes.CodeBase], object: Dict[str, Any], session: Session
-    ) -> bool:
-        """
-        Save object defined in the object dict as instance of code_class.
-        """
-        new_obj = self.create_object(code_class, object)
-        try:
-            session.merge(new_obj)
-        except SQLAlchemyError as e:
-            # We want to crash for now. That way we'll know if there is a problem with
-            # the data.
-            raise e
-            # LOGGER.exception(f"Error occurred while saving {object}")
-        return True
+        if instance:
+            # go figure, if we have the instance (and don't want to do the update right
+            # now) sqlalchemy has no way of supplying attribute dict to be updated.
+            # This is because dirtying sqlalchemy objects happens via the __setattr__
+            # method, so we will have to update instance fields one by one.
+            for key, value in values.items():
+                setattr(instance, key, value)
+        else:
+            instance = code_class(**values)
+            session.add(instance)
+        return instance
 
     def save_objects(self, objects: Dict[Type[codes.CodeBase], List[dict]]) -> str:
         """
@@ -201,11 +235,15 @@ def save_objects(self, objects: Dict[Type[codes.CodeBase], List[dict]]) -> str:
                         )
                     code = self.get_object(element)
                     if code is not None:
-                        succeeded = self.save_object(code_class, code, session)
+                        succeeded = self.update_or_create_object(
+                            code_class, code, session
+                        )
                         if succeeded:
                             successful_actions += 1
+                        else:
+                            LOGGER.debug(f"Could not save code data {element}")
                     else:
-                        LOGGER.debug(f"Could not save code data {element}")
+                        LOGGER.debug(f"Invalid code data {element}")
             session.commit()
         msg = f"{successful_actions} inserted or updated. 0 deleted."
         LOGGER.info(msg)
@@ -216,8 +254,14 @@ def handler(event: Event, _) -> Response:
     """Handler which is called when accessing the endpoint."""
     response: Response = {"statusCode": 200, "body": json.dumps("")}
     db_helper = DatabaseHelper()
+    load_suomifi_codes = event.get("suomifi_codes", True)
+    load_local_codes = event.get("local_codes", True)
 
-    loader = KoodistotLoader(db_helper.get_connection_string())
+    loader = KoodistotLoader(
+        db_helper.get_connection_string(),
+        load_suomifi_codes=load_suomifi_codes,
+        load_local_codes=load_local_codes,
+    )
     LOGGER.info("Getting objects...")
     objects = loader.get_objects()
 

database/migrations/versions/2024_02_28_1337-6bab5b3f52d1_add_type_of_plan_regulation_group_code_.py

@@ -0,0 +1,124 @@
+"""add type of plan regulation group code table
+
+Revision ID: 6bab5b3f52d1
+Revises: ad29908c50ad
+Create Date: 2024-02-28 13:37:39.733685
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# import geoalchemy2
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "6bab5b3f52d1"
+down_revision: Union[str, None] = "f085148de65d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "type_of_plan_regulation_group",
+        sa.Column("value", sa.String(), nullable=False),
+        sa.Column("short_name", sa.String(), server_default="", nullable=False),
+        sa.Column(
+            "name",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default='{"fin": "", "swe": "", "eng": ""}',
+            nullable=False,
+        ),
+        sa.Column(
+            "description",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default='{"fin": "", "swe": "", "eng": ""}',
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column("level", sa.Integer(), server_default="1", nullable=False),
+        sa.Column("parent_id", sa.UUID(), nullable=True),
+        sa.Column(
+            "id",
+            sa.UUID(),
+            server_default=sa.text("gen_random_uuid()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "modified_at",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"],
+            ["codes.type_of_plan_regulation_group.id"],
+            name="type_of_plan_regulation_group_parent_id_fkey",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="codes",
+    )
+    op.create_index(
+        op.f("ix_codes_type_of_plan_regulation_group_level"),
+        "type_of_plan_regulation_group",
+        ["level"],
+        unique=False,
+        schema="codes",
+    )
+    op.create_index(
+        op.f("ix_codes_type_of_plan_regulation_group_parent_id"),
+        "type_of_plan_regulation_group",
+        ["parent_id"],
+        unique=False,
+        schema="codes",
+    )
+    op.create_index(
+        op.f("ix_codes_type_of_plan_regulation_group_short_name"),
+        "type_of_plan_regulation_group",
+        ["short_name"],
+        unique=False,
+        schema="codes",
+    )
+    op.create_index(
+        op.f("ix_codes_type_of_plan_regulation_group_value"),
+        "type_of_plan_regulation_group",
+        ["value"],
+        unique=True,
+        schema="codes",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_codes_type_of_plan_regulation_group_value"),
+        table_name="type_of_plan_regulation_group",
+        schema="codes",
+    )
+    op.drop_index(
+        op.f("ix_codes_type_of_plan_regulation_group_short_name"),
+        table_name="type_of_plan_regulation_group",
+        schema="codes",
+    )
+    op.drop_index(
+        op.f("ix_codes_type_of_plan_regulation_group_parent_id"),
+        table_name="type_of_plan_regulation_group",
+        schema="codes",
+    )
+    op.drop_index(
+        op.f("ix_codes_type_of_plan_regulation_group_level"),
+        table_name="type_of_plan_regulation_group",
+        schema="codes",
+    )
+    op.drop_table("type_of_plan_regulation_group", schema="codes")
+    # ### end Alembic commands ###