diff --git a/database/base.py b/database/base.py index fa249ae..d2e1241 100644 --- a/database/base.py +++ b/database/base.py @@ -1,10 +1,10 @@ import uuid from datetime import datetime -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from geoalchemy2 import Geometry from shapely.geometry import MultiLineString, MultiPoint, MultiPolygon -from sqlalchemy import ForeignKey +from sqlalchemy import ForeignKey, Index from sqlalchemy.dialects.postgresql import JSONB, NUMRANGE, UUID, Range from sqlalchemy.orm import ( DeclarativeBase, @@ -56,7 +56,7 @@ class VersionedBase(Base): """ __abstract__ = True - __table_args__ = {"schema": "hame"} + __table_args__: Any = {"schema": "hame"} # Go figure. We have to *explicitly state* id is a mapped column, because id will # have to be defined inside all the subclasses for relationship remote_side @@ -152,11 +152,24 @@ class PlanObjectBase(PlanBase): __abstract__ = True + @declared_attr.directive + @classmethod + def __table_args__(cls): + return ( + Index( + f"ix_{cls.__tablename__}_plan_id_ordering", + "plan_id", + "ordering", + unique=True, + ), + PlanBase.__table_args__, + ) + description: Mapped[language_str] source_data_object: Mapped[str] = mapped_column(nullable=True) height_range: Mapped[numeric_range] height_unit: Mapped[str] = mapped_column(nullable=True) - ordering: Mapped[Optional[int]] = mapped_column(index=True) + ordering: Mapped[Optional[int]] type_of_underground_id: Mapped[uuid.UUID] = mapped_column( ForeignKey("codes.type_of_underground.id", name="type_of_underground_id_fkey"), index=True, diff --git a/database/migrations/versions/2024_12_02_1351-09f44e0e7110_ordering_index_fixes.py b/database/migrations/versions/2024_12_02_1351-09f44e0e7110_ordering_index_fixes.py new file mode 100644 index 0000000..3d6c3bd --- /dev/null +++ b/database/migrations/versions/2024_12_02_1351-09f44e0e7110_ordering_index_fixes.py @@ -0,0 +1,206 @@ +"""ordering index fixes + +Revision ID: 09f44e0e7110 +Revises: 4e3c0868ea98 +Create Date: 2024-12-02 15:22:07.740316 + +""" +from typing import Sequence, Union + +import geoalchemy2 +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "09f44e0e7110" +down_revision: Union[str, None] = "4e3c0868ea98" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "ix_hame_land_use_area_ordering", + table_name="land_use_area", + schema="hame", + ) + op.create_index( + "ix_land_use_area_plan_id_ordering", + "land_use_area", + ["plan_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index( + "ix_hame_land_use_point_ordering", + table_name="land_use_point", + schema="hame", + ) + op.create_index( + "ix_land_use_point_plan_id_ordering", + "land_use_point", + ["plan_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index("ix_hame_line_ordering", table_name="line", schema="hame") + op.create_index( + "ix_line_plan_id_ordering", + "line", + ["plan_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index("ix_hame_other_area_ordering", table_name="other_area", schema="hame") + op.create_index( + "ix_other_area_plan_id_ordering", + "other_area", + ["plan_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index( + "ix_hame_other_point_ordering", table_name="other_point", schema="hame" + ) + op.create_index( + "ix_other_point_plan_id_ordering", + "other_point", + ["plan_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index( + "ix_hame_plan_proposition_ordering", + table_name="plan_proposition", + schema="hame", + ) + op.create_index( + "ix_plan_proposition_plan_regulation_group_id_ordering", + "plan_proposition", + ["plan_regulation_group_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index( + "ix_hame_plan_regulation_ordering", + table_name="plan_regulation", + schema="hame", + ) + op.create_index( + "ix_plan_regulation_plan_regulation_group_id_ordering", + "plan_regulation", + ["plan_regulation_group_id", "ordering"], + unique=True, + schema="hame", + ) + op.drop_index( + "ix_hame_plan_regulation_group_ordering", + table_name="plan_regulation_group", + schema="hame", + ) + op.create_index( + "ix_plan_regulation_group_plan_id_ordering", + "plan_regulation_group", + ["plan_id", "ordering"], + unique=False, + schema="hame", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "ix_plan_regulation_group_plan_id_ordering", + table_name="plan_regulation_group", + schema="hame", + ) + op.create_index( + "ix_hame_plan_regulation_group_ordering", + "plan_regulation_group", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_plan_regulation_plan_regulation_group_id_ordering", + table_name="plan_regulation", + schema="hame", + ) + op.create_index( + "ix_hame_plan_regulation_ordering", + "plan_regulation", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_plan_proposition_plan_regulation_group_id_ordering", + table_name="plan_proposition", + schema="hame", + ) + op.create_index( + "ix_hame_plan_proposition_ordering", + "plan_proposition", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_other_point_plan_id_ordering", + table_name="other_point", + schema="hame", + ) + op.create_index( + "ix_hame_other_point_ordering", + "other_point", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_other_area_plan_id_ordering", + table_name="other_area", + schema="hame", + ) + op.create_index( + "ix_hame_other_area_ordering", + "other_area", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index("ix_line_plan_id_ordering", table_name="line", schema="hame") + op.create_index( + "ix_hame_line_ordering", + "line", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_land_use_point_plan_id_ordering", + table_name="land_use_point", + schema="hame", + ) + op.create_index( + "ix_hame_land_use_point_ordering", + "land_use_point", + ["ordering"], + unique=False, + schema="hame", + ) + op.drop_index( + "ix_land_use_area_plan_id_ordering", + table_name="land_use_area", + schema="hame", + ) + op.create_index( + "ix_hame_land_use_area_ordering", + "land_use_area", + ["ordering"], + unique=False, + schema="hame", + ) + # ### end Alembic commands ### diff --git a/database/migrations/versions/2024_12_02_1354-e9a045c08304_short_name_uniqueness_fix.py b/database/migrations/versions/2024_12_02_1354-e9a045c08304_short_name_uniqueness_fix.py new file mode 100644 index 0000000..218f54f --- /dev/null +++ b/database/migrations/versions/2024_12_02_1354-e9a045c08304_short_name_uniqueness_fix.py @@ -0,0 +1,66 @@ +"""short name uniqueness fix + +Revision ID: e9a045c08304 +Revises: 09f44e0e7110 +Create Date: 2024-12-02 13:54:23.903091 + +""" +from typing import Sequence, Union + +import geoalchemy2 +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e9a045c08304" +down_revision: Union[str, None] = "09f44e0e7110" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "plan_regulation_group", + "short_name", + existing_type=sa.VARCHAR(), + nullable=True, + schema="hame", + ) + op.drop_index( + "ix_hame_plan_regulation_group_short_name", + table_name="plan_regulation_group", + schema="hame", + ) + op.create_index( + "ix_plan_regulation_group_plan_id_short_name", + "plan_regulation_group", + ["plan_id", "short_name"], + unique=True, + schema="hame", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "ix_plan_regulation_group_plan_id_short_name", + table_name="plan_regulation_group", + schema="hame", + ) + op.create_index( + "ix_hame_plan_regulation_group_short_name", + "plan_regulation_group", + ["short_name"], + unique=True, + schema="hame", + ) + op.alter_column( + "plan_regulation_group", + "short_name", + existing_type=sa.VARCHAR(), + nullable=False, + schema="hame", + ) + # ### end Alembic commands ### diff --git a/database/models.py b/database/models.py index 99f5fe9..7d5f307 100644 --- a/database/models.py +++ b/database/models.py @@ -12,10 +12,9 @@ language_str, numeric_range, timestamp, - unique_str, ) from shapely.geometry import MultiLineString, MultiPoint, MultiPolygon -from sqlalchemy import Column, ForeignKey, Table, Uuid +from sqlalchemy import Column, ForeignKey, Index, Table, Uuid from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.sql import func @@ -193,8 +192,18 @@ class PlanRegulationGroup(VersionedBase): """ __tablename__ = "plan_regulation_group" + __table_args__ = ( + Index("ix_plan_regulation_group_plan_id_ordering", "plan_id", "ordering"), + Index( + "ix_plan_regulation_group_plan_id_short_name", + "plan_id", + "short_name", + unique=True, + ), + VersionedBase.__table_args__, + ) - short_name: Mapped[unique_str] + short_name: Mapped[str] = mapped_column(nullable=True) name: Mapped[language_str] plan_id: Mapped[uuid.UUID] = mapped_column( @@ -209,7 +218,7 @@ class PlanRegulationGroup(VersionedBase): ) plan: Mapped["Plan"] = relationship() - ordering: Mapped[Optional[int]] = mapped_column(index=True) + ordering: Mapped[Optional[int]] # värikoodi? type_of_plan_regulation_group_id: Mapped[uuid.UUID] = mapped_column( @@ -292,6 +301,15 @@ class PlanRegulation(PlanBase): """ __tablename__ = "plan_regulation" + __table_args__ = ( + Index( + "ix_plan_regulation_plan_regulation_group_id_ordering", + "plan_regulation_group_id", + "ordering", + unique=True, + ), + PlanBase.__table_args__, + ) plan_regulation_group_id: Mapped[uuid.UUID] = mapped_column( ForeignKey( @@ -446,7 +464,7 @@ class PlanRegulation(PlanBase): unit: Mapped[str] = mapped_column(nullable=True) text_value: Mapped[language_str] numeric_value: Mapped[float] = mapped_column(nullable=True) - ordering: Mapped[Optional[int]] = mapped_column(index=True) + ordering: Mapped[Optional[int]] class PlanProposition(PlanBase): @@ -455,6 +473,15 @@ class PlanProposition(PlanBase): """ __tablename__ = "plan_proposition" + __table_args__ = ( + Index( + "ix_plan_proposition_plan_regulation_group_id_ordering", + "plan_regulation_group_id", + "ordering", + unique=True, + ), + PlanBase.__table_args__, + ) plan_regulation_group_id: Mapped[uuid.UUID] = mapped_column( ForeignKey( @@ -471,7 +498,7 @@ class PlanProposition(PlanBase): # Let's load all the codes for objects joined. plan_theme = relationship("PlanTheme", backref="plan_propositions", lazy="joined") text_value: Mapped[language_str] - ordering: Mapped[Optional[int]] = mapped_column(index=True) + ordering: Mapped[Optional[int]] class SourceData(VersionedBase): diff --git a/database/test/conftest.py b/database/test/conftest.py index 307be81..491304f 100644 --- a/database/test/conftest.py +++ b/database/test/conftest.py @@ -27,6 +27,7 @@ codes_count: int = 16 # adjust me when adding tables matview_count: int = 0 # adjust me when adding views + USE_DOCKER = ( "1" # Use "" if you don't want pytest-docker to start and destroy the containers ) @@ -327,37 +328,49 @@ def assert_database_is_alright( # Check indexes cur.execute( - f"SELECT * FROM pg_indexes WHERE schemaname = 'hame' AND tablename = '{table_name}';" + f"SELECT indexdef FROM pg_indexes WHERE schemaname = 'hame' AND tablename = '{table_name}';" ) - indexes = cur.fetchall() + index_defs = [index_def for (index_def,) in cur] + cur.execute( f"SELECT column_name FROM information_schema.columns WHERE table_schema = 'hame' AND table_name = '{table_name}';" ) - columns = cur.fetchall() - if ("id",) in columns: - assert ( - "hame", - table_name, - f"{table_name}_pkey", - None, - f"CREATE UNIQUE INDEX {table_name}_pkey ON hame.{table_name} USING btree (id)", - ) in indexes - if ("geom",) in columns: + columns = [column for (column,) in cur] + + if "id" in columns: assert ( - "hame", - table_name, - f"idx_{table_name}_geom", - None, - f"CREATE INDEX idx_{table_name}_geom ON hame.{table_name} USING gist (geom)", - ) in indexes - if ("ordering",) in columns: + f"CREATE UNIQUE INDEX {table_name}_pkey ON hame.{table_name} USING btree (id)" + in index_defs + ) + if "geom" in columns: assert ( - "hame", - table_name, - f"ix_hame_{table_name}_ordering", - None, - f"CREATE INDEX ix_hame_{table_name}_ordering ON hame.{table_name} USING btree (ordering)", - ) in indexes + f"CREATE INDEX idx_{table_name}_geom ON hame.{table_name} USING gist (geom)" + in index_defs + ) + + # Check ordering index, all ordering columns should have an index + if "ordering" in columns: + if table_name == "plan_regulation_group": + assert ( + "CREATE INDEX ix_plan_regulation_group_plan_id_ordering " + "ON hame.plan_regulation_group USING btree (plan_id, ordering)" + ) in index_defs + elif table_name in ("plan_regulation", "plan_proposition"): + assert ( + f"CREATE UNIQUE INDEX ix_{table_name}_plan_regulation_group_id_ordering " + f"ON hame.{table_name} USING btree (plan_regulation_group_id, ordering)" + ) in index_defs + elif table_name in ( + "land_use_area", + "other_area", + "line", + "land_use_point", + "other_point", + ): + assert ( + f"CREATE UNIQUE INDEX ix_{table_name}_plan_id_ordering " + f"ON hame.{table_name} USING btree (plan_id, ordering)" + ) in index_defs # Check code tables cur.execute("SELECT tablename, tableowner FROM pg_tables WHERE schemaname='codes';")