Add dynamic adaptation properties prototype

README.md

@@ -164,7 +164,7 @@ As an example, below we update the backend on a development machine:
 
 ```bash
 # Edit docker-compose-prod-build.yaml image version:
-#     image: ghcr.io/nismod/gri-backend:1.7.2
+#     image: ghcr.io/nismod/gri-backend:1.8.0
 
 # Build
 docker compose -f docker-compose-prod-build.yaml build backend
@@ -173,17 +173,17 @@ docker compose -f docker-compose-prod-build.yaml build backend
 # see: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry
 
 # Push
-docker push ghcr.io/nismod/gri-backend:1.7.2
+docker push ghcr.io/nismod/gri-backend:1.8.0
 ```
 
 On the production remote, pull the image and restart the service:
 
 ```bash
 # Pull image
-docker pull ghcr.io/nismod/gri-backend:1.7.2
+docker pull ghcr.io/nismod/gri-backend:1.8.0
 
 # Edit docker-compose-prod-deploy.yaml image version (or sync up):
-#     image: ghcr.io/nismod/gri-backend:1.7.2
+#     image: ghcr.io/nismod/gri-backend:1.8.0
 
 # Restart service
 docker compose -f docker-compose-prod-deploy.yaml up -d backend

containers/backend/backend/app/internal/attribute_access.py

@@ -3,11 +3,13 @@
 from sqlalchemy import Column
 from sqlalchemy.orm import Query
 from sqlalchemy.sql import functions
-from sqlalchemy.sql.operators import ColumnOperators
 from pydantic import Json, ValidationError
 
-
 from backend.app import schemas
+from backend.app.internal.dynamic_data_config import (
+    DynamicDataConfig,
+    build_sql_expression,
+)
 from backend.db import models
 
 
@@ -41,6 +43,33 @@ def add_damages_expected_value_query(
 #     pass
 
 
+# potentially store the config in the database or a config file
+ADAPTATIONS_CONFIG_TEXT = """
+properties:
+  avoided_ead_amin:
+    type: float
+    json_key: "avoided_ead_amin"
+  avoided_ead_mean:
+    type: float
+    json_key: "avoided_ead_mean"
+  avoided_ead_amax:
+    type: float
+    json_key: "avoided_ead_amax"
+  adaptation_cost:
+    type: float
+    json_key: "adaptation_cost"
+  cost_benefit_ratio:
+    type: calculated
+    expression: "{avoided_ead_mean} / {adaptation_cost}"
+"""
+
+import yaml
+
+ADAPTATIONS_CONFIG = DynamicDataConfig.model_validate(
+    yaml.safe_load(ADAPTATIONS_CONFIG_TEXT)
+)
+
+
 def add_adaptation_value_query(
     fq: Query,
     dimensions: schemas.AdaptationDimensions,
@@ -55,18 +84,10 @@ def add_adaptation_value_query(
         adaptation_protection_level=dimensions.adaptation_protection_level,
     )
 
-    value: Column | ColumnOperators | None = None
-
-    if field == "cost_benefit_ratio":
-        cost_benefit_params: schemas.AdaptationCostBenefitRatioParameters = field_params
-        eael_days = cost_benefit_params.eael_days
-
-        value = (
-            models.AdaptationCostBenefit.avoided_ead_mean
-            + models.AdaptationCostBenefit.avoided_eael_mean * eael_days
-        ) / models.AdaptationCostBenefit.adaptation_cost
-    else:
-        value = getattr(models.AdaptationCostBenefit, field)
+    params = field_params.model_dump() if field_params else None
+    value = build_sql_expression(
+        models.AdaptationCostBenefit.properties, ADAPTATIONS_CONFIG, field, params
+    )
 
     return q.add_columns(value.label("value"))
 
@@ -101,9 +122,6 @@ class DataGroupConfig:
         dimensions_schema=schemas.AdaptationDimensions,
         variables_schema=schemas.AdaptationVariables,
         add_value_query=add_adaptation_value_query,
-        field_parameters_schemas={
-            "cost_benefit_ratio": schemas.AdaptationCostBenefitRatioParameters,
-        },
     ),
 }
 

containers/backend/backend/app/internal/dynamic_data_config.py

@@ -0,0 +1,48 @@
+from typing import Any, Literal
+from pydantic import BaseModel
+from sqlalchemy import Column, Float, cast, literal_column
+
+class StaticPropertyConfig(BaseModel):
+    type: str
+    json_key: str
+
+class CalculatedPropertyConfig(BaseModel):
+    type: Literal['calculated']
+    expression: str
+
+class DynamicDataConfig(BaseModel):
+    properties: dict[str, StaticPropertyConfig | CalculatedPropertyConfig]
+
+def build_sql_expression(data_column: Column, data_config: DynamicDataConfig, field, params: dict[str, Any]=None):
+    """
+    Build a SQL expression for the given field based on the configuration.
+    """
+    properties_config = data_config.properties
+    field_config = properties_config.get(field)
+    if not field_config:
+        raise KeyError(f"Field '{field}' is not defined in configuration.")
+
+    field_type = field_config.type
+    if field_type == "calculated":
+        # Get the fully qualified data column table.column specifier
+        data_column_spec = f"{data_column.table.name}.{data_column.key}"
+
+        # Substitute keys in the expression with their JSONB paths
+        expression: str = field_config.expression
+        for key, prop in properties_config.items():
+            if prop.type != 'calculated':
+                json_key = prop.json_key
+                expression = expression.replace(
+                    f"{{{key}}}",
+                    f"(CAST({data_column_spec}::jsonb->>'{json_key}' AS FLOAT))"
+                )
+        # Substitute parameters if provided
+        if params:
+            for param_key, param_value in params.items():
+                expression = expression.replace(f"{{{param_key}}}", str(param_value))
+        return literal_column(expression)
+    elif field_type == "float":
+        json_key = field_config.json_key
+        return cast(data_column.op('->>')(json_key), Float)
+    else:
+        raise ValueError(f"Unsupported field type '{field_type}'.")

containers/backend/backend/app/main.py

@@ -28,7 +28,7 @@ def custom_generate_unique_id(route: APIRoute):
     generate_unique_id_function=custom_generate_unique_id,
     title="GRI Infra-Risk-Vis API",
     description="API Supporting Global Resilience Initiative Visualisation UI.  Serving geospatial features (inc. related damages) and raster tiles (via Terracotta)",
-    version="0.8.0",
+    version="0.9.0",
     terms_of_service="",
     contact={
         "name": "Tom Russell",

containers/backend/backend/app/routers/features.py

@@ -1,9 +1,9 @@
-from typing import Optional
-
+from typing import Any, Optional
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi_pagination import Page, Params
 from fastapi_pagination.ext.sqlalchemy import paginate
-from sqlalchemy import desc, select
+from pydantic import Json
+from sqlalchemy import desc, select, Column, Text
 from sqlalchemy.exc import NoResultFound
 from geoalchemy2 import functions
 
@@ -46,6 +46,18 @@ def get_layer_spec(
     )
 
 
+# parse json dictionary from ranking_scope parameter
+def parse_ranking_scope(ranking_scope: Json = None):
+    return ranking_scope or {}
+
+
+def add_jsonb_filters(jsonb_column: Column, filters: dict[str, Any]):
+    return [
+        jsonb_column.op("->>")(key).cast(Text) == str(value)
+        for key, value in filters.items()
+    ]
+
+
 @router.get(
     "/sorted-by/{field_group}", response_model=Page[schemas.FeatureListItemOut[float]]
 )
@@ -57,8 +69,12 @@ def read_sorted_features(
     field_params: schemas.DataParameters = Depends(parse_parameters),
     layer_spec: schemas.LayerSpec = Depends(get_layer_spec),
     page_params: Params = Depends(),
+    ranking_scope: dict = Depends(parse_ranking_scope),
 ):
-    filled_layer_spec = {k: v for k, v in layer_spec.dict().items() if v is not None}
+    filled_layer_spec = {
+        k: v for k, v in layer_spec.model_dump().items() if v is not None
+    }
+    jsonb_filters = add_jsonb_filters(models.Feature.properties, ranking_scope)
     base_query = (
         select(
             models.Feature.id.label("id"),
@@ -67,6 +83,7 @@ def read_sorted_features(
             functions.ST_AsText(functions.Box2D(models.Feature.geom)).label("bbox_wkt"),
         )
         .select_from(models.Feature)
+        .filter(*jsonb_filters)
         .join(models.FeatureLayer)
         .filter_by(**filled_layer_spec)
     )

containers/backend/backend/app/schemas.py

@@ -128,7 +128,8 @@ def fix_eael_days(cls, eael_days: int) -> float:
         return eael_days / 15
 
 
-class Adaptation(AdaptationDimensions, AdaptationVariables):
+class Adaptation(AdaptationDimensions):
+    properties: dict
     model_config = ConfigDict(from_attributes=True)
 
 

containers/backend/backend/db/debug_query.py

@@ -0,0 +1,7 @@
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.orm import Query
+
+def stringify_query(query: Query) -> str:
+    return str(query.statement.compile(
+        dialect=postgresql.dialect(),
+        compile_kwargs={"literal_binds": True}))

containers/backend/backend/db/models.py

@@ -28,7 +28,7 @@ class Feature(Base):
     layer = Column(
         String, ForeignKey(FeatureLayer.layer_name), index=True, nullable=False
     )
-    properties = Column(JSON, nullable=False)
+    properties = Column(JSONB, nullable=False)
     geom = Column(Geometry("GEOMETRY", srid=4326), nullable=False)
 
     layer_info = relationship("FeatureLayer")
@@ -120,15 +120,8 @@ class AdaptationCostBenefit(Base):
     adaptation_name = Column(String, nullable=False, primary_key=True)
     adaptation_protection_level = Column(Float, nullable=False, primary_key=True)
 
-    adaptation_cost = Column(Float)
+    properties = Column(JSONB, nullable=False)
 
-    avoided_ead_amin = Column(Float)
-    avoided_ead_mean = Column(Float)
-    avoided_ead_amax = Column(Float)
-
-    avoided_eael_amin = Column(Float)
-    avoided_eael_mean = Column(Float)
-    avoided_eael_amax = Column(Float)
 
     def __repr__(self) -> str:
         return f"AdaptationCostBenefit(feature_id={self.feature_id!r}, hazard={self.hazard!r}, rcp={self.rcp!r}, adaptation_name={self.adaptation_name!r}, adaptation_protection_level={self.adaptation_protection_level!r}, adaptation_cost={self.adaptation_cost!r}, avoided_ead_mean={self.avoided_ead_mean!r})"

containers/vector/Dockerfile

@@ -1,6 +1,11 @@
 FROM maptiler/tileserver-gl:v4.11.1
 LABEL maintainer="frederick.thomas@ouce.ox.ac.uk"
 
+# used in run.sh - override through docker compose if needed
+# this URL will be contained in the responses from the tileserver
+# it should be a URL accessible from the client
+ENV TILESERVER_PUBLIC_URL="http://localhost/vector"
+
 WORKDIR /
 
 # copy in required config
@@ -12,4 +17,4 @@ COPY ./config.json .
 USER root
 RUN chmod -R u+r /data
 
-CMD ["tileserver-gl-light", "-c", "config.json", "-p", "8080", "--verbose", "-u", "http://localhost/vector"]
+CMD ["./run.sh"]

containers/vector/config.json

@@ -52,6 +52,33 @@
     },
     "seismic_and_flooding_exposure_per_country": {
       "mbtiles": "seismic_and_flooding_exposure_per_country.mbtiles"
+    },
+    "nbs": {
+      "mbtiles": "nbs.mbtiles"
+    },
+    "adm0": {
+      "mbtiles": "adm0.mbtiles"
+    },
+    "adm0_points": {
+      "mbtiles": "adm0_points.mbtiles"
+    },
+    "adm1": {
+      "mbtiles": "adm1.mbtiles"
+    },
+    "adm1_points": {
+      "mbtiles": "adm1_points.mbtiles"
+    },
+    "adm2": {
+      "mbtiles": "adm2.mbtiles"
+    },
+    "adm2_points": {
+      "mbtiles": "adm2_points.mbtiles"
+    },
+    "hybas": {
+      "mbtiles": "hybas.mbtiles"
+    },
+    "hybas_points": {
+      "mbtiles": "hybas_points.mbtiles"
     }
   }
 }

containers/vector/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+tileserver-gl-light -c config.json -p 8080 --verbose -u "$TILESERVER_PUBLIC_URL"

docker-compose-dev.yaml

@@ -45,7 +45,7 @@ services:
 
   backend:
     build: ./containers/backend
-    image: ghcr.io/nismod/gri-backend:1.7.2
+    image: ghcr.io/nismod/gri-backend:1.8.0
     volumes:
       - ./etl/raster/cog/:/data/
       - ./containers/backend/backend/:/code/backend/
@@ -68,6 +68,8 @@ services:
     volumes:
       - ./tileserver/vector/data:/data # bind mount
       - ./containers/vector/config-dev.json:/config.json
+    env_file:
+      - ./envs/dev/.vector-tileserver.env
     ports:
       - 8800:8080
     labels:
@@ -118,7 +120,7 @@ services:
   ## WARNING - this will wipe the existing tables. TODO: Alembic
   recreate-metadata-schema:
     build: ./containers/backend
-    image: ghcr.io/nismod/gri-backend:1.7.2
+    image: ghcr.io/nismod/gri-backend:1.8.0
     profiles: ["recreate-metadata-schema"]
     volumes:
       - ./containers/backend/backend/db/models.py:/code/backend/db/models.py

docker-compose-prod-build.yaml

@@ -1,6 +1,6 @@
 services:
   backend:
-    image: ghcr.io/nismod/gri-backend:1.7.2
+    image: ghcr.io/nismod/gri-backend:1.8.0
     build: ./containers/backend
 
   vector-tileserver:

docker-compose-prod-deploy.yaml

@@ -45,7 +45,7 @@ services:
     mem_limit: "250M"
 
   backend:
-    image: ghcr.io/nismod/gri-backend:1.7.2
+    image: ghcr.io/nismod/gri-backend:1.8.0
     restart: always
     volumes:
       - ./tileserver/raster/data:/data
@@ -68,17 +68,8 @@ services:
   vector-tileserver:
     image: ghcr.io/nismod/gri-vector-tileserver:0.10
     restart: always
-    command:
-      [
-        "tileserver-gl-light",
-        "-c",
-        "config.json",
-        "-p",
-        "8080",
-        "--verbose",
-        "-u",
-        "https://global.infrastructureresilience.org/vector",
-      ]
+    environment:
+      TILESERVER_PUBLIC_URL: https://global.infrastructureresilience.org/vector
     volumes:
       - ./tileserver/vector/data:/data
       - ./vector-tileserver/config.json:/config.json

envs/dev-example/.vector-tileserver.env

@@ -0,0 +1 @@
+TILESERVER_PUBLIC_URL=http://localhost/vector