fix: Pydantic, Pypgstac and Python Upgrades

.github/actions/cdk-deploy/action.yml

@@ -21,7 +21,7 @@ runs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         cache: 'pip'
         cache-dependency-path: |
           ${{ inputs.dir }}/setup.py

.github/workflows/pr.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - uses: actions/cache@v4
         with:
@@ -48,7 +48,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - uses: actions/cache@v4
         with:
@@ -102,7 +102,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Setup Node
         uses: actions/setup-node@v4

common/auth/setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"]
+inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic"]
 
 setup(
     name="veda_auth",

config.py

@@ -2,9 +2,10 @@
 from getpass import getuser
 from typing import List, Optional
 
-from pydantic import BaseSettings, Field, constr
+from pydantic import Field, constr
+from pydantic_settings import BaseSettings
 
-AwsSubnetId = constr(regex=r"^subnet-[a-z0-9]{17}$")
+AwsSubnetId = constr(pattern=r"^subnet-[a-z0-9]{17}$")
 
 
 class vedaAppSettings(BaseSettings):
@@ -146,6 +147,7 @@ class Config:
         """model config."""
 
         env_file = ".env"
+        extra = "ignore"
 
 
 veda_app_settings = vedaAppSettings()

database/infrastructure/config.py

@@ -2,7 +2,8 @@
 from typing import Optional
 
 from aws_cdk import aws_ec2, aws_rds
-from pydantic import BaseSettings, Field, validator
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings
 
 
 class vedaDBSettings(BaseSettings):
@@ -42,17 +43,17 @@ class vedaDBSettings(BaseSettings):
     max_locks_per_transaction: Optional[str] = Field(
         "1024",
         description="Number of database objects that can be locked simultaneously",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     work_mem: Optional[str] = Field(
         "64000",
         description="Maximum amount of memory to be used by a query operation before writing to temporary disk files",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     temp_buffers: Optional[str] = Field(
         "32000",
         description="maximum number of temporary buffers used by each session",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     use_rds_proxy: Optional[bool] = Field(
         False,
@@ -64,13 +65,15 @@ class vedaDBSettings(BaseSettings):
             "The instance class of the RDS instance "
             "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceClass.html"
         ),
+        validate_default=True,
     )
     rds_instance_size: Optional[str] = Field(
         aws_ec2.InstanceSize.SMALL.value,
         description=(
             "The size of the RDS instance "
             "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceSize.html"
         ),
+        validate_default=True,
     )
     rds_engine_full_version: Optional[str] = Field(
         aws_rds.PostgresEngineVersion.VER_14.postgres_full_version,
@@ -91,14 +94,14 @@ class vedaDBSettings(BaseSettings):
         description="Boolean if the RDS should be encrypted",
     )
 
-    @validator("rds_instance_class", pre=True, always=True)
+    @field_validator("rds_instance_class", mode="before")
     def convert_rds_class_to_uppercase(cls, value):
         """Convert to uppercase."""
         if isinstance(value, str):
             return value.upper()
         return value
 
-    @validator("rds_instance_size", pre=True, always=True)
+    @field_validator("rds_instance_size", mode="before")
     def convert_rds_size_to_uppercase(cls, value):
         """Convert to uppercase."""
         if isinstance(value, str):
@@ -110,6 +113,7 @@ class Config:
 
         env_file = ".env"
         env_prefix = "VEDA_DB_"
+        extra = "ignore"
 
 
 veda_db_settings = vedaDBSettings()

database/runtime/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.9:latest
+FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.11:latest
 
 ARG PGSTAC_VERSION
 RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"

docker-compose.yml

@@ -129,6 +129,12 @@ services:
       - PGDATABASE=postgis
       - DYNAMODB_ENDPOINT=http://localhost:8085
       - VEDA_DB_PGSTAC_VERSION=0.7.10
+      - AWS_REGION=us-west-2
+      - AWS_DEFAULT_REGION=us-west-2
+      - DYNAMODB_TABLE=veda
+      - STAC_URL=http://0.0.0.0:8081
+      - USERPOOL_ID=us-west-2_123456789
+      - CLIENT_ID=123456789
     ports:
       - "8083:8083"
     command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py"

ingest_api/infrastructure/config.py

@@ -2,9 +2,11 @@
 from typing import List, Optional
 
 import aws_cdk
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
+from pydantic import AnyHttpUrl, Field, StringConstraints
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing_extensions import Annotated
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = Annotated[str, StringConstraints(pattern=r"^arn:aws:iam::\d{12}:role/.+")]
 
 
 class IngestorConfig(BaseSettings):
@@ -92,11 +94,9 @@ class IngestorConfig(BaseSettings):
         False,
         description="Boolean to disable default API gateway endpoints for stac, raster, and ingest APIs. Defaults to false.",
     )
-
-    class Config:
-        case_sensitive = False
-        env_file = ".env"
-        env_prefix = "VEDA_"
+    model_config = SettingsConfigDict(
+        case_sensitive=False, env_file=".env", env_prefix="VEDA_", extra="ignore"
+    )
 
     @property
     def stack_name(self) -> str:

ingest_api/infrastructure/construct.py

@@ -52,7 +52,7 @@ def __init__(
             "RASTER_URL": config.veda_raster_api_cf_url,
             "ROOT_PATH": config.ingest_root_path,
             "STAGE": config.stage,
-            "COGNITO_DOMAIN": config.cognito_domain,
+            "COGNITO_DOMAIN": str(config.cognito_domain),
         }
 
         build_api_lambda_params = {

ingest_api/runtime/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/sam/build-python3.9:latest
+FROM public.ecr.aws/sam/build-python3.11:latest
 
 ARG PGSTAC_VERSION
 RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"

ingest_api/runtime/requirements.txt

@@ -1,17 +1,16 @@
-# Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 and 117
 cryptography>=42.0.5
 ddbcereal==2.1.1
 fastapi>=0.109.1
 fsspec==2023.3.0
 mangum>=0.15.0
 orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
-pydantic_ssm_settings>=0.2.0
-pydantic>=1.10.12
+pydantic>=2.4.1
 python-multipart==0.0.7
 requests>=2.27.1
 s3fs==2023.3.0
-stac-pydantic @ git+https://github.com/ividito/stac-pydantic.git@3f4cb381c85749bb4b15d1181179057ec0f51a94
+stac-pydantic==3.1.3
 boto3==1.24.59
 aws_xray_sdk>=2.6.0,<3
 aws-lambda-powertools>=1.18.0
+pydantic-settings>=2.4.1

ingest_api/runtime/src/config.py

@@ -1,23 +1,23 @@
-import os
-from getpass import getuser
 from typing import Optional
 
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
-from pydantic_ssm_settings import AwsSsmSourceConfig
+from pydantic import AnyHttpUrl, ConfigDict, Field, StringConstraints
+from pydantic_settings import BaseSettings
+from typing_extensions import Annotated
 from veda_auth import VedaAuth
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = Annotated[str, StringConstraints(pattern=r"^arn:aws:iam::\d{12}:role/.+")]
 
 
 class Settings(BaseSettings):
     dynamodb_table: str
 
     jwks_url: Optional[AnyHttpUrl] = Field(
-        description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json"  # noqa
+        None,
+        description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json",  # noqa
     )
 
     data_access_role_arn: Optional[AwsArn] = Field(  # type: ignore
-        description="ARN of AWS Role used to validate access to S3 data"
+        None, description="ARN of AWS Role used to validate access to S3 data"
     )
 
     aws_request_payer: Optional[str] = Field(
@@ -30,12 +30,13 @@ class Settings(BaseSettings):
     userpool_id: str = Field(description="The Cognito Userpool used for authentication")
 
     cognito_domain: Optional[AnyHttpUrl] = Field(
-        description="The base url of the Cognito domain for authorization and token urls"
+        None,
+        description="The base url of the Cognito domain for authorization and token urls",
     )
-    client_id: str = Field(description="The Cognito APP client ID")
+    client_id: str = Field(None, description="The Cognito APP client ID")
     client_secret: str = Field("", description="The Cognito APP client secret")
     root_path: Optional[str] = None
-    stage: Optional[str] = Field(description="API stage")
+    stage: Optional[str] = Field(None, description="API stage")
 
     @property
     def cognito_authorization_url(self) -> AnyHttpUrl:
@@ -47,22 +48,11 @@ def cognito_token_url(self) -> AnyHttpUrl:
         """Cognito user pool token and refresh url"""
         return f"{self.cognito_domain}/oauth2/token"
 
-    class Config(AwsSsmSourceConfig):
-        env_file = ".env"
-
-    @classmethod
-    def from_ssm(cls, stack: str):
-        return cls(_secrets_dir=f"/{stack}")
+    model_config = ConfigDict(
+        env_file=".env", env_file_encoding="utf-8", extra="ignore"
+    )
 
 
-settings = (
-    Settings()
-    if os.environ.get("NO_PYDANTIC_SSM_SETTINGS")
-    else Settings.from_ssm(
-        stack=os.environ.get(
-            "STACK", f"veda-stac-ingestion-system-{os.environ.get('STAGE', getuser())}"
-        ),
-    )
-)
+settings = Settings()
 
 auth = VedaAuth(settings)

ingest_api/runtime/src/schema_helpers.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import List, Union
 
-from pydantic import BaseModel, root_validator
+from pydantic import BaseModel, model_validator
 from stac_pydantic.collection import Extent, TimeInterval
 
 # Smaller utility models to support the larger models in schemas.py
@@ -24,7 +24,7 @@ class BboxExtent(BaseModel):
     xmax: float
     ymax: float
 
-    @root_validator
+    @model_validator(mode="before")
     def check_extent(cls, v):
         # mins must be below maxes
         if v["xmin"] >= v["xmax"] or v["ymin"] >= v["ymax"]:
@@ -40,10 +40,10 @@ def check_extent(cls, v):
 
 
 class TemporalExtent(BaseModel):
-    startdate: Union[datetime, None]
-    enddate: Union[datetime, None]
+    startdate: Union[datetime, None] = None
+    enddate: Union[datetime, None] = None
 
-    @root_validator
+    @model_validator(mode="before")
     def check_dates(cls, v):
         if (v["enddate"] is not None) and (v["startdate"] >= v["enddate"]):
             raise ValueError("Invalid extent - startdate must be before enddate")