Follow up to python 3.8 upgrade. ruff and mypy updates (#591)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Great Expectations <[email protected]>

great_expectations_cloud/agent/actions/agent_action.py

@@ -2,7 +2,8 @@
 
 import datetime
 from abc import abstractmethod
-from typing import TYPE_CHECKING, Generic, Optional, Sequence, TypeVar
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Generic, Optional, TypeVar
 from uuid import UUID
 
 from pydantic.v1 import BaseModel

great_expectations_cloud/agent/agent.py

@@ -10,7 +10,7 @@
 from concurrent.futures.thread import ThreadPoolExecutor
 from functools import partial
 from importlib.metadata import version as metadata_version
-from typing import TYPE_CHECKING, Any, Callable, Dict, Final, Literal
+from typing import TYPE_CHECKING, Any, Callable, Final, Literal
 from urllib.parse import urljoin
 from uuid import UUID
 
@@ -78,7 +78,7 @@
 LOGGER: Final[logging.Logger] = logging.getLogger(__name__)
 # TODO Set in log dict
 LOGGER.setLevel(logging.DEBUG)
-HandlerMap = Dict[str, OnMessageCallback]
+HandlerMap = dict[str, OnMessageCallback]
 
 
 class GXAgentConfig(AgentBaseExtraForbid):
@@ -112,7 +112,7 @@ def orjson_loads(v: bytes | bytearray | memoryview | str) -> Any:
 
 
 class Payload(AgentBaseExtraForbid):
-    data: Dict[str, Any]  # noqa: UP006  # Python 3.8 requires Dict instead of dict
+    data: dict[str, Any]
 
     class Config:
         extra = "forbid"

great_expectations_cloud/agent/message_service/subscriber.py

@@ -2,9 +2,10 @@
 
 import asyncio
 import time
+from collections.abc import Coroutine
 from dataclasses import dataclass
 from functools import partial
-from typing import TYPE_CHECKING, Callable, Coroutine, Protocol
+from typing import TYPE_CHECKING, Callable, Protocol
 
 from pika.exceptions import (
     AMQPError,

great_expectations_cloud/agent/models.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import uuid
-from typing import Any, Dict, Literal, Optional, Sequence, Set, Union
+from collections.abc import Sequence
+from typing import Annotated, Any, Literal, Optional, Union
 from uuid import UUID
 
 from great_expectations.experimental.metric_repository.metrics import MetricTypes
 from pydantic.v1 import BaseModel, Extra, Field
-from typing_extensions import Annotated
 
 from great_expectations_cloud.agent.exceptions import GXCoreError
 
@@ -53,27 +53,27 @@ class RunMissingnessDataAssistantEvent(RunDataAssistantEvent):
 
 class RunCheckpointEvent(EventBase):
     type: Literal["run_checkpoint_request"] = "run_checkpoint_request"
-    datasource_names_to_asset_names: Dict[str, Set[str]]
+    datasource_names_to_asset_names: dict[str, set[str]]
     checkpoint_id: uuid.UUID
-    splitter_options: Optional[Dict[str, Any]] = None
+    splitter_options: Optional[dict[str, Any]] = None
     # TODO: Remove optional once fully migrated to greatexpectations v1
     checkpoint_name: Optional[str] = None
 
 
 class RunScheduledCheckpointEvent(ScheduledEventBase):
     type: Literal["run_scheduled_checkpoint.received"] = "run_scheduled_checkpoint.received"
-    datasource_names_to_asset_names: Dict[str, Set[str]]
+    datasource_names_to_asset_names: dict[str, set[str]]
     checkpoint_id: uuid.UUID
-    splitter_options: Optional[Dict[str, Any]] = None
+    splitter_options: Optional[dict[str, Any]] = None
     # TODO: Remove optional once fully migrated to greatexpectations v1
     checkpoint_name: Optional[str] = None
 
 
 class RunWindowCheckpointEvent(EventBase):
     type: Literal["run_window_checkpoint.received"] = "run_window_checkpoint.received"
-    datasource_names_to_asset_names: Dict[str, Set[str]]
+    datasource_names_to_asset_names: dict[str, set[str]]
     checkpoint_id: uuid.UUID
-    splitter_options: Optional[Dict[str, Any]] = None
+    splitter_options: Optional[dict[str, Any]] = None
     # TODO: Remove optional once fully migrated to greatexpectations v1
     checkpoint_name: Optional[str] = None
 
@@ -117,7 +117,7 @@ class RunRdAgentEvent(EventBase):
     datasource_name: str
     data_asset_name: str
     batch_definition_name: str
-    batch_parameters: Optional[Dict[str, Any]] = None
+    batch_parameters: Optional[dict[str, Any]] = None
 
 
 class UnknownEvent(AgentBaseExtraForbid):
@@ -158,7 +158,7 @@ class JobCompleted(AgentBaseExtraForbid):
     created_resources: Sequence[CreatedResource] = []
     error_stack_trace: Union[str, None] = None
     error_code: Union[str, None] = None
-    error_params: Union[Dict[str, str], None] = None
+    error_params: Union[dict[str, str], None] = None
     processed_by: Union[Literal["agent", "runner"], None] = None
 
 
@@ -174,8 +174,8 @@ class CreateScheduledJobAndSetJobStarted(AgentBaseExtraForbid):
     correlation_id: UUID
     schedule_id: UUID
     checkpoint_id: UUID
-    datasource_names_to_asset_names: Dict[str, Set[str]]
-    splitter_options: Optional[Dict[str, Any]] = None
+    datasource_names_to_asset_names: dict[str, set[str]]
+    splitter_options: Optional[dict[str, Any]] = None
     # TODO: Remove optional once fully migrated to greatexpectations v1
     checkpoint_name: Optional[str] = None
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "great_expectations_cloud"
-version = "20241212.0"
+version = "20241212.1.dev0"
 description = "Great Expectations Cloud"
 authors = ["The Great Expectations Team <[email protected]>"]
 repository = "https://github.com/great-expectations/cloud"
@@ -80,7 +80,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.9"
 plugins = ["pydantic.mypy"]
 files = ["great_expectations_cloud/", "tests/", "tasks.py"]
 strict = true
@@ -102,11 +102,11 @@ module = "tests.*"
 disable_error_code = ['no-untyped-def']
 
 [tool.black]
-target-version = ["py38", "py39", "py310", "py311"]
+target-version = ["py39", "py310", "py311"]
 line-length = 100
 
 [tool.ruff]
-target-version = "py38"
+target-version = "py39"
 # TODO: change this back to black default of 88 if desired
 line-length = 100
 lint.select = [

tasks.py

@@ -5,8 +5,9 @@
 import logging
 import os
 import pathlib
+from collections.abc import MutableMapping
 from pprint import pformat as pf
-from typing import TYPE_CHECKING, Any, Final, Literal, MutableMapping
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import invoke
 import requests

tests/agent/conftest.py

@@ -4,7 +4,8 @@
 import time
 import uuid
 from collections import deque
-from typing import TYPE_CHECKING, Any, Iterable, Literal, NamedTuple, TypedDict
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypedDict
 
 import pytest
 from great_expectations import (

tests/integration/actions/conftest.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import uuid
-from typing import TYPE_CHECKING, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
 
 import great_expectations as gx
 import great_expectations.exceptions.exceptions as gx_exceptions

tests/integration/actions/test_checkpoint_action.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import uuid
-from typing import TYPE_CHECKING, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
 
 import pytest
 

tests/integration/actions/test_run_scheduled_checkpoint.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import uuid
-from typing import TYPE_CHECKING, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
 
 import pytest
 

tests/test_project.py

@@ -5,9 +5,10 @@
 import re
 import sys
 import warnings
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from pprint import pformat as pf
-from typing import TYPE_CHECKING, Any, Final, Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Final
 
 import pytest
 import tomlkit