Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BUGFIX] Ensure that file-backed domain objects are stored in JSON files #10523

Merged
merged 14 commits into from
Oct 21, 2024
17 changes: 17 additions & 0 deletions great_expectations/data_context/store/checkpoint_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from great_expectations.core.data_context_key import DataContextKey, StringKey
from great_expectations.data_context.cloud_constants import GXCloudRESTResource
from great_expectations.data_context.store.store import Store
from great_expectations.data_context.store.tuple_store_backend import TupleStoreBackend
from great_expectations.data_context.types.base import DataContextConfigDefaults
from great_expectations.data_context.types.resource_identifiers import (
GXCloudIdentifier,
Expand All @@ -25,6 +26,22 @@
class CheckpointStore(Store):
_key_class = StringKey

def __init__(
self,
store_backend: dict | None = None,
runtime_environment: dict | None = None,
store_name: str = "no_store_name",
) -> None:
store_backend_class = self._determine_store_backend_class(store_backend)
if store_backend and issubclass(store_backend_class, TupleStoreBackend):
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")

super().__init__(
store_backend=store_backend,
runtime_environment=runtime_environment,
store_name=store_name,
)

def get_key(self, name: str, id: str | None = None) -> GXCloudIdentifier | StringKey:
"""Given a name and optional ID, build the correct key for use in the CheckpointStore."""
if self.cloud_mode:
Expand Down
24 changes: 10 additions & 14 deletions great_expectations/data_context/store/expectations_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@
ExpectationSuiteIdentifier,
GXCloudIdentifier,
)
from great_expectations.data_context.util import load_class
from great_expectations.util import (
filter_properties_dict,
verify_dynamic_loading_support,
)

if TYPE_CHECKING:
from great_expectations.data_context.data_context.abstract_data_context import (
AbstractDataContext,
)
from great_expectations.expectations.expectation import Expectation

_TExpectation = TypeVar("_TExpectation", bound=Expectation)
Expand Down Expand Up @@ -65,22 +66,17 @@ class ExpectationsStore(Store):

def __init__(
self,
store_backend=None,
runtime_environment=None,
store_name=None,
data_context=None,
store_backend: dict | None = None,
runtime_environment: dict | None = None,
store_name: str = "no_store_name",
data_context: AbstractDataContext | None = None,
) -> None:
self._expectationSuiteSchema = ExpectationSuiteSchema()
self._data_context = data_context
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store"
)
store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name, store_backend_module_name)

# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
store_backend_class = self._determine_store_backend_class(store_backend)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if store_backend:
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")
Expand Down
11 changes: 11 additions & 0 deletions great_expectations/data_context/store/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
DataContextError,
StoreBackendError,
)
from great_expectations.util import load_class, verify_dynamic_loading_support

if TYPE_CHECKING:
# min version of typing_extension missing `NotRequired`, so it can't be imported at runtime
Expand Down Expand Up @@ -111,6 +112,16 @@ def __init__(
)
self._use_fixed_length_key = self._store_backend.fixed_length_key

@staticmethod
def _determine_store_backend_class(store_backend: dict | None) -> type:
store_backend = store_backend or {}
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store"
)
store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
return load_class(store_backend_class_name, store_backend_module_name)

@classmethod
def gx_cloud_response_json_to_object_dict(cls, response_json: Dict) -> Dict:
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from great_expectations.core.data_context_key import DataContextKey, StringKey
from great_expectations.data_context.cloud_constants import GXCloudRESTResource
from great_expectations.data_context.store.store import Store
from great_expectations.data_context.store.tuple_store_backend import TupleStoreBackend
from great_expectations.data_context.types.resource_identifiers import (
GXCloudIdentifier,
)
Expand All @@ -19,6 +20,22 @@
class ValidationDefinitionStore(Store):
_key_class = StringKey

def __init__(
self,
store_backend: dict | None = None,
runtime_environment: dict | None = None,
store_name: str = "no_store_name",
) -> None:
store_backend_class = self._determine_store_backend_class(store_backend)
if store_backend and issubclass(store_backend_class, TupleStoreBackend):
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")

super().__init__(
store_backend=store_backend,
runtime_environment=runtime_environment,
store_name=store_name,
)

def get_key(self, name: str, id: str | None = None) -> GXCloudIdentifier | StringKey:
"""Given a name and optional ID, build the correct key for use in the ValidationDefinitionStore.""" # noqa: E501
if self.cloud_mode:
Expand Down
9 changes: 9 additions & 0 deletions tests/data_context/store/test_store_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -1545,3 +1545,12 @@ def test_store_backend_path_special_character_escape():
escaped_path
== "/validations/default/pandas_data_asset/20230315T205136.109084Z/default_pandas_datasource-%23ephemeral_pandas_asset.html" # noqa: E501
)


@pytest.mark.filesystem
def test_file_backed_store_backends_use_json(empty_data_context):
context = empty_data_context
for store in context.stores.values():
backend = store.store_backend
assert isinstance(backend, TupleFilesystemStoreBackend)
assert backend.filepath_suffix == ".json"
Loading