Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BUGFIX] Ensure that file-backed domain objects are stored in JSON files #10523

Merged
merged 14 commits into from
Oct 21, 2024
7 changes: 1 addition & 6 deletions great_expectations/data_context/store/expectations_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
DatabaseStoreBackend,
)
from great_expectations.data_context.store.store import Store
from great_expectations.data_context.store.tuple_store_backend import TupleStoreBackend
from great_expectations.data_context.types.refs import GXCloudResourceRef
from great_expectations.data_context.types.resource_identifiers import (
ExpectationSuiteIdentifier,
Expand Down Expand Up @@ -80,11 +79,7 @@ def __init__(
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name, store_backend_module_name)

# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Duplicate logic now that default is set

elif issubclass(store_backend_class, DatabaseStoreBackend):
if issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get(
"table_name", "ge_expectations_store"
Expand Down
9 changes: 5 additions & 4 deletions great_expectations/data_context/store/tuple_store_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def __init__( # noqa: PLR0913
self,
filepath_template=None,
filepath_prefix=None,
filepath_suffix=None,
filepath_suffix: str | None = None,
forbidden_substrings=None,
platform_specific_separator=True,
fixed_length_key=False,
Expand All @@ -53,8 +53,9 @@ def __init__( # noqa: PLR0913
self.forbidden_substrings = forbidden_substrings
self.platform_specific_separator = platform_specific_separator

if filepath_template is not None and filepath_suffix is not None:
raise ValueError("filepath_suffix may only be used when filepath_template is None") # noqa: TRY003
if filepath_template and filepath_suffix:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd log an info message here saying both were specified so we are taking template.

# Template takes precedence over suffix if both are provided
filepath_suffix = None

self.filepath_template = filepath_template
if filepath_prefix and len(filepath_prefix) > 0:
Expand Down Expand Up @@ -227,7 +228,7 @@ def __init__( # noqa: PLR0913
base_directory,
filepath_template=None,
filepath_prefix=None,
filepath_suffix=None,
filepath_suffix: str | None = ".json",
forbidden_substrings=None,
platform_specific_separator=True,
root_directory=None,
Expand Down
11 changes: 10 additions & 1 deletion tests/data_context/store/test_store_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ def test_TupleFilesystemStoreBackend_ignores_jupyter_notebook_checkpoints(
== f"""\
{test_dir}/
.ge_store_backend_id
AAA
AAA.json
.ipynb_checkpoints/
foo.json
"""
Expand Down Expand Up @@ -1300,6 +1300,15 @@ def test_TupleS3StoreBackend_list_over_1000_keys(aws_credentials):
assert len(keys) == num_keys_to_add + 1


@pytest.mark.filesystem
def test_file_backed_store_backends_use_json(empty_data_context):
context = empty_data_context
for store in context.stores.values():
backend = store.store_backend
assert isinstance(backend, TupleFilesystemStoreBackend)
assert backend.filepath_suffix == ".json"


@pytest.mark.filesystem
def test_InlineStoreBackend(empty_data_context) -> None:
inline_store_backend: InlineStoreBackend = InlineStoreBackend(
Expand Down
Loading