Merge branch 'develop' into kml/DOC-1001/expectationSelection

.github/workflows/ci.yml

@@ -306,7 +306,7 @@ jobs:
           # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io
           # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github)
           # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS)
-          args: "--exclude='http.*' 'docs/docusaurus/build/**/*.html'"
+          args: "--exclude='http.*' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'"
 
   docs-tests:
     runs-on: ubuntu-latest

docs/adr/0005-public-api-docstrings.md

@@ -0,0 +1,19 @@
+# 5. Docstrings for public API functions/classes 
+
+Date: 2024-12-19
+
+## Status
+
+Accepted
+
+## Context
+
+By marking an object as part of the public API, it automatically renders in our docs site. In order to provide a clear and informative experience for our end users, we should make sure we always include docstrings for public functions/classes. These docstrings should adhere to a consistent format such that they can be rendered in the same manner by our public API infrastructure. 
+
+## Decision
+
+All objects decorated with `@public_api` will have docstrings. In order to be considered public, they must meet this criteria.
+
+## Consequences
+
+Marginal increase in developer burden but worthwhile due to a higher level of thought and care around what actually gets marked and rendered as part of our public API.

docs/docusaurus/docs/components/_data.jsx

@@ -1,5 +1,5 @@
 export default {
-  release_version: 'great_expectations, version 1.2.6',
+  release_version: 'great_expectations, version 1.3.0',
   min_python: '3.9',
   max_python: '3.12'
 }

docs/docusaurus/docs/core/connect_to_data/sql_data/_create_a_data_source/_create_a_data_source.md

@@ -70,7 +70,7 @@ import DatasourceMethodReferenceTable from './_datasource_method_reference_table
       "schema": "my_schema"
    }
 
-   data_source = context.sources.add_snowflake(
+   data_source = context.data_sources.add_snowflake(
       name=datasource_name,
       connection_string=connection_details,
       kwargs={"connect_args": connect_args}

docs/docusaurus/docs/oss/changelog.md

@@ -14,6 +14,39 @@ When we deprecate our public APIs, we will
 
 Before we completely remove the functionality in a new major release, there will be at least one minor release that contains the deprecation so that you can smoothly transition to the new API.
 
+### 1.3.0
+* [BUGFIX] Handle expectation description from cloud ([#10768](https://github.com/great-expectations/great_expectations/pull/10768))
+* [BUGFIX] Renderer for UnexpectedRowsExpectation ([#10758](https://github.com/great-expectations/great_expectations/pull/10758))
+* [BUGFIX] Fix format of UnexpectedRowsExpectation observed_value to be int ([#10777](https://github.com/great-expectations/great_expectations/pull/10777))
+* [BUGFIX] Metric table.column_type should properly evaluate for snowflake ([#10776](https://github.com/great-expectations/great_expectations/pull/10776))
+* [BUGFIX] Fix expectation description rendering in DataDocs ([#10789](https://github.com/great-expectations/great_expectations/pull/10789))
+* [BUGFIX] `Databricks` Fix Type Translation - `ExpectColumnValuesToBeInTypeList` and `ExpectColumnValuesToBeInType` ([#10791](https://github.com/great-expectations/great_expectations/pull/10791))
+* [BUGFIX] Metric `table.column_type` should properly evaluate for Postgres ([#10793](https://github.com/great-expectations/great_expectations/pull/10793))
+* [DOCS] Clarify Connect GX Cloud landing page ([#10761](https://github.com/great-expectations/great_expectations/pull/10761))
+* [DOCS] Remove `print(validation_results.result_url)` as it isn't supported ([#10760](https://github.com/great-expectations/great_expectations/pull/10760))
+* [DOCS] SUJ-E ([#10665](https://github.com/great-expectations/great_expectations/pull/10665))
+* [DOCS] Fix underline in links on inline code ([#10783](https://github.com/great-expectations/great_expectations/pull/10783))
+* [DOCS] Remove unnecessary escape character in Expectation for Gallery ([#10780](https://github.com/great-expectations/great_expectations/pull/10780))
+* [DOCS] Custom Actions ([#10772](https://github.com/great-expectations/great_expectations/pull/10772))
+* [DOCS] Schema change detection ([#10755](https://github.com/great-expectations/great_expectations/pull/10755))
+* [DOCS] Update public_api to include `ValidationAction` components ([#10752](https://github.com/great-expectations/great_expectations/pull/10752))
+* [MAINTENANCE] Change ci pipeline to pull_request_target with a permissions check to allow CI on forks ([#10467](https://github.com/great-expectations/great_expectations/pull/10467))
+* [MAINTENANCE] Revert "[MAINTENANCE] Change ci pipeline to pull_request_target with a permissions check to allow CI on forks" ([#10773](https://github.com/great-expectations/great_expectations/pull/10773))
+* [MAINTENANCE] Minor code clean up and refactor around column type exp… ([#10764](https://github.com/great-expectations/great_expectations/pull/10764))
+* [MAINTENANCE] Diagram on how multi-datasource test setup works ([#10766](https://github.com/great-expectations/great_expectations/pull/10766))
+* [MAINTENANCE] Add `UnexpectedRowsExpectation` observed value renderer ([#10779](https://github.com/great-expectations/great_expectations/pull/10779))
+* [MAINTENANCE] Remove `docs_link_checker.py` ([#10781](https://github.com/great-expectations/great_expectations/pull/10781))
+* [MAINTENANCE] Add ENUM for Data Quality Issues, Update Core Expectations Docstrings/Schemas ([#10759](https://github.com/great-expectations/great_expectations/pull/10759))
+* [MAINTENANCE] Adding more test cases for snowflake types ([#10786](https://github.com/great-expectations/great_expectations/pull/10786))
+* [MAINTENANCE] Update `tasks.py` to remove reference to `isort` ([#10782](https://github.com/great-expectations/great_expectations/pull/10782))
+* [MAINTENANCE] Remove GX Cloud onboarding script ([#10785](https://github.com/great-expectations/great_expectations/pull/10785))
+* [MAINTENANCE] Check filepath existence when evaluating public API report ([#10754](https://github.com/great-expectations/great_expectations/pull/10754))
+* [MAINTENANCE] Add EventBridge Scheduler service to `cloud-tests` ([#10774](https://github.com/great-expectations/great_expectations/pull/10774))
+* [MAINTENANCE] Deprecate `DataContext.add_or_update_datasource` ([#10784](https://github.com/great-expectations/great_expectations/pull/10784))
+* [MAINTENANCE] Allow `CheckpointResult` and `ActionContext` to be importable from top-level checkpoint module ([#10788](https://github.com/great-expectations/great_expectations/pull/10788))
+* [MAINTENANCE] Clean up `cloud-tests` environment variables ([#10792](https://github.com/great-expectations/great_expectations/pull/10792))
+* [MAINTENANCE] Adding `databricks` compatibility types ([#10787](https://github.com/great-expectations/great_expectations/pull/10787))
+
 ### 1.2.6
 * [BUGFIX] Enable custom actions in V1 ([#10743](https://github.com/great-expectations/great_expectations/pull/10743))
 * [BUGFIX] Fix like pattern expectations to work without optional SQL deps ([#10745](https://github.com/great-expectations/great_expectations/pull/10745))

docs/docusaurus/docs/reference/learn/data_quality_use_cases/schema_resources/schema_strict_and_relaxed.py

@@ -63,7 +63,7 @@
 )
 
 strict_suite.add_expectation(
-    gxe.ExpectColumnValuesToBeOfType(column="transfer_amount", type_="DOUBLE_PRECISION")
+    gxe.ExpectColumnValuesToBeOfType(column="transfer_amount", type_="DOUBLE PRECISION")
 )
 
 strict_results = batch.validate(strict_suite)
@@ -85,7 +85,7 @@
 
 relaxed_suite.add_expectation(
     gxe.ExpectColumnValuesToBeInTypeList(
-        column="transfer_amount", type_list=["DOUBLE_PRECISION", "STRING"]
+        column="transfer_amount", type_list=["DOUBLE PRECISION", "STRING"]
     )
 )
 

docs/docusaurus/docs/reference/learn/migration_guide.md

@@ -673,7 +673,7 @@ A few configurations are **NO LONGER SUPPORTED**:
     ```python title="Python"
     # Create datasource
     connection_string = "snowflake://<user_login_name>:<password>@<account_identifier>/<database_name>/<schema_name>?warehouse=<warehouse_name>&role=<role_name>"
-    snowflake_ds = context.sources.add_snowflake(name="snowflake_ds", connection_string=connection_string)
+    snowflake_ds = context.data_sources.add_snowflake(name="snowflake_ds", connection_string=connection_string)
 
     # Create table asset and batch definitions
     table_asset = snowflake_ds.add_table_asset(name="taxi_data", table_name="TAXI_DATA_ALL_SAMPLES")

docs/docusaurus/docusaurus.config.js

@@ -52,15 +52,15 @@ module.exports = {
     gxCard: {
       title: 'What is GX Cloud?',
       description:
-        'GX Cloud is a fully-managed SaaS solution that simplifies deployment, scaling, and collaboration and lets you focus on data validation.',
+        'GX Cloud is a fully-managed SaaS solution that simplifies deployment, scaling, and collaboration—so you can focus on data validation.',
       buttons: {
         primary: {
-          label: 'Try GX Cloud',
-          href: 'https://greatexpectations.io/cloud',
+          label: 'Request a demo',
+          href: 'https://greatexpectations.io/demo',
         },
         secondary: {
-          label: 'GX Cloud overview',
-          href: '/docs/cloud/overview/gx_cloud_overview',
+          label: 'Why GX Cloud?',
+          href: 'https://greatexpectations.io/why-gx-cloud',
         },
       },
     },
@@ -299,7 +299,7 @@ module.exports = {
           lastVersion: 'current',
           versions: {
             current: {
-              label: '1.2.6',
+              label: '1.3.0',
             },
             ['0.18']: {
               label: '0.18.21',

docs/docusaurus/src/css/alerts.scss

@@ -31,6 +31,10 @@
   border-left-width: var(--ifm-alert-border-left-width);
   box-shadow: none;
 
+  [class^="admonitionHeading"] {
+    text-transform: none;
+  }
+
   div:first-child {
     font-size: var(--p-font-size);
 

great_expectations/compatibility/postgresql.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from great_expectations.compatibility.not_imported import NotImported
+
+POSTGRESQL_NOT_IMPORTED = NotImported(
+    "postgresql connection components are not installed, please 'pip install psycopg2'"
+)
+
+try:
+    import psycopg2  # noqa: F401
+    import sqlalchemy.dialects.postgresql as postgresqltypes
+except ImportError:
+    postgresqltypes = POSTGRESQL_NOT_IMPORTED  # type: ignore[assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import TEXT
+except (ImportError, AttributeError):
+    TEXT = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import CHAR
+except (ImportError, AttributeError):
+    CHAR = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import INTEGER
+except (ImportError, AttributeError):
+    INTEGER = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import SMALLINT
+except (ImportError, AttributeError):
+    SMALLINT = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import BIGINT
+except (ImportError, AttributeError):
+    BIGINT = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import TIMESTAMP
+except (ImportError, AttributeError):
+    TIMESTAMP = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import DATE
+except (ImportError, AttributeError):
+    DATE = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION
+except (ImportError, AttributeError):
+    DOUBLE_PRECISION = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import BOOLEAN
+except (ImportError, AttributeError):
+    BOOLEAN = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+try:
+    from sqlalchemy.dialects.postgresql import NUMERIC
+except (ImportError, AttributeError):
+    NUMERIC = POSTGRESQL_NOT_IMPORTED  # type: ignore[misc, assignment]
+
+
+class POSTGRESQL_TYPES:
+    """Namespace for PostgreSQL dialect types."""
+
+    TEXT = TEXT
+    CHAR = CHAR
+    INTEGER = INTEGER
+    SMALLINT = SMALLINT
+    BIGINT = BIGINT
+    TIMESTAMP = TIMESTAMP
+    DATE = DATE
+    DOUBLE_PRECISION = DOUBLE_PRECISION
+    BOOLEAN = BOOLEAN
+    NUMERIC = NUMERIC

great_expectations/core/factory/suite_factory.py

@@ -33,7 +33,7 @@ def _include_rendered_content(self) -> bool:
     def add(self, suite: ExpectationSuite) -> ExpectationSuite:
         """Add an ExpectationSuite to the collection.
 
-        Parameters:
+        Args:
             suite: ExpectationSuite to add
 
         Raises:
@@ -62,7 +62,7 @@ def add(self, suite: ExpectationSuite) -> ExpectationSuite:
     def delete(self, name: str) -> None:
         """Delete an ExpectationSuite from the collection.
 
-        Parameters:
+        Args:
             name: The name of the ExpectationSuite to delete
 
         Raises:
@@ -89,7 +89,7 @@ def delete(self, name: str) -> None:
     def get(self, name: str) -> ExpectationSuite:
         """Get an ExpectationSuite from the collection by name.
 
-        Parameters:
+        Args:
             name: Name of ExpectationSuite to get
 
         Raises:
@@ -125,3 +125,33 @@ def all(self) -> Iterable[ExpectationSuite]:
                 self._store.submit_all_deserialization_event(e)
                 raise
         return deserializable_suites
+
+    @public_api
+    def add_or_update(self, suite: ExpectationSuite) -> ExpectationSuite:
+        """Add or update an ExpectationSuite by name.
+
+        If an ExpectationSuite with the same name exists, overwrite it, otherwise
+        create a new ExpectationSuite. On update, Expectations in the Suite which
+        match a previously existing Expectation maintain a stable ID, and
+        Expectations which have changed receive a new ID.
+
+        Args:
+            suite: ExpectationSuite to add or update
+        """
+        try:
+            existing_suite = self.get(name=suite.name)
+        except DataContextError:
+            return self.add(suite=suite)
+
+        # add IDs to expectations that haven't changed
+        existing_expectations = existing_suite.expectations
+        for expectation in suite.expectations:
+            try:
+                index = existing_expectations.index(expectation)
+                expectation.id = existing_expectations[index].id
+            except ValueError:
+                pass  # expectation is new or updated
+
+        suite.id = existing_suite.id
+        suite.save()
+        return suite

great_expectations/data_context/data_context/abstract_data_context.py

@@ -702,7 +702,7 @@ def _validate_add_datasource_args(
         # "type" is only used in FDS so we check for its existence (equivalent for block-style would be "class_name" and "module_name")  # noqa: E501
         if "type" in kwargs:
             raise TypeError(  # noqa: TRY003
-                "Creation of fluent-datasources with individual arguments is not supported and should be done through the `context.sources` API."  # noqa: E501
+                "Creation of fluent-datasources with individual arguments is not supported and should be done through the `context.data_sources` API."  # noqa: E501
             )
 
     def _add_datasource(

great_expectations/datasource/fluent/interfaces.py

@@ -216,7 +216,7 @@ class GxContextWarning(GxDatasourceWarning):
     """
     Warning related to a Datasource with a missing context.
     Usually because the Datasource was created directly rather than using a
-    `context.sources` factory method.
+    `context.data_sources` factory method.
     """
 
 

great_expectations/datasource/fluent/sources.py

@@ -112,7 +112,7 @@ def register_datasource(cls, ds_type: Type[Datasource]) -> None:
         Example
         -------
 
-        An `.add_pandas_filesystem()` pandas_filesystem factory method will be added to `context.sources`.
+        An `.add_pandas_filesystem()` pandas_filesystem factory method will be added to `context.data_sources`.
 
         >>> class PandasFilesystemDatasource(_PandasFilePathDatasource):
         >>>     type: str = 'pandas_filesystem'

great_expectations/deployment_version

@@ -1 +1 @@
-1.2.6
+1.3.0

great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py

@@ -458,7 +458,11 @@ def _validate_pandas(  # noqa: C901, PLR0912
     def _validate_sqlalchemy(self, actual_column_type, expected_types_list, execution_engine):
         if expected_types_list is None:
             success = True
-        elif execution_engine.dialect_name in [GXSqlDialect.SNOWFLAKE, GXSqlDialect.DATABRICKS]:
+        elif execution_engine.dialect_name in [
+            GXSqlDialect.DATABRICKS,
+            GXSqlDialect.POSTGRESQL,
+            GXSqlDialect.SNOWFLAKE,
+        ]:
             success = isinstance(actual_column_type, str) and any(
                 actual_column_type.lower() == expected_type.lower()
                 for expected_type in expected_types_list

great_expectations/expectations/core/expect_column_values_to_be_of_type.py

@@ -412,7 +412,11 @@ def _validate_sqlalchemy(self, actual_column_type, expected_type, execution_engi
 
         if expected_type is None:
             success = True
-        elif execution_engine.dialect_name in [GXSqlDialect.SNOWFLAKE, GXSqlDialect.DATABRICKS]:
+        elif execution_engine.dialect_name in [
+            GXSqlDialect.DATABRICKS,
+            GXSqlDialect.POSTGRESQL,
+            GXSqlDialect.SNOWFLAKE,
+        ]:
             success = (
                 isinstance(actual_column_type, str)
                 and actual_column_type.lower() == expected_type.lower()

great_expectations/expectations/metrics/util.py

@@ -360,7 +360,7 @@ def __getitem__(self, key: Any) -> Any:
         return item
 
 
-def get_sqlalchemy_column_metadata(  # noqa: C901
+def get_sqlalchemy_column_metadata(  # noqa: C901, PLR0912
     execution_engine: SqlAlchemyExecutionEngine,
     table_selectable: sqlalchemy.Select,
     schema_name: Optional[str] = None,
@@ -414,11 +414,18 @@ def get_sqlalchemy_column_metadata(  # noqa: C901
             )
 
         dialect_name = execution_engine.dialect.name
-        if dialect_name in [GXSqlDialect.SNOWFLAKE, GXSqlDialect.DATABRICKS]:
+        if dialect_name in [
+            GXSqlDialect.DATABRICKS,
+            GXSqlDialect.POSTGRESQL,
+            GXSqlDialect.SNOWFLAKE,
+        ]:
             # WARNING: Do not alter columns in place, as they are cached on the inspector
             columns_copy = [column.copy() for column in columns]
             for column in columns_copy:
-                column["type"] = column["type"].compile(dialect=execution_engine.dialect)
+                if column.get("type"):
+                    # When using column_reflection_fallback, we might not be able to
+                    # extract the column type, and only have the column name
+                    column["type"] = column["type"].compile(dialect=execution_engine.dialect)
             if dialect_name == GXSqlDialect.SNOWFLAKE:
                 return [
                     # TODO: SmartColumn should know the dialect and do lookups based on that

great_expectations/util.py

@@ -1164,7 +1164,7 @@ def convert_to_json_serializable(  # noqa: C901, PLR0911, PLR0912
     if isinstance(data, np.float64):
         return float(data)
 
-    if isinstance(data, (datetime.datetime, datetime.date)):
+    if isinstance(data, (datetime.datetime, datetime.date, datetime.time)):
         return data.isoformat()
 
     if isinstance(data, (np.datetime64)):
@@ -1309,7 +1309,7 @@ def ensure_json_serializable(data: Any) -> None:  # noqa: C901, PLR0911, PLR0912
         _ = [ensure_json_serializable(x) for x in data.tolist()]  # type: ignore[func-returns-value]
         return
 
-    if isinstance(data, (datetime.datetime, datetime.date)):
+    if isinstance(data, (datetime.datetime, datetime.date, datetime.time)):
         return
 
     if isinstance(data, pathlib.PurePath):