Merge branch 'main' into bug/adap-435-begin-end-scripting

.changes/unreleased/Features-20240109-165520.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Update base adapter references as part of decoupling migration
+time: 2024-01-09T16:55:20.859657-06:00
+custom:
+  Author: McKnight-42
+  Issue: "882"

.changes/unreleased/Under the Hood-20240117-112026.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Migrate to dbt-common and dbt-adapters package
+time: 2024-01-17T11:20:26.713584-06:00
+custom:
+  Author: McKnight-42
+  Issue: "890"

dbt/adapters/snowflake/column.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from dbt.adapters.base.column import Column
-from dbt.exceptions import DbtRuntimeError
+from dbt_common.exceptions import DbtRuntimeError
 
 
 @dataclass

dbt/adapters/snowflake/connections.py

@@ -11,7 +11,7 @@
 from typing import Optional, Tuple, Union, Any, List
 
 import agate
-import dbt.clients.agate_helper
+from dbt_common.clients.agate_helper import empty_table
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
@@ -32,20 +32,19 @@
     BindUploadError,
 )
 
-from dbt.exceptions import (
+from dbt_common.exceptions import (
     DbtInternalError,
     DbtRuntimeError,
-    FailedToConnectError,
-    DbtDatabaseError,
-    DbtProfileError,
+    DbtConfigError,
 )
-from dbt.adapters.base import Credentials  # type: ignore
-from dbt.contracts.connection import AdapterResponse, Connection
+from dbt_common.exceptions import DbtDatabaseError
+from dbt.adapters.exceptions.connection import FailedToConnectError
+from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials
 from dbt.adapters.sql import SQLConnectionManager  # type: ignore
-from dbt.events import AdapterLogger  # type: ignore
-from dbt.events.functions import warn_or_error
-from dbt.events.types import AdapterEventWarning
-from dbt.ui import line_wrap_message, warning_tag
+from dbt.adapters.events.logging import AdapterLogger  # type: ignore
+from dbt_common.events.functions import warn_or_error
+from dbt.adapters.events.types import AdapterEventWarning
+from dbt_common.ui import line_wrap_message, warning_tag
 
 
 logger = AdapterLogger("Snowflake")
@@ -247,7 +246,7 @@ def _get_access_token(self) -> str:
     def _get_private_key(self):
         """Get Snowflake private key by path, from a Base64 encoded DER bytestring or None."""
         if self.private_key and self.private_key_path:
-            raise DbtProfileError("Cannot specify both `private_key`  and `private_key_path`")
+            raise DbtConfigError("Cannot specify both `private_key`  and `private_key_path`")
 
         if self.private_key_passphrase:
             encoded_passphrase = self.private_key_passphrase.encode()
@@ -476,7 +475,7 @@ def execute(
         if fetch:
             table = self.get_result_from_cursor(cursor, limit)
         else:
-            table = dbt.clients.agate_helper.empty_table()
+            table = empty_table()
         return response, table
 
     def add_standard_query(self, sql: str, **kwargs) -> Tuple[Connection, Any]:

dbt/adapters/snowflake/impl.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Mapping, Any, Optional, List, Union, Dict
+from typing import Mapping, Any, Optional, List, Union, Dict, FrozenSet, Tuple
 
 import agate
 
@@ -15,10 +15,9 @@
 from dbt.adapters.snowflake import SnowflakeConnectionManager
 from dbt.adapters.snowflake import SnowflakeRelation
 from dbt.adapters.snowflake import SnowflakeColumn
-from dbt.contracts.graph.manifest import Manifest
-from dbt.contracts.graph.nodes import ConstraintType
-from dbt.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError
-from dbt.utils import filter_null_values
+from dbt_common.contracts.constraints import ConstraintType
+from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError
+from dbt_common.utils import filter_null_values
 
 
 @dataclass
@@ -62,11 +61,13 @@ def date_function(cls):
         return "CURRENT_TIMESTAMP()"
 
     @classmethod
-    def _catalog_filter_table(cls, table: agate.Table, manifest: Manifest) -> agate.Table:
+    def _catalog_filter_table(
+        cls, table: agate.Table, used_schemas: FrozenSet[Tuple[str, str]]
+    ) -> agate.Table:
         # On snowflake, users can set QUOTED_IDENTIFIERS_IGNORE_CASE, so force
         # the column names to their lowercased forms.
         lowered = table.rename(column_names=[c.lower() for c in table.column_names])
-        return super()._catalog_filter_table(lowered, manifest)
+        return super()._catalog_filter_table(lowered, used_schemas)
 
     def _make_match_kwargs(self, database, schema, identifier):
         quoting = self.config.quoting

dbt/adapters/snowflake/relation.py

@@ -3,8 +3,8 @@
 
 from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.relation_configs import RelationConfigChangeAction, RelationResults
-from dbt.context.providers import RuntimeConfigObject
-from dbt.utils import classproperty
+from dbt.adapters.contracts.relation import RelationConfig
+from dbt.adapters.utils import classproperty
 
 from dbt.adapters.snowflake.relation_configs import (
     SnowflakeDynamicTableConfig,
@@ -43,12 +43,12 @@ def get_relation_type(cls) -> Type[SnowflakeRelationType]:
 
     @classmethod
     def dynamic_table_config_changeset(
-        cls, relation_results: RelationResults, runtime_config: RuntimeConfigObject
+        cls, relation_results: RelationResults, relation_config: RelationConfig
     ) -> Optional[SnowflakeDynamicTableConfigChangeset]:
         existing_dynamic_table = SnowflakeDynamicTableConfig.from_relation_results(
             relation_results
         )
-        new_dynamic_table = SnowflakeDynamicTableConfig.from_model_node(runtime_config.model)
+        new_dynamic_table = SnowflakeDynamicTableConfig.from_relation_config(relation_config)
 
         config_change_collection = SnowflakeDynamicTableConfigChangeset()
 

dbt/adapters/snowflake/relation_configs/base.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
 from typing import Any, Dict, Optional
-
 import agate
 from dbt.adapters.base.relation import Policy
 from dbt.adapters.relation_configs import (
     RelationConfigBase,
     RelationResults,
 )
-from dbt.contracts.graph.nodes import ModelNode
-from dbt.contracts.relation import ComponentName
+
+from dbt.adapters.contracts.relation import ComponentName, RelationConfig
 
 from dbt.adapters.snowflake.relation_configs.policies import (
     SnowflakeIncludePolicy,
@@ -31,22 +30,22 @@ def quote_policy(cls) -> Policy:
         return SnowflakeQuotePolicy()
 
     @classmethod
-    def from_model_node(cls, model_node: ModelNode):
-        relation_config = cls.parse_model_node(model_node)
-        relation = cls.from_dict(relation_config)
+    def from_relation_config(cls, relation_config: RelationConfig):
+        relation_config_dict = cls.parse_relation_config(relation_config)
+        relation = cls.from_dict(relation_config_dict)
         return relation
 
     @classmethod
-    def parse_model_node(cls, model_node: ModelNode) -> Dict[str, Any]:
+    def parse_relation_config(cls, relation_config: RelationConfig) -> Dict:
         raise NotImplementedError(
-            "`parse_model_node()` needs to be implemented on this RelationConfigBase instance"
+            "`parse_relation_config()` needs to be implemented on this RelationConfigBase instance"
         )
 
     @classmethod
     def from_relation_results(cls, relation_results: RelationResults):
         relation_config = cls.parse_relation_results(relation_results)
         relation = cls.from_dict(relation_config)
-        return relation
+        return relation  # type: ignore
 
     @classmethod
     def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]:

dbt/adapters/snowflake/relation_configs/dynamic_table.py

@@ -1,10 +1,10 @@
 from dataclasses import dataclass
-from typing import Optional
+from typing import Optional, Dict, Any
 
 import agate
 from dbt.adapters.relation_configs import RelationConfigChange, RelationResults
-from dbt.contracts.graph.nodes import ModelNode
-from dbt.contracts.relation import ComponentName
+from dbt.adapters.contracts.relation import RelationConfig
+from dbt.adapters.contracts.relation import ComponentName
 
 from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase
 
@@ -48,20 +48,20 @@ def from_dict(cls, config_dict) -> "SnowflakeDynamicTableConfig":
         return dynamic_table
 
     @classmethod
-    def parse_model_node(cls, model_node: ModelNode) -> dict:
+    def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]:
         config_dict = {
-            "name": model_node.identifier,
-            "schema_name": model_node.schema,
-            "database_name": model_node.database,
-            "query": model_node.compiled_code,
-            "target_lag": model_node.config.extra.get("target_lag"),
-            "snowflake_warehouse": model_node.config.extra.get("snowflake_warehouse"),
+            "name": relation_config.identifier,
+            "schema_name": relation_config.schema,
+            "database_name": relation_config.database,
+            "query": relation_config.compiled_code,  # type: ignore
+            "target_lag": relation_config.config.extra.get("target_lag"),  # type: ignore
+            "snowflake_warehouse": relation_config.config.extra.get("snowflake_warehouse"),  # type: ignore
         }
 
         return config_dict
 
     @classmethod
-    def parse_relation_results(cls, relation_results: RelationResults) -> dict:
+    def parse_relation_results(cls, relation_results: RelationResults) -> Dict:
         dynamic_table: agate.Row = relation_results["dynamic_table"].rows[0]
 
         config_dict = {

dbt/adapters/snowflake/relation_configs/policies.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from dbt.adapters.base.relation import Policy
-from dbt.dataclass_schema import StrEnum
+from dbt_common.dataclass_schema import StrEnum
 
 
 class SnowflakeRelationType(StrEnum):

dbt/include/snowflake/macros/adapters.sql

@@ -73,7 +73,7 @@
   {% for _ in range(0, max_iter) %}
 
       {%- set paginated_sql -%}
-         show terse objects in {{ schema_relation }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}'
+         show terse objects in {{ schema_relation.database }}.{{ schema_relation.schema }} limit {{ max_results_per_iter }} from '{{ watermark.table_name }}'
       {%- endset -%}
 
       {%- set paginated_result = run_query(paginated_sql) %}
@@ -96,7 +96,7 @@
 
       {%- if loop.index == max_iter -%}
         {%- set msg -%}
-           dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation }}.
+           dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation.database }}.{{ schema_relation.schema }}.
            Your schema exceeds this limit. Please contact support@getdbt.com for troubleshooting tips,
            or review and reduce the number of objects contained.
         {%- endset -%}
@@ -124,7 +124,7 @@
   {%- set max_total_results = max_results_per_iter * max_iter -%}
 
   {%- set sql -%}
-    show terse objects in {{ schema_relation }} limit {{ max_results_per_iter }}
+    show terse objects in {{ schema_relation.database }}.{{ schema_relation.schema }} limit {{ max_results_per_iter }}
   {%- endset -%}
 
   {%- set result = run_query(sql) -%}

dbt/include/snowflake/macros/materializations/dynamic_table.sql

@@ -92,6 +92,6 @@
 
 {% macro snowflake__get_dynamic_table_configuration_changes(existing_relation, new_config) -%}
     {% set _existing_dynamic_table = snowflake__describe_dynamic_table(existing_relation) %}
-    {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config) %}
+    {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config.model) %}
     {% do return(_configuration_changes) %}
 {%- endmacro %}

setup.py

@@ -68,6 +68,8 @@ def _get_dbt_core_version():
     include_package_data=True,
     install_requires=[
         "dbt-core~={}".format(dbt_core_version),
+        "dbt-common~=0.1.0",
+        "dbt-adapters~=0.1.0a2",
         "snowflake-connector-python[secure-local-storage]~=3.0",
         # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
         "agate",

tests/functional/adapter/column_types/fixtures.py

@@ -23,7 +23,7 @@
 version: 2
 models:
   - name: model
-    tests:
+    data_tests:
       - is_type:
           column_map:
             smallint_col: ['numeric', 'number', 'not string', 'not float', 'not integer']

tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from dbt.contracts.graph.model_config import OnConfigurationChangeOption
+from dbt_common.contracts.config.materialization import OnConfigurationChangeOption
 from dbt.tests.util import (
     assert_message_in_logs,
     get_model_file,

tests/functional/adapter/simple_seed/test_simple_seed_override.py

@@ -8,46 +8,46 @@
 - name: seed_enabled
   columns:
   - name: birthday
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
   - name: seed_id
-    tests:
+    data_tests:
     - column_type:
         type: FLOAT
 
 - name: seed_tricky
   columns:
   - name: seed_id
-    tests:
+    data_tests:
     - column_type:
         type: NUMBER(38,0)
   - name: seed_id_str
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
   - name: a_bool
-    tests:
+    data_tests:
     - column_type:
         type: BOOLEAN
   - name: looks_like_a_bool
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
   - name: a_date
-    tests:
+    data_tests:
     - column_type:
         type: TIMESTAMP_NTZ
   - name: looks_like_a_date
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
   - name: relative
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
   - name: weekday
-    tests:
+    data_tests:
     - column_type:
         type: character varying(16777216)
 """.lstrip()