Merge branch 'main' into ADAP-209/update_iceberg_docs_note

.changes/unreleased/Features-20241107-170307.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: 'Allow configurable pagination on list_relations_without_caching to support
+  users with a large number of objects per schema'
+time: 2024-11-07T17:03:07.826352-05:00
+custom:
+  Author: mikealfare
+  Issue: "1234"

.changes/unreleased/Fixes-20241104-104610.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: 'Performance fixes for snowflake microbatch strategy: use temp view instead
+  of table, remove unnecessary ''using'' clause'
+time: 2024-11-04T10:46:10.005317-05:00
+custom:
+  Author: michelleark
+  Issue: "1228"

.changes/unreleased/Fixes-20241104-172349.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Iceberg quoting ignore fix.
+time: 2024-11-04T17:23:49.706297-08:00
+custom:
+    Author: versusfacit
+    Issue: "1227"

.changes/unreleased/Under the Hood-20241106-113249.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base
+time: 2024-11-06T11:32:49.503467-08:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1233"

dbt/adapters/snowflake/connections.py

@@ -84,11 +84,6 @@ def snowflake_private_key(private_key: RSAPrivateKey) -> bytes:
     )
 
 
-@dataclass
-class SnowflakeAdapterResponse(AdapterResponse):
-    query_id: str = ""
-
-
 @dataclass
 class SnowflakeCredentials(Credentials):
     account: str
@@ -447,17 +442,17 @@ def cancel(self, connection):
         logger.debug("Cancel query '{}': {}".format(connection_name, res))
 
     @classmethod
-    def get_response(cls, cursor) -> SnowflakeAdapterResponse:
+    def get_response(cls, cursor) -> AdapterResponse:
         code = cursor.sqlstate
 
         if code is None:
             code = "SUCCESS"
-
-        return SnowflakeAdapterResponse(
+        query_id = str(cursor.sfqid) if cursor.sfqid is not None else None
+        return AdapterResponse(
             _message="{} {}".format(code, cursor.rowcount),
             rows_affected=cursor.rowcount,
             code=code,
-            query_id=cursor.sfqid,
+            query_id=query_id,
         )
 
     # disable transactional logic by default on Snowflake

dbt/adapters/snowflake/impl.py

@@ -264,6 +264,11 @@ def list_relations_without_caching(
         # this can be collapsed once Snowflake adds is_iceberg to show objects
         columns = ["database_name", "schema_name", "name", "kind", "is_dynamic"]
         if self.behavior.enable_iceberg_materializations.no_warn:
+            # The QUOTED_IDENTIFIERS_IGNORE_CASE setting impacts column names like
+            # is_iceberg which is created by dbt, but it does not affect the case
+            # of column values in Snowflake's SHOW OBJECTS query! This
+            # normalization step ensures metadata queries are handled consistently.
+            schema_objects = schema_objects.rename(column_names={"IS_ICEBERG": "is_iceberg"})
             columns.append("is_iceberg")
 
         return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)]

dbt/include/snowflake/macros/adapters.sql

@@ -111,9 +111,10 @@
 
       {%- if loop.index == max_iter -%}
         {%- set msg -%}
-           dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation }}.
-           Your schema exceeds this limit. Please contact support@getdbt.com for troubleshooting tips,
-           or review and reduce the number of objects contained.
+            dbt is currently configured to list a maximum of {{ max_total_results }} objects per schema.
+            {{ schema_relation }} exceeds this limit. If this is expected, you may configure this limit
+            by setting list_relations_per_page and list_relations_page_limit in your project flags.
+            It is recommended to start by increasing list_relations_page_limit to something more than the default of 10.
         {%- endset -%}
 
         {% do exceptions.raise_compiler_error(msg) %}
@@ -136,6 +137,8 @@
 
 {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %}
 
+  {%- set max_results_per_iter = adapter.config.flags.get('list_relations_per_page', max_results_per_iter) -%}
+  {%- set max_iter = adapter.config.flags.get('list_relations_page_limit', max_iter) -%}
   {%- set max_total_results = max_results_per_iter * max_iter -%}
   {%- set sql -%}
     {% if schema_relation is string %}
@@ -147,7 +150,7 @@
     {# -- Gated for performance reason. If you don't want Iceberg, you shouldn't pay the
        -- latency penalty. #}
     {% if adapter.behavior.enable_iceberg_materializations.no_warn %}
-      select all_objects.*, is_iceberg as "is_iceberg"
+      select all_objects.*, is_iceberg
       from table(result_scan(last_query_id(-1))) all_objects
       left join INFORMATION_SCHEMA.tables as all_tables
         on all_tables.table_name = all_objects."name"

dbt/include/snowflake/macros/materializations/incremental.sql

@@ -20,7 +20,7 @@
 
        The append strategy can use a view because it will run a single INSERT statement.
 
-       When unique_key is none, the delete+insert strategy can use a view beacuse a
+       When unique_key is none, the delete+insert and microbatch strategies can use a view beacuse a
        single INSERT statement is run with no DELETES as part of the statement.
        Otherwise, play it safe by using a temporary table.
   #} */
@@ -32,10 +32,10 @@
     ) %}
   {% endif %}
 
-  {% if strategy == "delete+insert" and tmp_relation_type is not none and tmp_relation_type != "table" and unique_key is not none %}
+  {% if strategy in ["delete+insert", "microbatch"] and tmp_relation_type is not none and tmp_relation_type != "table" and unique_key is not none %}
     {% do exceptions.raise_compiler_error(
       "In order to maintain consistent results when `unique_key` is not none,
-      the `delete+insert` strategy only supports `table` for `tmp_relation_type` but "
+      the `" ~ strategy ~ "` strategy only supports `table` for `tmp_relation_type` but "
       ~ tmp_relation_type ~ " was specified."
       )
   %}
@@ -49,7 +49,7 @@
     {{ return("view") }}
   {% elif strategy in ("default", "merge", "append") %}
     {{ return("view") }}
-  {% elif strategy == "delete+insert" and unique_key is none %}
+  {% elif strategy in ["delete+insert", "microbatch"] and unique_key is none %}
     {{ return("view") }}
   {% else %}
     {{ return("table") }}

dbt/include/snowflake/macros/materializations/merge.sql

@@ -66,7 +66,6 @@
     {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %}
 
     delete from {{ target }} DBT_INTERNAL_TARGET
-    using {{ source }}
     where (
     {% for predicate in incremental_predicates %}
         {%- if not loop.first %}and {% endif -%} {{ predicate }}