ADAP-821: Refactor prep for dynamic table / table/view swap (#751)

* restructure macros directory, create separate files and directories for each ddl statement and relation type
dbt-labs · Aug 28, 2023 · 8d5fca9 · 8d5fca9
1 parent 43d12bc
commit 8d5fca9
Show file tree

Hide file tree

Showing 14 changed files with 184 additions and 191 deletions.
diff --git a/.changes/unreleased/Under the Hood-20230821-230921.yaml b/.changes/unreleased/Under the Hood-20230821-230921.yaml
@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Restructure macros files - move relation ddl statements into separate files
+  and directories
+time: 2023-08-21T23:09:21.317975-04:00
+custom:
+  Author: mikealfare
+  Issue: "750"
diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql
@@ -1,58 +1,3 @@
-{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%}
-  {%- if language == 'sql' -%}
-    {%- set transient = config.get('transient', default=true) -%}
-    {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
-    {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
-    {%- set copy_grants = config.get('copy_grants', default=false) -%}
-
-    {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
-      {%- set cluster_by_keys = [cluster_by_keys] -%}
-    {%- endif -%}
-    {%- if cluster_by_keys is not none -%}
-      {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
-    {% else %}
-      {%- set cluster_by_string = none -%}
-    {%- endif -%}
-    {%- set sql_header = config.get('sql_header', none) -%}
-
-    {{ sql_header if sql_header is not none }}
-
-        create or replace {% if temporary -%}
-          temporary
-        {%- elif transient -%}
-          transient
-        {%- endif %} table {{ relation }}
-        {%- set contract_config = config.get('contract') -%}
-        {%- if contract_config.enforced -%}
-          {{ get_assert_columns_equivalent(sql) }}
-          {{ get_table_columns_and_constraints() }}
-          {% set compiled_code = get_select_subquery(compiled_code) %}
-        {% endif %}
-        {% if copy_grants and not temporary -%} copy grants {%- endif %} as
-        (
-          {%- if cluster_by_string is not none -%}
-            select * from (
-              {{ compiled_code }}
-              ) order by ({{ cluster_by_string }})
-          {%- else -%}
-            {{ compiled_code }}
-          {%- endif %}
-        );
-      {% if cluster_by_string is not none and not temporary -%}
-        alter table {{relation}} cluster by ({{cluster_by_string}});
-      {%- endif -%}
-      {% if enable_automatic_clustering and cluster_by_string is not none and not temporary  -%}
-        alter table {{relation}} resume recluster;
-      {%- endif -%}
-
-  {%- elif language == 'python' -%}
-    {{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }}
-  {%- else -%}
-      {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %}
-  {%- endif -%}
-
-{% endmacro %}
-
 {% macro get_column_comment_sql(column_name, column_dict) -%}
   {% if (column_name|upper in column_dict) -%}
     {% set matched_column = column_name|upper -%}
@@ -79,35 +24,6 @@
 )
 {% endmacro %}
 
-{% macro snowflake__create_view_as_with_temp_flag(relation, sql, is_temporary=False) -%}
-  {%- set secure = config.get('secure', default=false) -%}
-  {%- set copy_grants = config.get('copy_grants', default=false) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-  create or replace {% if secure -%}
-    secure
-  {%- endif %} {% if is_temporary -%}
-    temporary
-  {%- endif %} view {{ relation }}
-  {% if config.persist_column_docs() -%}
-    {% set model_columns = model.columns %}
-    {% set query_columns = get_columns_in_query(sql) %}
-    {{ get_persist_docs_column_list(model_columns, query_columns) }}
-
-  {%- endif %}
-  {%- set contract_config = config.get('contract') -%}
-  {%- if contract_config.enforced -%}
-    {{ get_assert_columns_equivalent(sql) }}
-  {%- endif %}
-  {% if copy_grants -%} copy grants {%- endif %} as (
-    {{ sql }}
-  );
-{% endmacro %}
-
-{% macro snowflake__create_view_as(relation, sql) -%}
-  {{ snowflake__create_view_as_with_temp_flag(relation, sql) }}
-{% endmacro %}
 
 {% macro snowflake__get_columns_in_relation(relation) -%}
   {%- set sql -%}
@@ -248,13 +164,6 @@
 {%- endmacro %}
 
 
-{% macro snowflake__rename_relation(from_relation, to_relation) -%}
-  {% call statement('rename_relation') -%}
-    alter table {{ from_relation }} rename to {{ to_relation }}
-  {%- endcall %}
-{% endmacro %}
-
-
 {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%}
   {% call statement('alter_column_type') %}
     alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }};
@@ -372,14 +281,3 @@
     {{ snowflake_dml_explicit_transaction(truncate_dml) }}
   {%- endcall %}
 {% endmacro %}
-
-
-{% macro snowflake__drop_relation(relation) -%}
-    {%- if relation.is_dynamic_table -%}
-        {% call statement('drop_relation', auto_begin=False) -%}
-            drop dynamic table if exists {{ relation }}
-        {%- endcall %}
-    {%- else -%}
-        {{- default__drop_relation(relation) -}}
-    {%- endif -%}
-{% endmacro %}
diff --git a/...zations/dynamic_table/materialization.sql → ...macros/materializations/dynamic_table.sql b/...zations/dynamic_table/materialization.sql → ...macros/materializations/dynamic_table.sql
@@ -118,3 +118,10 @@
     {% do persist_docs(target_relation, model) %}
 
 {% endmacro %}
+
+
+{% macro snowflake__get_dynamic_table_configuration_changes(existing_relation, new_config) -%}
+    {% set _existing_dynamic_table = snowflake__describe_dynamic_table(existing_relation) %}
+    {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config) %}
+    {% do return(_configuration_changes) %}
+{%- endmacro %}
diff --git a/dbt/include/snowflake/macros/materializations/dynamic_table/ddl.sql b/dbt/include/snowflake/macros/materializations/dynamic_table/ddl.sql
diff --git a/dbt/include/snowflake/macros/relations/drop.sql b/dbt/include/snowflake/macros/relations/drop.sql
@@ -0,0 +1,9 @@
+{% macro snowflake__drop_relation(relation) -%}
+    {%- if relation.is_dynamic_table -%}
+        {% call statement('drop_relation', auto_begin=False) -%}
+            drop dynamic table if exists {{ relation }}
+        {%- endcall %}
+    {%- else -%}
+        {{- default__drop_relation(relation) -}}
+    {%- endif -%}
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql
@@ -0,0 +1,5 @@
+{% macro snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -%}
+    {{- log('Applying REPLACE to: ' ~ target_relation) -}}
+    {{ snowflake__get_drop_dynamic_table_sql(existing_relation) }};
+    {{ snowflake__get_create_dynamic_table_as_sql(target_relation, sql) }}
+{%- endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql b/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql
@@ -0,0 +1,27 @@
+{% macro snowflake__get_alter_dynamic_table_as_sql(
+    target_relation,
+    configuration_changes,
+    sql,
+    existing_relation,
+    backup_relation,
+    intermediate_relation
+) -%}
+    {{- log('Applying ALTER to: ' ~ target_relation) -}}
+
+    {% if configuration_changes.requires_full_refresh %}
+        {{- snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -}}
+
+    {% else %}
+
+        {%- set target_lag = configuration_changes.target_lag -%}
+        {%- if target_lag -%}{{- log('Applying UPDATE TARGET_LAG to: ' ~ existing_relation) -}}{%- endif -%}
+        {%- set snowflake_warehouse = configuration_changes.snowflake_warehouse -%}
+        {%- if snowflake_warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%}
+
+        alter dynamic table {{ existing_relation }} set
+            {% if target_lag %}target_lag = '{{ target_lag.context }}'{% endif %}
+            {% if snowflake_warehouse %}warehouse = {{ snowflake_warehouse.context }}{% endif %}
+
+    {%- endif -%}
+
+{%- endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql
@@ -0,0 +1,13 @@
+{% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%}
+    {{- log('Applying CREATE to: ' ~ relation) -}}
+
+    create or replace dynamic table {{ relation }}
+        target_lag = '{{ config.get("target_lag") }}'
+        warehouse = {{ config.get("snowflake_warehouse") }}
+        as (
+            {{ sql }}
+        )
+    ;
+    {{ snowflake__refresh_dynamic_table(relation) }}
+
+{%- endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql b/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql
@@ -0,0 +1,19 @@
+{% macro snowflake__describe_dynamic_table(relation) %}
+    {%- set _dynamic_table_sql -%}
+        show dynamic tables
+            like '{{ relation.identifier }}'
+            in schema {{ relation.database }}.{{ relation.schema }}
+        ;
+        select
+            "name",
+            "schema_name",
+            "database_name",
+            "text",
+            "target_lag",
+            "warehouse"
+        from table(result_scan(last_query_id()))
+    {%- endset %}
+    {% set _dynamic_table = run_query(_dynamic_table_sql) %}
+
+    {% do return({'dynamic_table': _dynamic_table}) %}
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql b/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql
@@ -0,0 +1,3 @@
+{% macro snowflake__get_drop_dynamic_table_sql(relation) %}
+    drop dynamic table if exists {{ relation }}
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql b/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql
@@ -0,0 +1,5 @@
+{% macro snowflake__refresh_dynamic_table(relation) -%}
+    {{- log('Applying REFRESH to: ' ~ relation) -}}
+
+    alter dynamic table {{ relation }} refresh
+{%- endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/rename.sql b/dbt/include/snowflake/macros/relations/rename.sql
@@ -0,0 +1,5 @@
+{% macro snowflake__rename_relation(from_relation, to_relation) -%}
+  {% call statement('rename_relation') -%}
+    alter table {{ from_relation }} rename to {{ to_relation }}
+  {%- endcall %}
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql
@@ -0,0 +1,54 @@
+{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%}
+  {%- if language == 'sql' -%}
+    {%- set transient = config.get('transient', default=true) -%}
+    {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
+    {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
+    {%- set copy_grants = config.get('copy_grants', default=false) -%}
+
+    {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
+      {%- set cluster_by_keys = [cluster_by_keys] -%}
+    {%- endif -%}
+    {%- if cluster_by_keys is not none -%}
+      {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
+    {% else %}
+      {%- set cluster_by_string = none -%}
+    {%- endif -%}
+    {%- set sql_header = config.get('sql_header', none) -%}
+
+    {{ sql_header if sql_header is not none }}
+
+        create or replace {% if temporary -%}
+          temporary
+        {%- elif transient -%}
+          transient
+        {%- endif %} table {{ relation }}
+        {%- set contract_config = config.get('contract') -%}
+        {%- if contract_config.enforced -%}
+          {{ get_assert_columns_equivalent(sql) }}
+          {{ get_table_columns_and_constraints() }}
+          {% set compiled_code = get_select_subquery(compiled_code) %}
+        {% endif %}
+        {% if copy_grants and not temporary -%} copy grants {%- endif %} as
+        (
+          {%- if cluster_by_string is not none -%}
+            select * from (
+              {{ compiled_code }}
+              ) order by ({{ cluster_by_string }})
+          {%- else -%}
+            {{ compiled_code }}
+          {%- endif %}
+        );
+      {% if cluster_by_string is not none and not temporary -%}
+        alter table {{relation}} cluster by ({{cluster_by_string}});
+      {%- endif -%}
+      {% if enable_automatic_clustering and cluster_by_string is not none and not temporary  -%}
+        alter table {{relation}} resume recluster;
+      {%- endif -%}
+
+  {%- elif language == 'python' -%}
+    {{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }}
+  {%- else -%}
+      {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %}
+  {%- endif -%}
+
+{% endmacro %}