Skip to content

Commit

Permalink
remove iceberg dynamic table updates
Browse files Browse the repository at this point in the history
  • Loading branch information
mikealfare committed Sep 20, 2024
1 parent fb9aee1 commit 1fed9c3
Show file tree
Hide file tree
Showing 8 changed files with 3 additions and 303 deletions.
2 changes: 1 addition & 1 deletion dbt/adapters/snowflake/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def _behavior_flags(self) -> List[BehaviorFlag]:
return [
{
"name": "enable_iceberg_materializations",
"default": True,
"default": False,
"description": (
"Enabling Iceberg materializations introduces latency to metadata queries, "
"specifically within the list_relations_without_caching macro. Since Iceberg "
Expand Down
7 changes: 0 additions & 7 deletions dbt/adapters/snowflake/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from dbt_common.events.functions import fire_event, warn_or_error

from dbt.adapters.snowflake.relation_configs import (
SnowflakeCatalogConfigChange,
SnowflakeDynamicTableConfig,
SnowflakeDynamicTableConfigChangeset,
SnowflakeDynamicTableRefreshModeConfigChange,
Expand Down Expand Up @@ -115,12 +114,6 @@ def dynamic_table_config_changeset(
context=new_dynamic_table.refresh_mode,
)

if new_dynamic_table.catalog != existing_dynamic_table.catalog:
config_change_collection.catalog = SnowflakeCatalogConfigChange(
action=RelationConfigChangeAction.create,
context=new_dynamic_table.catalog,
)

if config_change_collection.has_changes:
return config_change_collection
return None
Expand Down
4 changes: 0 additions & 4 deletions dbt/adapters/snowflake/relation_configs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
from dbt.adapters.snowflake.relation_configs.catalog import (
SnowflakeCatalogConfig,
SnowflakeCatalogConfigChange,
)
from dbt.adapters.snowflake.relation_configs.dynamic_table import (
SnowflakeDynamicTableConfig,
SnowflakeDynamicTableConfigChangeset,
Expand Down
121 changes: 0 additions & 121 deletions dbt/adapters/snowflake/relation_configs/catalog.py

This file was deleted.

12 changes: 1 addition & 11 deletions dbt/adapters/snowflake/relation_configs/dynamic_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,6 @@
from typing_extensions import Self

from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase
from dbt.adapters.snowflake.relation_configs.catalog import (
SnowflakeCatalogConfig,
SnowflakeCatalogConfigChange,
)


if TYPE_CHECKING:
Expand Down Expand Up @@ -60,7 +56,6 @@ class SnowflakeDynamicTableConfig(SnowflakeRelationConfigBase):
query: str
target_lag: str
snowflake_warehouse: str
catalog: SnowflakeCatalogConfig
refresh_mode: Optional[RefreshMode] = RefreshMode.default()
initialize: Optional[Initialize] = Initialize.default()

Expand All @@ -75,7 +70,6 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> Self:
"query": config_dict.get("query"),
"target_lag": config_dict.get("target_lag"),
"snowflake_warehouse": config_dict.get("snowflake_warehouse"),
"catalog": SnowflakeCatalogConfig.from_dict(config_dict["catalog"]),
"refresh_mode": config_dict.get("refresh_mode"),
"initialize": config_dict.get("initialize"),
}
Expand All @@ -91,7 +85,6 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any
"query": relation_config.compiled_code,
"target_lag": relation_config.config.extra.get("target_lag"),
"snowflake_warehouse": relation_config.config.extra.get("snowflake_warehouse"),
"catalog": SnowflakeCatalogConfig.parse_relation_config(relation_config),
}

if refresh_mode := relation_config.config.extra.get("refresh_mode"):
Expand All @@ -113,7 +106,6 @@ def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str,
"query": dynamic_table.get("text"),
"target_lag": dynamic_table.get("target_lag"),
"snowflake_warehouse": dynamic_table.get("warehouse"),
"catalog": SnowflakeCatalogConfig.parse_relation_results(relation_results),
"refresh_mode": dynamic_table.get("refresh_mode"),
# we don't get initialize since that's a one-time scheduler attribute, not a DT attribute
}
Expand Down Expand Up @@ -153,7 +145,6 @@ class SnowflakeDynamicTableConfigChangeset:
target_lag: Optional[SnowflakeDynamicTableTargetLagConfigChange] = None
snowflake_warehouse: Optional[SnowflakeDynamicTableWarehouseConfigChange] = None
refresh_mode: Optional[SnowflakeDynamicTableRefreshModeConfigChange] = None
catalog: Optional[SnowflakeCatalogConfigChange] = None

@property
def requires_full_refresh(self) -> bool:
Expand All @@ -166,10 +157,9 @@ def requires_full_refresh(self) -> bool:
else False
),
self.refresh_mode.requires_full_refresh if self.refresh_mode else False,
self.catalog.requires_full_refresh if self.catalog else False,
]
)

@property
def has_changes(self) -> bool:
return any([self.target_lag, self.snowflake_warehouse, self.refresh_mode, self.catalog])
return any([self.target_lag, self.snowflake_warehouse, self.refresh_mode])
71 changes: 0 additions & 71 deletions dbt/include/snowflake/macros/relations/dynamic_table/create.sql
Original file line number Diff line number Diff line change
@@ -1,46 +1,7 @@
{% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%}
{#-
Produce DDL that creates a dynamic table

Args:
- relation: Union[SnowflakeRelation, str]
- SnowflakeRelation - required for relation.render()
- str - is already the rendered relation name
- sql: str - the code defining the model
Globals:
- config: NodeConfig - contains the attribution required to produce a SnowflakeDynamicTableConfig
Returns:
A valid DDL statement which will result in a new dynamic table.
-#}

{%- set dynamic_table = relation.from_config(config.model) -%}

{%- if dynamic_table.catalog.table_format == 'iceberg' -%}
{{ _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) }}
{%- else -%}
{{ _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) }}
{%- endif -%}

{%- endmacro %}


{% macro _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) -%}
{#-
Produce DDL that creates a standard dynamic table

This follows the syntax outlined here:
https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#syntax

Args:
- dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table
- relation: Union[SnowflakeRelation, str]
- SnowflakeRelation - required for relation.render()
- str - is already the rendered relation name
- sql: str - the code defining the model
Returns:
A valid DDL statement which will result in a new dynamic standard table.
-#}

create dynamic table {{ relation }}
target_lag = '{{ dynamic_table.target_lag }}'
warehouse = {{ dynamic_table.snowflake_warehouse }}
Expand All @@ -51,35 +12,3 @@ create dynamic table {{ relation }}
)

{%- endmacro %}


{% macro _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) -%}
{#-
Produce DDL that creates a dynamic iceberg table

This follows the syntax outlined here:
https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table

Args:
- dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table
- relation: Union[SnowflakeRelation, str]
- SnowflakeRelation - required for relation.render()
- str - is already the rendered relation name
- sql: str - the code defining the model
Returns:
A valid DDL statement which will result in a new dynamic iceberg table.
-#}

create dynamic iceberg table {{ relation }}
target_lag = '{{ dynamic_table.target_lag }}'
warehouse = {{ dynamic_table.snowflake_warehouse }}
{{ optional('external_volume', dynamic_table.catalog.external_volume) }}
{{ optional('catalog', dynamic_table.catalog.name) }}
base_location = {{ dynamic_table.catalog.base_location }}
{{ optional('refresh_mode', dynamic_table.refresh_mode) }}
{{ optional('initialize', dynamic_table.initialize) }}
as (
{{ sql }}
)

{%- endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,5 @@ from table(result_scan(last_query_id()))
{%- endset %}
{% set _dynamic_table = run_query(_dynamic_table_sql) %}

{% if adapter.behavior.enable_iceberg_materializations.no_warn %}
{%- set _catalog_sql -%}
show iceberg tables
like '{{ relation.identifier }}'
in schema {{ relation.database }}.{{ relation.schema }}
;
select
"catalog_name",
"external_volume_name",
"base_location"
from table(result_scan(last_query_id()))
{%- endset %}
{% set _catalog = run_query(_catalog_sql) %}
{% else %}
{% set _catalog = none %}
{% endif %}

{% do return({'dynamic_table': _dynamic_table, 'catalog': _catalog}) %}
{% do return({'dynamic_table': _dynamic_table}) %}
{% endmacro %}
Loading

0 comments on commit 1fed9c3

Please sign in to comment.