Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix Dynamic Iceberg Table Required DDL Params #1201

Merged
merged 6 commits into from
Oct 8, 2024
Merged

Fix Dynamic Iceberg Table Required DDL Params #1201

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9b8e04c
Fix base location not rendering without subpath and add tests.
VersusFacit Oct 8, 2024
5b20f15
Add changelog.
VersusFacit Oct 8, 2024
40c33e3
Revert changes to external volume
VersusFacit Oct 8, 2024
ca430cc
revert changes to catalog optionality.
VersusFacit Oct 8, 2024
c22bed0
Tabs.
VersusFacit Oct 8, 2024
9acac49
Fix base_location_subpath generation for dynamic tables.
VersusFacit Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions dbt/adapters/snowflake/relation_configs/catalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import Any, Dict, Optional, TYPE_CHECKING, Set
from typing import Any, Dict, Optional, TYPE_CHECKING, Set, List

if TYPE_CHECKING:
import agate
Expand Down Expand Up @@ -82,8 +82,10 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any
if external_volume := relation_config.config.extra.get("external_volume"):
config_dict["external_volume"] = external_volume

if base_location := relation_config.config.extra.get("base_location_subpath"):
config_dict["base_location"] = base_location
catalog_dirs: List[str] = ["_dbt", relation_config.schema, relation_config.name]
if base_location_subpath := relation_config.config.extra.get("base_location_subpath"):
catalog_dirs.append(base_location_subpath)
config_dict["base_location"] = "/".join(catalog_dirs)

return config_dict

Expand Down
6 changes: 3 additions & 3 deletions dbt/include/snowflake/macros/relations/dynamic_table/create.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@
create dynamic iceberg table {{ relation }}
target_lag = '{{ dynamic_table.target_lag }}'
warehouse = {{ dynamic_table.snowflake_warehouse }}
{{ optional('external_volume', dynamic_table.catalog.external_volume) }}
{{ optional('catalog', dynamic_table.catalog.name) }}
base_location = {{ dynamic_table.catalog.base_location }}
external_volume = '{{ dynamic_table.catalog.external_volume }}'
VersusFacit marked this conversation as resolved.
Show resolved Hide resolved
catalog = '{{ dynamic_table.catalog.name }}'
base_location = '{{ dynamic_table.catalog.base_location }}'
{{ optional('refresh_mode', dynamic_table.refresh_mode) }}
{{ optional('initialize', dynamic_table.initialize) }}
as (
Expand Down
6 changes: 3 additions & 3 deletions dbt/include/snowflake/macros/relations/dynamic_table/replace.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,9 @@
create or replace dynamic iceberg table {{ relation }}
target_lag = '{{ dynamic_table.target_lag }}'
warehouse = {{ dynamic_table.snowflake_warehouse }}
{{ optional('external_volume', dynamic_table.catalog.external_volume) }}
{{ optional('catalog', dynamic_table.catalog.name) }}
base_location = {{ dynamic_table.catalog.base_location }}
external_volume = '{{ dynamic_table.catalog.external_volume }}'
catalog = '{{ dynamic_table.catalog.name }}'
base_location = '{{ dynamic_table.catalog.base_location }}'
{{ optional('refresh_mode', dynamic_table.refresh_mode) }}
{{ optional('initialize', dynamic_table.initialize) }}
as (
Expand Down
85 changes: 85 additions & 0 deletions tests/functional/iceberg/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
_MODEL_BASIC_TABLE_MODEL = """
{{
config(
materialized = "table",
cluster_by=['id'],
)
}}
select 1 as id
"""

_MODEL_BASIC_ICEBERG_MODEL = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location_subpath="subpath",
)
}}

select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_DYNAMIC_TABLE_MODEL = """
{{ config(
materialized='dynamic_table',
snowflake_warehouse='DBT_TESTING',
target_lag='1 minute',
refresh_mode='INCREMENTAL',
table_format='iceberg',
external_volume='s3_iceberg_snow',
) }}

select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH = """
{{ config(
materialized='dynamic_table',
snowflake_warehouse='DBT_TESTING',
target_lag='1 minute',
refresh_mode='INCREMENTAL',
table_format='iceberg',
external_volume='s3_iceberg_snow',
base_location_subpath='subpath',
) }}

select * from {{ ref('first_table') }}
"""

_MODEL_BUILT_ON_ICEBERG_TABLE = """
{{
config(
materialized = "table",
)
}}
select * from {{ ref('iceberg_table') }}
"""

_MODEL_TABLE_BEFORE_SWAP = """
{{
config(
materialized = "table",
)
}}
select 1 as id
"""

_MODEL_VIEW_BEFORE_SWAP = """
select 1 as id
"""

_MODEL_TABLE_FOR_SWAP_ICEBERG = """
{{
config(
materialized = "table",
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location_subpath="subpath",
)
}}
select 1 as id
"""
72 changes: 13 additions & 59 deletions tests/functional/iceberg/test_table_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,64 +4,16 @@

from dbt.tests.util import run_dbt, rm_file, write_file

_MODEL_BASIC_TABLE_MODEL = """
{{
config(
materialized = "table",
cluster_by=['id'],
)
}}
select 1 as id
"""

_MODEL_BASIC_ICEBERG_MODEL = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location_subpath="subpath",
)
}}

select * from {{ ref('first_table') }}
"""

_MODEL_BUILT_ON_ICEBERG_TABLE = """
{{
config(
materialized = "table",
)
}}
select * from {{ ref('iceberg_table') }}
"""

_MODEL_TABLE_BEFORE_SWAP = """
{{
config(
materialized = "table",
)
}}
select 1 as id
"""

_MODEL_VIEW_BEFORE_SWAP = """
select 1 as id
"""

_MODEL_TABLE_FOR_SWAP_ICEBERG = """
{{
config(
materialized = "table",
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location_subpath="subpath",
)
}}
select 1 as id
"""
from tests.functional.iceberg.models import (
_MODEL_BASIC_TABLE_MODEL,
_MODEL_BASIC_ICEBERG_MODEL,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH,
_MODEL_BUILT_ON_ICEBERG_TABLE,
_MODEL_TABLE_BEFORE_SWAP,
_MODEL_VIEW_BEFORE_SWAP,
_MODEL_TABLE_FOR_SWAP_ICEBERG,
)


class TestIcebergTableBuilds:
Expand All @@ -75,11 +27,13 @@ def models(self):
"first_table.sql": _MODEL_BASIC_TABLE_MODEL,
"iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL,
"table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE,
"dynamic_table.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL,
"dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH,
}

def test_iceberg_tables_build_and_can_be_referred(self, project):
run_results = run_dbt()
assert len(run_results) == 3
assert len(run_results) == 5


class TestIcebergTableTypeBuildsOnExistingTable:
Expand Down
Loading