Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bug] Fix scenarios where --empty flag pushes limit 0 into metadata queries #1100

Merged
merged 5 commits into from
Jul 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/unreleased/Fixes-20240628-190140.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Fixes
body: Fix scenario where using the `--empty` flag causes metadata queries to contain
limit clauses
time: 2024-06-28T19:01:40.558234-04:00
custom:
Author: mikealfare
Issue: "1033"
16 changes: 8 additions & 8 deletions dbt/include/snowflake/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@

{% macro snowflake__get_columns_in_relation(relation) -%}
{%- set sql -%}
describe table {{ relation }}
describe table {{ relation.render() }}
{%- endset -%}
{%- set result = run_query(sql) -%}

{% set maximum = 10000 %}
{% if (result | length) >= maximum %}
{% set msg %}
Too many columns in relation {{ relation }}! dbt can only get
Too many columns in relation {{ relation.render() }}! dbt can only get
information about relations with fewer than {{ maximum }} columns.
{% endset %}
{% do exceptions.raise_compiler_error(msg) %}
Expand Down Expand Up @@ -186,7 +186,7 @@

{% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%}
{% call statement('alter_column_type') %}
alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }};
alter table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }};
{% endcall %}
{% endmacro %}

Expand All @@ -196,7 +196,7 @@
{%- else -%}
{%- set relation_type = relation.type -%}
{%- endif -%}
comment on {{ relation_type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$;
comment on {{ relation_type }} {{ relation.render() }} IS $${{ relation_comment | replace('$', '[$]') }}$$;
{% endmacro %}


Expand All @@ -207,7 +207,7 @@
{% else -%}
{% set relation_type = relation.type %}
{% endif %}
alter {{ relation_type }} {{ relation }} alter
alter {{ relation_type }} {{ relation.render() }} alter
{% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %}
{{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }}
{% endfor %}
Expand Down Expand Up @@ -266,7 +266,7 @@
{% if add_columns %}

{% set sql -%}
alter {{ relation_type }} {{ relation }} add column
alter {{ relation_type }} {{ relation.render() }} add column
{% for column in add_columns %}
{{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
{% endfor %}
Expand All @@ -279,7 +279,7 @@
{% if remove_columns %}

{% set sql -%}
alter {{ relation_type }} {{ relation }} drop column
alter {{ relation_type }} {{ relation.render() }} drop column
{% for column in remove_columns %}
{{ column.name }}{{ ',' if not loop.last }}
{% endfor %}
Expand Down Expand Up @@ -312,7 +312,7 @@

{% macro snowflake__truncate_relation(relation) -%}
{% set truncate_dml %}
truncate table {{ relation }}
truncate table {{ relation.render() }}
{% endset %}
{% call statement('truncate_relation') -%}
{{ snowflake_dml_explicit_transaction(truncate_dml) }}
Expand Down
72 changes: 72 additions & 0 deletions tests/functional/adapter/empty/_models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
SEED = """
my_id,my_value
1,a
2,b
3,c
""".strip()


SCHEMA = """
version: 2

seeds:
- name: my_seed
description: "This is my_seed"
columns:
- name: id
description: "This is my_seed.my_id"
"""

CONTROL = """
select * from {{ ref("my_seed") }}
"""


GET_COLUMNS_IN_RELATION = """
{{ config(materialized="table") }}
{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %}
select * from {{ ref("my_seed") }}
"""


ALTER_COLUMN_TYPE = """
{{ config(materialized="table") }}
{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }}
select * from {{ ref("my_seed") }}
"""


ALTER_RELATION_COMMENT = """
{{ config(
materialized="table",
persist_docs={"relations": True},
) }}
select * from {{ ref("my_seed") }}
"""


ALTER_COLUMN_COMMENT = """
{{ config(
materialized="table",
persist_docs={"columns": True},
) }}
select * from {{ ref("my_seed") }}
"""


ALTER_RELATION_ADD_REMOVE_COLUMNS = """
{{ config(materialized="table") }}
{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %}
{% set my_column = api.Column("my_column", "varchar") %}
{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %}
{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %}
select * from {{ ref("my_seed") }}
"""


TRUNCATE_RELATION = """
{{ config(materialized="table") }}
{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %}
{{ truncate_relation(my_seed) }}
select * from {{ ref("my_seed") }}
"""
42 changes: 42 additions & 0 deletions tests/functional/adapter/empty/test_empty.py
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most of this will move into dbt-tests-adapter and then imported here afterwards.

Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef
from dbt.tests.util import run_dbt
import pytest

from tests.functional.adapter.empty import _models


class TestSnowflakeEmpty(BaseTestEmpty):
Expand All @@ -7,3 +11,41 @@ class TestSnowflakeEmpty(BaseTestEmpty):

class TestSnowflakeEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef):
pass


class TestMetadataWithEmptyFlag:
@pytest.fixture(scope="class")
def seeds(self):
return {"my_seed.csv": _models.SEED}

@pytest.fixture(scope="class")
def models(self):
return {
"schema.yml": _models.SCHEMA,
"control.sql": _models.CONTROL,
"get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION,
"alter_column_type.sql": _models.ALTER_COLUMN_TYPE,
"alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT,
"alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT,
"alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS,
"truncate_relation.sql": _models.TRUNCATE_RELATION,
}

@pytest.fixture(scope="class", autouse=True)
def setup(self, project):
run_dbt(["seed"])

@pytest.mark.parametrize(
"model",
[
"control",
"get_columns_in_relation",
"alter_column_type",
"alter_relation_comment",
"alter_column_comment",
"alter_relation_add_remove_columns",
"truncate_relation",
],
)
def test_run(self, project, model):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some of these tests pass even without fixing {{ relation }} to be {{ relation.render() }}. I think it has something to do with whether the template returns just the sql or whether it does the call. I added the tests in case this changes in the future, and updated the macros to use {{ relation.render() }}.

run_dbt(["run", "--empty", "--select", model])
Loading