From 6906eb0b32a7b729d8f053d674f8951ae2773a55 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 9 Feb 2024 10:23:02 -0500 Subject: [PATCH 01/61] [Redshift] Add tests for unit testing (#680) * first pass * fix dbt-common version * try to fix dbt-postgres version * fix import * array and json types as TODO * add json test, TestRedshiftUnitTestCaseInsensitivity, TestRedshiftUnitTestInvalidInput * restore requirements --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- dev-requirements.txt | 3 +- setup.py | 2 +- .../adapter/unit_testing/test_unit_testing.py | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/unit_testing/test_unit_testing.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 0c8c9b1a3..8814be848 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,8 @@ # install latest changes in dbt-core + dbt-postgres # TODO: how to switch from HEAD to x.y.latest branches after minor releases? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-postgres.git@main +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version diff --git a/setup.py b/setup.py index 4925bb04b..09b1cea88 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common<1.0", + "dbt-common>=0.1.0,<0.2.0", "dbt-adapters~=0.1.0a1", f"dbt-postgres~={_plugin_version()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..4d89c4b08 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,42 @@ +import pytest +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["1.0", "1.0"], + ["'1'", "1"], + ["'1'::numeric", "1"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + [ + """JSON_PARSE('{"bar": "baz", "balance": 7.77, "active": false}')""", + """'{"bar": "baz", "balance": 7.77, "active": false}'""", + ], + # TODO: array types + # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + # ["ARRAY[1.0,2.0,3.0]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY[1::numeric,2::numeric,3::numeric]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + # ["ARRAY[true,true,false]", """'{true, true, false}'"""], + # ["ARRAY[DATE '2020-01-02']", """'{"2020-01-02"}'"""], + # ["ARRAY[TIMESTAMP '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + # ["ARRAY[TIMESTAMPTZ '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + ] + + +class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestRedshiftUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass From b79ced39b5e81421a255cd3bec9992463d6f42bb Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:26:47 -0500 Subject: [PATCH 02/61] Implement relation filtering on get_catalog macro (#692) * changelog * split out get_catalog by schema and by relation * fix column references, reverse union order so nulls load second, remove column union and join on multiple columns * turn on get_catalog by relation capability * add test cases that demonstrate both catalog methods work * fix broken import in materialized view tests --- .../unreleased/Features-20231214-195655.yaml | 6 + dbt/adapters/redshift/impl.py | 5 + dbt/include/redshift/macros/catalog.sql | 258 ------------------ .../redshift/macros/catalog/by_relation.sql | 82 ++++++ .../redshift/macros/catalog/by_schema.sql | 70 +++++ .../redshift/macros/catalog/catalog.sql | 176 ++++++++++++ .../adapter/catalog_tests/test_get_catalog.py | 142 ++++++++++ .../test_materialized_views.py | 2 +- 8 files changed, 482 insertions(+), 259 deletions(-) create mode 100644 .changes/unreleased/Features-20231214-195655.yaml delete mode 100644 dbt/include/redshift/macros/catalog.sql create mode 100644 dbt/include/redshift/macros/catalog/by_relation.sql create mode 100644 dbt/include/redshift/macros/catalog/by_schema.sql create mode 100644 dbt/include/redshift/macros/catalog/catalog.sql create mode 100644 tests/functional/adapter/catalog_tests/test_get_catalog.py diff --git a/.changes/unreleased/Features-20231214-195655.yaml b/.changes/unreleased/Features-20231214-195655.yaml new file mode 100644 index 000000000..93f372c67 --- /dev/null +++ b/.changes/unreleased/Features-20231214-195655.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support limiting get_catalog by object name +time: 2023-12-14T19:56:55.124051-05:00 +custom: + Author: mikealfare + Issue: "625" diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index b41308db0..8aee75e62 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -6,6 +6,7 @@ from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available +from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support from dbt.adapters.sql import SQLAdapter from dbt.adapters.contracts.connection import AdapterResponse from dbt.adapters.events.logging import AdapterLogger @@ -53,6 +54,10 @@ class RedshiftAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, } + _capabilities = CapabilityDict( + {Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)} + ) + @classmethod def date_function(cls): return "getdate()" diff --git a/dbt/include/redshift/macros/catalog.sql b/dbt/include/redshift/macros/catalog.sql deleted file mode 100644 index 69dc71713..000000000 --- a/dbt/include/redshift/macros/catalog.sql +++ /dev/null @@ -1,258 +0,0 @@ - -{% macro redshift__get_base_catalog(information_schema, schemas) -%} - {%- call statement('base_catalog', fetch_result=True) -%} - {% set database = information_schema.database %} - {{ adapter.verify_database(database) }} - - with late_binding as ( - select - '{{ database }}'::varchar as table_database, - table_schema, - table_name, - 'LATE BINDING VIEW'::varchar as table_type, - null::text as table_comment, - - column_name, - column_index, - column_type, - null::text as column_comment - from pg_get_late_binding_view_cols() - cols(table_schema name, table_name name, column_name name, - column_type varchar, - column_index int) - order by "column_index" - ), - - materialized_views as ( - select - table_schema as nspname, - table_name as relname - from information_schema.views - where ( - {%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and table_catalog = '{{ database }}' - and view_definition ilike '%create materialized view%' - ), - - early_binding as ( - select - '{{ database }}'::varchar as table_database, - sch.nspname as table_schema, - tbl.relname as table_name, - case - when tbl.relkind = 'v' and materialized_views.relname is not null then 'MATERIALIZED VIEW' - when tbl.relkind = 'v' then 'VIEW' - else 'BASE TABLE' - end as table_type, - tbl_desc.description as table_comment, - col.attname as column_name, - col.attnum as column_index, - pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, - col_desc.description as column_comment - - from pg_catalog.pg_namespace sch - join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid - join pg_catalog.pg_attribute col on col.attrelid = tbl.oid - left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) - left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) - left outer join materialized_views on (materialized_views.nspname = sch.nspname and materialized_views.relname = tbl.relname) - where ( - {%- for schema in schemas -%} - upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and tbl.relkind in ('r', 'v', 'f', 'p') - and col.attnum > 0 - and not col.attisdropped - ), - - table_owners as ( - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - tablename as table_name, - tableowner as table_owner - - from pg_tables - - union all - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - viewname as table_name, - viewowner as table_owner - - from pg_views - - ), - - unioned as ( - - select * - from early_binding - - union all - - select * - from late_binding - - ) - - select *, - table_database || '.' || table_schema || '.' || table_name as table_id - - from unioned - join table_owners using (table_database, table_schema, table_name) - - where ( - {%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - order by "column_index" - {%- endcall -%} - - {{ return(load_result('base_catalog').table) }} -{%- endmacro %} - -{% macro redshift__get_extended_catalog(schemas) %} - {%- call statement('extended_catalog', fetch_result=True) -%} - - select - "database" || '.' || "schema" || '.' || "table" as table_id, - - 'Encoded'::text as "stats:encoded:label", - encoded as "stats:encoded:value", - 'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description", - true as "stats:encoded:include", - - 'Dist Style' as "stats:diststyle:label", - diststyle as "stats:diststyle:value", - 'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description", - true as "stats:diststyle:include", - - 'Sort Key 1' as "stats:sortkey1:label", - -- handle 0xFF byte in response for interleaved sort styles - case - when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text - else sortkey1 - end as "stats:sortkey1:value", - 'First column in the sort key.'::text as "stats:sortkey1:description", - (sortkey1 is not null) as "stats:sortkey1:include", - - 'Max Varchar' as "stats:max_varchar:label", - max_varchar as "stats:max_varchar:value", - 'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description", - true as "stats:max_varchar:include", - - -- exclude this, as the data is strangely returned with null-byte characters - 'Sort Key 1 Encoding' as "stats:sortkey1_enc:label", - sortkey1_enc as "stats:sortkey1_enc:value", - 'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description", - false as "stats:sortkey1_enc:include", - - '# Sort Keys' as "stats:sortkey_num:label", - sortkey_num as "stats:sortkey_num:value", - 'Number of columns defined as sort keys.' as "stats:sortkey_num:description", - (sortkey_num > 0) as "stats:sortkey_num:include", - - 'Approximate Size' as "stats:size:label", - size * 1000000 as "stats:size:value", - 'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description", - true as "stats:size:include", - - 'Disk Utilization' as "stats:pct_used:label", - pct_used / 100.0 as "stats:pct_used:value", - 'Percent of available space that is used by the table.'::text as "stats:pct_used:description", - true as "stats:pct_used:include", - - 'Unsorted %' as "stats:unsorted:label", - unsorted / 100.0 as "stats:unsorted:value", - 'Percent of unsorted rows in the table.'::text as "stats:unsorted:description", - (unsorted is not null) as "stats:unsorted:include", - - 'Stats Off' as "stats:stats_off:label", - stats_off as "stats:stats_off:value", - 'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description", - true as "stats:stats_off:include", - - 'Approximate Row Count' as "stats:rows:label", - tbl_rows as "stats:rows:value", - 'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description", - true as "stats:rows:include", - - 'Sort Key Skew' as "stats:skew_sortkey1:label", - skew_sortkey1 as "stats:skew_sortkey1:value", - 'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description", - (skew_sortkey1 is not null) as "stats:skew_sortkey1:include", - - 'Skew Rows' as "stats:skew_rows:label", - skew_rows as "stats:skew_rows:value", - 'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description", - (skew_rows is not null) as "stats:skew_rows:include" - - from svv_table_info - where ( - {%- for schema in schemas -%} - upper(schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - {%- endcall -%} - - {{ return(load_result('extended_catalog').table) }} - -{% endmacro %} - -{% macro redshift__can_select_from(table_name) %} - - {%- call statement('has_table_privilege', fetch_result=True) -%} - - select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select - - {%- endcall -%} - - {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %} - {{ return(can_select) }} - -{% endmacro %} - -{% macro redshift__no_svv_table_info_warning() %} - - {% set msg %} - - Warning: The database user "{{ target.user }}" has insufficient permissions to - query the "svv_table_info" table. Please grant SELECT permissions on this table - to the "{{ target.user }}" user to fetch extended table details from Redshift. - - {% endset %} - - {{ log(msg, info=True) }} - -{% endmacro %} - - -{% macro redshift__get_catalog(information_schema, schemas) %} - - {#-- Compute a left-outer join in memory. Some Redshift queries are - -- leader-only, and cannot be joined to other compute-based queries #} - - {% set catalog = redshift__get_base_catalog(information_schema, schemas) %} - - {% set select_extended = redshift__can_select_from('svv_table_info') %} - {% if select_extended %} - {% set extended_catalog = redshift__get_extended_catalog(schemas) %} - {% set catalog = catalog.join(extended_catalog, 'table_id') %} - {% else %} - {{ redshift__no_svv_table_info_warning() }} - {% endif %} - - {{ return(catalog.exclude(['table_id'])) }} - -{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/by_relation.sql b/dbt/include/redshift/macros/catalog/by_relation.sql new file mode 100644 index 000000000..d0d79c65a --- /dev/null +++ b/dbt/include/redshift/macros/catalog/by_relation.sql @@ -0,0 +1,82 @@ +{% macro redshift__get_catalog_relations(information_schema, relations) -%} + + {% set database = information_schema.database %} + {{ adapter.verify_database(database) }} + + {#-- Compute a left-outer join in memory. Some Redshift queries are + -- leader-only, and cannot be joined to other compute-based queries #} + + {% set catalog = _redshift__get_base_catalog_by_relation(database, relations) %} + + {% set select_extended = redshift__can_select_from('svv_table_info') %} + {% if select_extended %} + {% set extended_catalog = _redshift__get_extended_catalog_by_relation(relations) %} + {% set catalog = catalog.join(extended_catalog, ['table_schema', 'table_name']) %} + {% else %} + {{ redshift__no_svv_table_info_warning() }} + {% endif %} + + {{ return(catalog) }} + +{% endmacro %} + + +{% macro _redshift__get_base_catalog_by_relation(database, relations) -%} + {%- call statement('base_catalog', fetch_result=True) -%} + with + late_binding as ({{ _redshift__get_late_binding_by_relation_sql(relations) }}), + early_binding as ({{ _redshift__get_early_binding_by_relation_sql(database, relations) }}), + unioned as (select * from early_binding union all select * from late_binding), + table_owners as ({{ redshift__get_table_owners_sql() }}) + select '{{ database }}' as table_database, * + from unioned + join table_owners using (table_schema, table_name) + order by "column_index" + {%- endcall -%} + {{ return(load_result('base_catalog').table) }} +{%- endmacro %} + + +{% macro _redshift__get_late_binding_by_relation_sql(relations) %} + {{ redshift__get_late_binding_sql() }} + where ( + {%- for relation in relations -%} + ( + upper(table_schema) = upper('{{ relation.schema }}') + and upper(table_name) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_early_binding_by_relation_sql(database, relations) %} + {{ redshift__get_early_binding_sql(database) }} + and ( + {%- for relation in relations -%} + ( + upper(sch.nspname) = upper('{{ relation.schema }}') + and upper(tbl.relname) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_extended_catalog_by_relation(relations) %} + {%- call statement('extended_catalog', fetch_result=True) -%} + {{ redshift__get_extended_catalog_sql() }} + where ( + {%- for relation in relations -%} + ( + upper("schema") = upper('{{ relation.schema }}') + and upper("table") = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {%- endcall -%} + {{ return(load_result('extended_catalog').table) }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/by_schema.sql b/dbt/include/redshift/macros/catalog/by_schema.sql new file mode 100644 index 000000000..99325f765 --- /dev/null +++ b/dbt/include/redshift/macros/catalog/by_schema.sql @@ -0,0 +1,70 @@ +{% macro redshift__get_catalog(information_schema, schemas) %} + + {% set database = information_schema.database %} + {{ adapter.verify_database(database) }} + + {#-- Compute a left-outer join in memory. Some Redshift queries are + -- leader-only, and cannot be joined to other compute-based queries #} + + {% set catalog = _redshift__get_base_catalog_by_schema(database, schemas) %} + + {% set select_extended = redshift__can_select_from('svv_table_info') %} + {% if select_extended %} + {% set extended_catalog = _redshift__get_extended_catalog_by_schema(schemas) %} + {% set catalog = catalog.join(extended_catalog, ['table_schema', 'table_name']) %} + {% else %} + {{ redshift__no_svv_table_info_warning() }} + {% endif %} + + {{ return(catalog) }} + +{% endmacro %} + + +{% macro _redshift__get_base_catalog_by_schema(database, schemas) -%} + {%- call statement('base_catalog', fetch_result=True) -%} + with + late_binding as ({{ _redshift__get_late_binding_by_schema_sql(schemas) }}), + early_binding as ({{ _redshift__get_early_binding_by_schema_sql(database, schemas) }}), + unioned as (select * from early_binding union all select * from late_binding), + table_owners as ({{ redshift__get_table_owners_sql() }}) + select '{{ database }}' as table_database, * + from unioned + join table_owners using (table_schema, table_name) + order by "column_index" + {%- endcall -%} + {{ return(load_result('base_catalog').table) }} +{%- endmacro %} + + +{% macro _redshift__get_late_binding_by_schema_sql(schemas) %} + {{ redshift__get_late_binding_sql() }} + where ( + {%- for schema in schemas -%} + upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_early_binding_by_schema_sql(database, schemas) %} + {{ redshift__get_early_binding_sql(database) }} + and ( + {%- for schema in schemas -%} + upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_extended_catalog_by_schema(schemas) %} + {%- call statement('extended_catalog', fetch_result=True) -%} + {{ redshift__get_extended_catalog_sql() }} + where ( + {%- for schema in schemas -%} + upper("schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {%- endcall -%} + {{ return(load_result('extended_catalog').table) }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/catalog.sql b/dbt/include/redshift/macros/catalog/catalog.sql new file mode 100644 index 000000000..694a9441b --- /dev/null +++ b/dbt/include/redshift/macros/catalog/catalog.sql @@ -0,0 +1,176 @@ +{% macro redshift__get_late_binding_sql() %} + select + table_schema, + table_name, + 'LATE BINDING VIEW'::varchar as table_type, + null::text as table_comment, + column_name, + column_index, + column_type, + null::text as column_comment + from pg_get_late_binding_view_cols() + cols( + table_schema name, + table_name name, + column_name name, + column_type varchar, + column_index int + ) +{% endmacro %} + + +{% macro redshift__get_early_binding_sql(database) %} + select + sch.nspname as table_schema, + tbl.relname as table_name, + case + when tbl.relkind = 'v' and mat_views.table_name is not null then 'MATERIALIZED VIEW' + when tbl.relkind = 'v' then 'VIEW' + else 'BASE TABLE' + end as table_type, + tbl_desc.description as table_comment, + col.attname as column_name, + col.attnum as column_index, + pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, + col_desc.description as column_comment + from pg_catalog.pg_namespace sch + join pg_catalog.pg_class tbl + on tbl.relnamespace = sch.oid + join pg_catalog.pg_attribute col + on col.attrelid = tbl.oid + left outer join pg_catalog.pg_description tbl_desc + on tbl_desc.objoid = tbl.oid + and tbl_desc.objsubid = 0 + left outer join pg_catalog.pg_description col_desc + on col_desc.objoid = tbl.oid + and col_desc.objsubid = col.attnum + left outer join information_schema.views mat_views + on mat_views.table_schema = sch.nspname + and mat_views.table_name = tbl.relname + and mat_views.view_definition ilike '%create materialized view%' + and mat_views.table_catalog = '{{ database }}' + where tbl.relkind in ('r', 'v', 'f', 'p') + and col.attnum > 0 + and not col.attisdropped +{% endmacro %} + + +{% macro redshift__get_table_owners_sql() %} + select + schemaname as table_schema, + tablename as table_name, + tableowner as table_owner + from pg_tables + union all + select + schemaname as table_schema, + viewname as table_name, + viewowner as table_owner + from pg_views +{% endmacro %} + + +{% macro redshift__get_extended_catalog_sql() %} + select + "schema" as table_schema, + "table" as table_name, + + 'Encoded'::text as "stats:encoded:label", + encoded as "stats:encoded:value", + 'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description", + true as "stats:encoded:include", + + 'Dist Style' as "stats:diststyle:label", + diststyle as "stats:diststyle:value", + 'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description", + true as "stats:diststyle:include", + + 'Sort Key 1' as "stats:sortkey1:label", + -- handle 0xFF byte in response for interleaved sort styles + case + when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text + else sortkey1 + end as "stats:sortkey1:value", + 'First column in the sort key.'::text as "stats:sortkey1:description", + (sortkey1 is not null) as "stats:sortkey1:include", + + 'Max Varchar' as "stats:max_varchar:label", + max_varchar as "stats:max_varchar:value", + 'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description", + true as "stats:max_varchar:include", + + -- exclude this, as the data is strangely returned with null-byte characters + 'Sort Key 1 Encoding' as "stats:sortkey1_enc:label", + sortkey1_enc as "stats:sortkey1_enc:value", + 'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description", + false as "stats:sortkey1_enc:include", + + '# Sort Keys' as "stats:sortkey_num:label", + sortkey_num as "stats:sortkey_num:value", + 'Number of columns defined as sort keys.' as "stats:sortkey_num:description", + (sortkey_num > 0) as "stats:sortkey_num:include", + + 'Approximate Size' as "stats:size:label", + size * 1000000 as "stats:size:value", + 'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description", + true as "stats:size:include", + + 'Disk Utilization' as "stats:pct_used:label", + pct_used / 100.0 as "stats:pct_used:value", + 'Percent of available space that is used by the table.'::text as "stats:pct_used:description", + true as "stats:pct_used:include", + + 'Unsorted %' as "stats:unsorted:label", + unsorted / 100.0 as "stats:unsorted:value", + 'Percent of unsorted rows in the table.'::text as "stats:unsorted:description", + (unsorted is not null) as "stats:unsorted:include", + + 'Stats Off' as "stats:stats_off:label", + stats_off as "stats:stats_off:value", + 'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description", + true as "stats:stats_off:include", + + 'Approximate Row Count' as "stats:rows:label", + tbl_rows as "stats:rows:value", + 'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description", + true as "stats:rows:include", + + 'Sort Key Skew' as "stats:skew_sortkey1:label", + skew_sortkey1 as "stats:skew_sortkey1:value", + 'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description", + (skew_sortkey1 is not null) as "stats:skew_sortkey1:include", + + 'Skew Rows' as "stats:skew_rows:label", + skew_rows as "stats:skew_rows:value", + 'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description", + (skew_rows is not null) as "stats:skew_rows:include" + + from svv_table_info +{% endmacro %} + + +{% macro redshift__can_select_from(table_name) %} + + {%- call statement('has_table_privilege', fetch_result=True) -%} + select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select + {%- endcall -%} + + {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %} + {{ return(can_select) }} + +{% endmacro %} + + +{% macro redshift__no_svv_table_info_warning() %} + + {% set msg %} + + Warning: The database user "{{ target.user }}" has insufficient permissions to + query the "svv_table_info" table. Please grant SELECT permissions on this table + to the "{{ target.user }}" user to fetch extended table details from Redshift. + + {% endset %} + + {{ log(msg, info=True) }} + +{% endmacro %} diff --git a/tests/functional/adapter/catalog_tests/test_get_catalog.py b/tests/functional/adapter/catalog_tests/test_get_catalog.py new file mode 100644 index 000000000..519c806c4 --- /dev/null +++ b/tests/functional/adapter/catalog_tests/test_get_catalog.py @@ -0,0 +1,142 @@ +from dbt.adapters.contracts.relation import RelationType +from dbt.tests.util import get_connection +import pytest + + +class TestGetCatalog: + @pytest.fixture(scope="class") + def my_schema(self, project, adapter): + schema = adapter.Relation.create( + database=project.database, + schema=project.test_schema, + identifier="", + ) + yield schema + + @pytest.fixture(scope="class") + def my_seed(self, adapter, my_schema): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_seed", + type=RelationType.Table, + ) + with get_connection(adapter): + sql = f""" + create table {relation.database}.{relation.schema}.{relation.identifier} ( + id integer, + value integer, + record_valid_date timestamp + ); + insert into {relation.database}.{relation.schema}.{relation.identifier} + (id, value, record_valid_date) values + (1,100,'2023-01-01 00:00:00'), + (2,200,'2023-01-02 00:00:00'), + (3,300,'2023-01-02 00:00:00') + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_table(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_table", + type=RelationType.Table, + ) + with get_connection(adapter): + sql = f""" + create table {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_view(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_view", + type=RelationType.View, + ) + with get_connection(adapter): + sql = f""" + create view {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_materialized_view(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_materialized_view", + type=RelationType.MaterializedView, + ) + with get_connection(adapter): + sql = f""" + create materialized view {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_information_schema(self, adapter, my_schema): + yield adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="INFORMATION_SCHEMA", + ).information_schema() + + def test_get_one_catalog_by_relations( + self, + adapter, + my_schema, + my_seed, + my_table, + my_view, + my_materialized_view, + my_information_schema, + ): + my_schemas = frozenset({(my_schema.database, my_schema.schema)}) + my_relations = [my_seed, my_table, my_view, my_materialized_view] + with get_connection(adapter): + catalog = adapter._get_one_catalog_by_relations( + information_schema=my_information_schema, + relations=my_relations, + used_schemas=my_schemas, + ) + # my_seed, my_table, my_view, my_materialized_view each have 3 cols = 12 cols + # my_materialized_view creates an underlying table with 2 additional = 5 cols + # note the underlying table is missing as it's not in `my_relations` + assert len(catalog) == 12 + + def test_get_one_catalog_by_schemas( + self, + adapter, + my_schema, + my_seed, + my_table, + my_view, + my_materialized_view, + my_information_schema, + ): + my_schemas = frozenset({(my_schema.database, my_schema.schema)}) + with get_connection(adapter): + catalog = adapter._get_one_catalog( + information_schema=my_information_schema, + schemas={my_schema.schema}, + used_schemas=my_schemas, + ) + # my_seed, my_table, my_view, my_materialized_view each have 3 cols = 12 cols + # my_materialized_view creates an underlying table with 2 additional = 5 cols + assert len(catalog) == 17 diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index 63bcede61..64f697e77 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -3,7 +3,7 @@ import pytest from dbt.adapters.base.relation import BaseRelation -from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.adapters.contracts.relation import OnConfigurationChangeOption from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic from dbt.tests.adapter.materialized_view.changes import ( From d77f5ee9561bf11eae56b2832340e3c11005b6f9 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:26:52 -0500 Subject: [PATCH 03/61] Implementation of metadata-based freshness (#694) * changelog * turn on metadata-based source freshness * add tests for metadata-based source freshness * add metadata-based source freshness sql stub * mark metadata-based source freshness as unsupported * add test to show existing method is working * turn on metadata-based source freshness * add dbt-common to the dev-requirements so it comes from github * add test case with no last modified date column * add metadata-based source freshness query * restrict to just insert steps --- .../unreleased/Features-20231219-120533.yaml | 6 +++ dbt/adapters/redshift/impl.py | 5 ++- .../metadata/relation_last_modified.sql | 29 +++++++++++++ dev-requirements.txt | 6 +-- .../adapter/sources_freshness_tests/files.py | 38 +++++++++++++++++ .../test_get_relation_last_modified.py | 42 +++++++++++++++++++ 6 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Features-20231219-120533.yaml create mode 100644 dbt/include/redshift/macros/metadata/relation_last_modified.sql create mode 100644 tests/functional/adapter/sources_freshness_tests/files.py create mode 100644 tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py diff --git a/.changes/unreleased/Features-20231219-120533.yaml b/.changes/unreleased/Features-20231219-120533.yaml new file mode 100644 index 000000000..15b5ba1f1 --- /dev/null +++ b/.changes/unreleased/Features-20231219-120533.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for checking table-last-modified by metadata +time: 2023-12-19T12:05:33.784649-05:00 +custom: + Author: mikealfare + Issue: "615" diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 8aee75e62..a77601895 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -55,7 +55,10 @@ class RedshiftAdapter(SQLAdapter): } _capabilities = CapabilityDict( - {Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)} + { + Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), + } ) @classmethod diff --git a/dbt/include/redshift/macros/metadata/relation_last_modified.sql b/dbt/include/redshift/macros/metadata/relation_last_modified.sql new file mode 100644 index 000000000..f21299c72 --- /dev/null +++ b/dbt/include/redshift/macros/metadata/relation_last_modified.sql @@ -0,0 +1,29 @@ +{% macro redshift__get_relation_last_modified(information_schema, relations) -%} + + {%- call statement('last_modified', fetch_result=True) -%} + select + ns.nspname as "schema", + c.relname as identifier, + max(qd.start_time) as last_modified, + {{ current_timestamp() }} as snapshotted_at + from pg_class c + join pg_namespace ns + on ns.oid = c.relnamespace + join sys_query_detail qd + on qd.table_id = c.oid + where qd.step_name = 'insert' + and ( + {%- for relation in relations -%} + ( + upper(ns.nspname) = upper('{{ relation.schema }}') + and upper(c.relname) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + group by 1, 2, 4 + {%- endcall -%} + + {{ return(load_result('last_modified')) }} + +{% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 8814be848..6d29276cf 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,9 @@ # install latest changes in dbt-core + dbt-postgres -# TODO: how to switch from HEAD to x.y.latest branches after minor releases? -git+https://github.com/dbt-labs/dbt-postgres.git@main -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-common.git +git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core +git+https://github.com/dbt-labs/dbt-postgres.git # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/tests/functional/adapter/sources_freshness_tests/files.py b/tests/functional/adapter/sources_freshness_tests/files.py new file mode 100644 index 000000000..f2dd6fbbe --- /dev/null +++ b/tests/functional/adapter/sources_freshness_tests/files.py @@ -0,0 +1,38 @@ +SCHEMA_YML = """version: 2 +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_source_no_last_modified + - name: test_source_last_modified + loaded_at_field: last_modified +""" + +SEED_TEST_SOURCE_NO_LAST_MODIFIED_CSV = """ +id,name +1,Martin +2,Jeter +3,Ruth +4,Gehrig +5,DiMaggio +6,Torre +7,Mantle +8,Berra +9,Maris +""".strip() + +SEED_TEST_SOURCE_LAST_MODIFIED_CSV = """ +id,name,last_modified +1,Martin,2023-01-01 00:00:00 +2,Jeter,2023-02-01 00:00:00 +3,Ruth,2023-03-01 00:00:00 +4,Gehrig,2023-04-01 00:00:00 +5,DiMaggio,2023-05-01 00:00:00 +6,Torre,2023-06-01 00:00:00 +7,Mantle,2023-07-01 00:00:00 +8,Berra,2023-08-01 00:00:00 +9,Maris,2023-09-01 00:00:00 +""".strip() diff --git a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py new file mode 100644 index 000000000..6a77d22ae --- /dev/null +++ b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py @@ -0,0 +1,42 @@ +import os + +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.sources_freshness_tests import files + + +class TestGetLastRelationModified: + @pytest.fixture(scope="class") + def seeds(self): + return { + "test_source_no_last_modified.csv": files.SEED_TEST_SOURCE_NO_LAST_MODIFIED_CSV, + "test_source_last_modified.csv": files.SEED_TEST_SOURCE_LAST_MODIFIED_CSV, + } + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": files.SCHEMA_YML} + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + # we need the schema name for the sources section + os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema + run_dbt(["seed"]) + yield + del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + + @pytest.mark.parametrize( + "source,status,expect_pass", + [ + ("test_source.test_source_no_last_modified", "pass", True), + ("test_source.test_source_last_modified", "error", False), # stale + ], + ) + def test_get_last_relation_modified(self, project, source, status, expect_pass): + results = run_dbt( + ["source", "freshness", "--select", f"source:{source}"], expect_pass=expect_pass + ) + assert len(results) == 1 + result = results[0] + assert result.status == status From 95961f2e8243abe582560a76660c0fcb78222f9d Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:23:32 -0800 Subject: [PATCH 04/61] update dbt-common dependency to <2.0 (#725) * update dbt-common dependency to <2.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 09b1cea88..b979c47a9 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=0.1.0,<0.2.0", - "dbt-adapters~=0.1.0a1", + "dbt-common<2.0", + "dbt-adapters<2.0", f"dbt-postgres~={_plugin_version()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. From 5c02c55f4a5c974ffdf815e62cb21008a12e54bf Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:26:56 -0800 Subject: [PATCH 05/61] update install_requires to allow for pre-release common/adapters (#727) --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b979c47a9..a1b63306b 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common<2.0", - "dbt-adapters<2.0", + "dbt-common>=0.1.0a1,<2.0", + "dbt-adapters>=0.1.0a1,<2.0", f"dbt-postgres~={_plugin_version()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. From c91eed79db4fabee4b4b9e876ef9628d28c73dba Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Fri, 1 Mar 2024 23:46:35 +0000 Subject: [PATCH 06/61] Bumping version to 1.8.0b1 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.0-b1.md | 49 ++++++++++++++++++ .../Dependencies-20230912-120620.yaml | 0 .../Dependencies-20231002-164037.yaml | 0 .../Dependencies-20231009-192801.yaml | 0 .../Dependencies-20231010-195348.yaml | 0 .../Dependencies-20231013-190517.yaml | 0 .../Dependencies-20231017-191545.yaml | 0 .../Dependencies-20231027-173152.yaml | 0 .../Dependencies-20231030-193514.yaml | 0 .../Dependencies-20231108-190800.yaml | 0 .../Dependencies-20231110-192349.yaml | 0 .../Dependencies-20231113-195504.yaml | 0 .../Dependencies-20231116-194405.yaml | 0 .../Dependencies-20231127-201640.yaml | 0 .../Dependencies-20231127-201942.yaml | 0 .../Dependencies-20231128-194822.yaml | 0 .../Dependencies-20231129-195044.yaml | 0 .../Dependencies-20231130-044332.yaml | 0 .../Dependencies-20231204-193730.yaml | 0 .../Dependencies-20231212-195417.yaml | 0 .../Dependencies-20240118-095025.yaml | 0 .../Dependencies-20240124-111727.yaml | 0 .../Features-20231030-101055.yaml | 0 .../Features-20231214-195655.yaml | 0 .../Features-20231219-120533.yaml | 0 .../Fixes-20231025-203732.yaml | 0 .../Fixes-20231026-164623.yaml | 0 .../Fixes-20231030-234315.yaml | 0 .../Fixes-20231103-181357.yaml | 0 .../Fixes-20240206-132326.yaml | 0 .../Under the Hood-20231119-132157.yaml | 0 .../Under the Hood-20240102-152425.yaml | 0 CHANGELOG.md | 51 +++++++++++++++++++ dbt/adapters/redshift/__version__.py | 2 +- 35 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 .changes/1.8.0-b1.md rename .changes/{unreleased => 1.8.0}/Dependencies-20230912-120620.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231002-164037.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231009-192801.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231010-195348.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231013-190517.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231017-191545.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231027-173152.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231030-193514.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231108-190800.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231110-192349.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231113-195504.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231116-194405.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231127-201640.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231127-201942.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231128-194822.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231129-195044.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231130-044332.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231204-193730.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231212-195417.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240118-095025.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240124-111727.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20231030-101055.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20231214-195655.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20231219-120533.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231025-203732.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231026-164623.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231030-234315.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231103-181357.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240206-132326.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20231119-132157.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240102-152425.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f54032556..54264b726 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0a1 +current_version = 1.8.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b1.md b/.changes/1.8.0-b1.md new file mode 100644 index 000000000..e8e4cbd2b --- /dev/null +++ b/.changes/1.8.0-b1.md @@ -0,0 +1,49 @@ +## dbt-redshift 1.8.0-b1 - March 01, 2024 + +### Features + +- allow user to set debug level for redshift-connector via env var ([#650](https://github.com/dbt-labs/dbt-redshift/issues/650)) +- Support limiting get_catalog by object name ([#625](https://github.com/dbt-labs/dbt-redshift/issues/625)) +- Add support for checking table-last-modified by metadata ([#615](https://github.com/dbt-labs/dbt-redshift/issues/615)) + +### Fixes + +- Fix parsing of database results for materialized view auto refresh ([#643](https://github.com/dbt-labs/dbt-redshift/issues/643)) +- Fix describe_materialized_view for Redshift Serverless ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) +- Catalog queries assign the appropriate type for materialized views ([#652](https://github.com/dbt-labs/dbt-redshift/issues/652)) +- Fix handling of `backup` parameter during config change monitoring ([#621](https://github.com/dbt-labs/dbt-redshift/issues/621)) +- Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes external tables ([#710](https://github.com/dbt-labs/dbt-redshift/issues/710)) + +### Under the Hood + +- Add tests for --empty flag ([#667](https://github.com/dbt-labs/dbt-redshift/issues/667)) +- Update base adapter references as part of decoupling migration ([#698](https://github.com/dbt-labs/dbt-redshift/issues/698)) + +### Dependencies + +- Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) +- Update ddtrace requirement from ~=1.19 to ~=1.20 ([#622](https://github.com/dbt-labs/dbt-redshift/pull/622)) +- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#627](https://github.com/dbt-labs/dbt-redshift/pull/627)) +- Bump mypy from 1.5.1 to 1.6.0 ([#629](https://github.com/dbt-labs/dbt-redshift/pull/629)) +- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#634](https://github.com/dbt-labs/dbt-redshift/pull/634)) +- Update black requirement from ~=23.9 to ~=23.10 ([#636](https://github.com/dbt-labs/dbt-redshift/pull/636)) +- Bump mypy from 1.6.0 to 1.6.1 ([#648](https://github.com/dbt-labs/dbt-redshift/pull/648)) +- Update ddtrace requirement from ~=1.20 to ~=2.1 ([#651](https://github.com/dbt-labs/dbt-redshift/pull/651)) +- Update black requirement from ~=23.10 to ~=23.11 ([#660](https://github.com/dbt-labs/dbt-redshift/pull/660)) +- Bump mypy from 1.6.1 to 1.7.0 ([#662](https://github.com/dbt-labs/dbt-redshift/pull/662)) +- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#664](https://github.com/dbt-labs/dbt-redshift/pull/664)) +- Update ddtrace requirement from ~=2.1 to ~=2.2 ([#665](https://github.com/dbt-labs/dbt-redshift/pull/665)) +- Update ddtrace requirement from ~=2.2 to ~=2.3 ([#669](https://github.com/dbt-labs/dbt-redshift/pull/669)) +- Update wheel requirement from ~=0.41 to ~=0.42 ([#670](https://github.com/dbt-labs/dbt-redshift/pull/670)) +- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#672](https://github.com/dbt-labs/dbt-redshift/pull/672)) +- Remove direct boto3 dependency ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) +- Bump mypy from 1.7.0 to 1.7.1 ([#676](https://github.com/dbt-labs/dbt-redshift/pull/676)) +- Update freezegun requirement from ~=1.2 to ~=1.3 ([#681](https://github.com/dbt-labs/dbt-redshift/pull/681)) +- Update black requirement from ~=23.11 to ~=23.12 ([#688](https://github.com/dbt-labs/dbt-redshift/pull/688)) +- upgrade redshift driver to 2.0.918 ([#700](https://github.com/dbt-labs/dbt-redshift/pull/700)) +- Migrate to dbt-common and dbt-adapters ([#706](https://github.com/dbt-labs/dbt-redshift/pull/706)) + +### Contributors +- [@hexDoor](https://github.com/hexDoor) ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) +- [@reptillicus](https://github.com/reptillicus) ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) +- [@soksamnanglim](https://github.com/soksamnanglim) ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) diff --git a/.changes/unreleased/Dependencies-20230912-120620.yaml b/.changes/1.8.0/Dependencies-20230912-120620.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20230912-120620.yaml rename to .changes/1.8.0/Dependencies-20230912-120620.yaml diff --git a/.changes/unreleased/Dependencies-20231002-164037.yaml b/.changes/1.8.0/Dependencies-20231002-164037.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231002-164037.yaml rename to .changes/1.8.0/Dependencies-20231002-164037.yaml diff --git a/.changes/unreleased/Dependencies-20231009-192801.yaml b/.changes/1.8.0/Dependencies-20231009-192801.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231009-192801.yaml rename to .changes/1.8.0/Dependencies-20231009-192801.yaml diff --git a/.changes/unreleased/Dependencies-20231010-195348.yaml b/.changes/1.8.0/Dependencies-20231010-195348.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231010-195348.yaml rename to .changes/1.8.0/Dependencies-20231010-195348.yaml diff --git a/.changes/unreleased/Dependencies-20231013-190517.yaml b/.changes/1.8.0/Dependencies-20231013-190517.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231013-190517.yaml rename to .changes/1.8.0/Dependencies-20231013-190517.yaml diff --git a/.changes/unreleased/Dependencies-20231017-191545.yaml b/.changes/1.8.0/Dependencies-20231017-191545.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231017-191545.yaml rename to .changes/1.8.0/Dependencies-20231017-191545.yaml diff --git a/.changes/unreleased/Dependencies-20231027-173152.yaml b/.changes/1.8.0/Dependencies-20231027-173152.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231027-173152.yaml rename to .changes/1.8.0/Dependencies-20231027-173152.yaml diff --git a/.changes/unreleased/Dependencies-20231030-193514.yaml b/.changes/1.8.0/Dependencies-20231030-193514.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231030-193514.yaml rename to .changes/1.8.0/Dependencies-20231030-193514.yaml diff --git a/.changes/unreleased/Dependencies-20231108-190800.yaml b/.changes/1.8.0/Dependencies-20231108-190800.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231108-190800.yaml rename to .changes/1.8.0/Dependencies-20231108-190800.yaml diff --git a/.changes/unreleased/Dependencies-20231110-192349.yaml b/.changes/1.8.0/Dependencies-20231110-192349.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231110-192349.yaml rename to .changes/1.8.0/Dependencies-20231110-192349.yaml diff --git a/.changes/unreleased/Dependencies-20231113-195504.yaml b/.changes/1.8.0/Dependencies-20231113-195504.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231113-195504.yaml rename to .changes/1.8.0/Dependencies-20231113-195504.yaml diff --git a/.changes/unreleased/Dependencies-20231116-194405.yaml b/.changes/1.8.0/Dependencies-20231116-194405.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231116-194405.yaml rename to .changes/1.8.0/Dependencies-20231116-194405.yaml diff --git a/.changes/unreleased/Dependencies-20231127-201640.yaml b/.changes/1.8.0/Dependencies-20231127-201640.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231127-201640.yaml rename to .changes/1.8.0/Dependencies-20231127-201640.yaml diff --git a/.changes/unreleased/Dependencies-20231127-201942.yaml b/.changes/1.8.0/Dependencies-20231127-201942.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231127-201942.yaml rename to .changes/1.8.0/Dependencies-20231127-201942.yaml diff --git a/.changes/unreleased/Dependencies-20231128-194822.yaml b/.changes/1.8.0/Dependencies-20231128-194822.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231128-194822.yaml rename to .changes/1.8.0/Dependencies-20231128-194822.yaml diff --git a/.changes/unreleased/Dependencies-20231129-195044.yaml b/.changes/1.8.0/Dependencies-20231129-195044.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231129-195044.yaml rename to .changes/1.8.0/Dependencies-20231129-195044.yaml diff --git a/.changes/unreleased/Dependencies-20231130-044332.yaml b/.changes/1.8.0/Dependencies-20231130-044332.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231130-044332.yaml rename to .changes/1.8.0/Dependencies-20231130-044332.yaml diff --git a/.changes/unreleased/Dependencies-20231204-193730.yaml b/.changes/1.8.0/Dependencies-20231204-193730.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231204-193730.yaml rename to .changes/1.8.0/Dependencies-20231204-193730.yaml diff --git a/.changes/unreleased/Dependencies-20231212-195417.yaml b/.changes/1.8.0/Dependencies-20231212-195417.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231212-195417.yaml rename to .changes/1.8.0/Dependencies-20231212-195417.yaml diff --git a/.changes/unreleased/Dependencies-20240118-095025.yaml b/.changes/1.8.0/Dependencies-20240118-095025.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240118-095025.yaml rename to .changes/1.8.0/Dependencies-20240118-095025.yaml diff --git a/.changes/unreleased/Dependencies-20240124-111727.yaml b/.changes/1.8.0/Dependencies-20240124-111727.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240124-111727.yaml rename to .changes/1.8.0/Dependencies-20240124-111727.yaml diff --git a/.changes/unreleased/Features-20231030-101055.yaml b/.changes/1.8.0/Features-20231030-101055.yaml similarity index 100% rename from .changes/unreleased/Features-20231030-101055.yaml rename to .changes/1.8.0/Features-20231030-101055.yaml diff --git a/.changes/unreleased/Features-20231214-195655.yaml b/.changes/1.8.0/Features-20231214-195655.yaml similarity index 100% rename from .changes/unreleased/Features-20231214-195655.yaml rename to .changes/1.8.0/Features-20231214-195655.yaml diff --git a/.changes/unreleased/Features-20231219-120533.yaml b/.changes/1.8.0/Features-20231219-120533.yaml similarity index 100% rename from .changes/unreleased/Features-20231219-120533.yaml rename to .changes/1.8.0/Features-20231219-120533.yaml diff --git a/.changes/unreleased/Fixes-20231025-203732.yaml b/.changes/1.8.0/Fixes-20231025-203732.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231025-203732.yaml rename to .changes/1.8.0/Fixes-20231025-203732.yaml diff --git a/.changes/unreleased/Fixes-20231026-164623.yaml b/.changes/1.8.0/Fixes-20231026-164623.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231026-164623.yaml rename to .changes/1.8.0/Fixes-20231026-164623.yaml diff --git a/.changes/unreleased/Fixes-20231030-234315.yaml b/.changes/1.8.0/Fixes-20231030-234315.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231030-234315.yaml rename to .changes/1.8.0/Fixes-20231030-234315.yaml diff --git a/.changes/unreleased/Fixes-20231103-181357.yaml b/.changes/1.8.0/Fixes-20231103-181357.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231103-181357.yaml rename to .changes/1.8.0/Fixes-20231103-181357.yaml diff --git a/.changes/unreleased/Fixes-20240206-132326.yaml b/.changes/1.8.0/Fixes-20240206-132326.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240206-132326.yaml rename to .changes/1.8.0/Fixes-20240206-132326.yaml diff --git a/.changes/unreleased/Under the Hood-20231119-132157.yaml b/.changes/1.8.0/Under the Hood-20231119-132157.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20231119-132157.yaml rename to .changes/1.8.0/Under the Hood-20231119-132157.yaml diff --git a/.changes/unreleased/Under the Hood-20240102-152425.yaml b/.changes/1.8.0/Under the Hood-20240102-152425.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240102-152425.yaml rename to .changes/1.8.0/Under the Hood-20240102-152425.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 04a8e7db1..2aebed2a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,57 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.8.0-b1 - March 01, 2024 + +### Features + +- allow user to set debug level for redshift-connector via env var ([#650](https://github.com/dbt-labs/dbt-redshift/issues/650)) +- Support limiting get_catalog by object name ([#625](https://github.com/dbt-labs/dbt-redshift/issues/625)) +- Add support for checking table-last-modified by metadata ([#615](https://github.com/dbt-labs/dbt-redshift/issues/615)) + +### Fixes + +- Fix parsing of database results for materialized view auto refresh ([#643](https://github.com/dbt-labs/dbt-redshift/issues/643)) +- Fix describe_materialized_view for Redshift Serverless ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) +- Catalog queries assign the appropriate type for materialized views ([#652](https://github.com/dbt-labs/dbt-redshift/issues/652)) +- Fix handling of `backup` parameter during config change monitoring ([#621](https://github.com/dbt-labs/dbt-redshift/issues/621)) +- Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes external tables ([#710](https://github.com/dbt-labs/dbt-redshift/issues/710)) + +### Under the Hood + +- Add tests for --empty flag ([#667](https://github.com/dbt-labs/dbt-redshift/issues/667)) +- Update base adapter references as part of decoupling migration ([#698](https://github.com/dbt-labs/dbt-redshift/issues/698)) + +### Dependencies + +- Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) +- Update ddtrace requirement from ~=1.19 to ~=1.20 ([#622](https://github.com/dbt-labs/dbt-redshift/pull/622)) +- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#627](https://github.com/dbt-labs/dbt-redshift/pull/627)) +- Bump mypy from 1.5.1 to 1.6.0 ([#629](https://github.com/dbt-labs/dbt-redshift/pull/629)) +- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#634](https://github.com/dbt-labs/dbt-redshift/pull/634)) +- Update black requirement from ~=23.9 to ~=23.10 ([#636](https://github.com/dbt-labs/dbt-redshift/pull/636)) +- Bump mypy from 1.6.0 to 1.6.1 ([#648](https://github.com/dbt-labs/dbt-redshift/pull/648)) +- Update ddtrace requirement from ~=1.20 to ~=2.1 ([#651](https://github.com/dbt-labs/dbt-redshift/pull/651)) +- Update black requirement from ~=23.10 to ~=23.11 ([#660](https://github.com/dbt-labs/dbt-redshift/pull/660)) +- Bump mypy from 1.6.1 to 1.7.0 ([#662](https://github.com/dbt-labs/dbt-redshift/pull/662)) +- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#664](https://github.com/dbt-labs/dbt-redshift/pull/664)) +- Update ddtrace requirement from ~=2.1 to ~=2.2 ([#665](https://github.com/dbt-labs/dbt-redshift/pull/665)) +- Update ddtrace requirement from ~=2.2 to ~=2.3 ([#669](https://github.com/dbt-labs/dbt-redshift/pull/669)) +- Update wheel requirement from ~=0.41 to ~=0.42 ([#670](https://github.com/dbt-labs/dbt-redshift/pull/670)) +- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#672](https://github.com/dbt-labs/dbt-redshift/pull/672)) +- Remove direct boto3 dependency ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) +- Bump mypy from 1.7.0 to 1.7.1 ([#676](https://github.com/dbt-labs/dbt-redshift/pull/676)) +- Update freezegun requirement from ~=1.2 to ~=1.3 ([#681](https://github.com/dbt-labs/dbt-redshift/pull/681)) +- Update black requirement from ~=23.11 to ~=23.12 ([#688](https://github.com/dbt-labs/dbt-redshift/pull/688)) +- upgrade redshift driver to 2.0.918 ([#700](https://github.com/dbt-labs/dbt-redshift/pull/700)) +- Migrate to dbt-common and dbt-adapters ([#706](https://github.com/dbt-labs/dbt-redshift/pull/706)) + +### Contributors +- [@hexDoor](https://github.com/hexDoor) ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) +- [@reptillicus](https://github.com/reptillicus) ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) +- [@soksamnanglim](https://github.com/soksamnanglim) ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index f15b401d1..6496f3e22 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0a1" +version = "1.8.0b1" From 8077150102d0c09c325766743231d01d8c330ee6 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 1 Mar 2024 21:42:13 -0500 Subject: [PATCH 07/61] Add explicit dependency on sqlparse that used to inherit from core (#728) * add explicit dependency on sqlparse that used to inherit from core * update build verification command * remove core installation check since it is no longer installed --- .github/workflows/main.yml | 5 ++--- .github/workflows/release.yml | 4 ++-- setup.py | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a76df7e9e..c361cb987 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,7 +59,6 @@ jobs: python -m pip --version pre-commit --version mypy --version - dbt --version - name: pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -204,7 +203,7 @@ jobs: - name: Check wheel distributions run: | - dbt --version + python -c "import dbt.adapters.redshift" - name: Install source distributions run: | @@ -212,4 +211,4 @@ jobs: - name: Check source distributions run: | - dbt --version + python -c "import dbt.adapters.redshift" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 88942e251..f68eb3320 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ on: package_test_command: description: "Package test command" type: string - default: "dbt --version" + default: "python -c \"import dbt.adapters.redshift\"" required: true test_run: description: "Test run (Publish release as draft)" @@ -92,7 +92,7 @@ on: package_test_command: description: "Package test command" type: string - default: "dbt --version" + default: "python -c \"import dbt.adapters.redshift\"" required: true test_run: description: "Test run (Publish release as draft)" diff --git a/setup.py b/setup.py index a1b63306b..6cfd2823e 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ def _plugin_version() -> str: # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<=2.0.918, >=2.0.913, !=2.0.914", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "sqlparse>=0.2.3,<0.5", "agate", ], zip_safe=False, From 1e217f8354e4b8af34d135c33bb5f350abec6824 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:32:23 -0700 Subject: [PATCH 08/61] Add Redshift internal build process workflow (#732) * ho boy * Add internal release flow. * Minor fixes for testing. * Add core team bucket name. * Tweak setup steps * Pair down job * Fix booleans * Tweak caps * in repo solutioning * Add runs on * Add the python commands in step scope * Put jobs together * Refactor to use reusable workflow. * What's the syntax error. * Adding rename code. * Get rid of the internal build stamp for postgres grab * Update versioning metadata rules. * Correct regex * Add sha * bump to 1.8.3 * Revert version bump used in testing. * Remove bad comments * Add changelog * Lint * code review comments --------- Co-authored-by: Mila Page --- .../unreleased/Features-20240318-030018.yaml | 6 +++ .github/workflows/release-internal.yml | 39 +++++++++++++++++++ setup.py | 13 ++++++- 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20240318-030018.yaml create mode 100644 .github/workflows/release-internal.yml diff --git a/.changes/unreleased/Features-20240318-030018.yaml b/.changes/unreleased/Features-20240318-030018.yaml new file mode 100644 index 000000000..badab8088 --- /dev/null +++ b/.changes/unreleased/Features-20240318-030018.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add new workflow for internal patch releases +time: 2024-03-18T03:00:18.845618-07:00 +custom: + Author: versusfacit + Issue: "38" diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml new file mode 100644 index 000000000..e9063a53f --- /dev/null +++ b/.github/workflows/release-internal.yml @@ -0,0 +1,39 @@ +name: Release internal patch + +on: + workflow_dispatch: + inputs: + version_number: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "The sha to use (leave empty to use latest on main)" + type: string + required: false + package_test_command: + description: "Package test command" + type: string + default: "python -c \"import dbt.adapters.redshift\"" + required: true + +defaults: + run: + shell: "bash" + +env: + PYTHON_TARGET_VERSION: 3.11 + +jobs: + invoke-reusable-workflow: + name: "Build and Release Internally" + + uses: "VersusFacit/dbt-release/.github/workflows/internal-archive-release.yml@main" + + with: + version_number: "${{ inputs.version_number }}" + package_test_command: "${{ inputs.package_test_command }}" + dbms_name: "redshift" + sha: "${{ inputs.sha }}" + + secrets: "inherit" diff --git a/setup.py b/setup.py index 6cfd2823e..beffa3348 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import re import sys if sys.version_info < (3, 8): @@ -36,6 +37,16 @@ def _plugin_version() -> str: return attributes["version"] +def _plugin_version_trim() -> str: + """ + Pull the package version from the main package version file + """ + attributes = {} + exec(VERSION.read_text(), attributes) + pattern = r"\+build\d+$" + return re.sub(pattern, "", attributes["version"]) + + setup( name="dbt-redshift", version=_plugin_version(), @@ -50,7 +61,7 @@ def _plugin_version() -> str: install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version()}", + f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<=2.0.918, >=2.0.913, !=2.0.914", From 13ada0a84b466e53f76c8a1bd1633cde7f5c72a0 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:33:35 -0500 Subject: [PATCH 09/61] Test ddtrace dependency range (#733) * set bounds for ddtrace due to tests failing * hard pin ddtrace, create changelog --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240319-101832.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240319-101832.yaml diff --git a/.changes/unreleased/Dependencies-20240319-101832.yaml b/.changes/unreleased/Dependencies-20240319-101832.yaml new file mode 100644 index 000000000..7b5f0eb3a --- /dev/null +++ b/.changes/unreleased/Dependencies-20240319-101832.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: hard pin ddtrace +time: 2024-03-19T10:18:32.426032-05:00 +custom: + Author: McKnight-42 + PR: "733" diff --git a/dev-requirements.txt b/dev-requirements.txt index 6d29276cf..079a56062 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,7 +10,7 @@ git+https://github.com/dbt-labs/dbt-postgres.git black~=23.12 bumpversion~=0.6.0 click~=8.1 -ddtrace~=2.3 +ddtrace==2.3.0 flake8~=6.1 flaky~=3.7 freezegun~=1.3 From f92ec4889a9faaaeb74a4b14da50c87ca65a0e89 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:38:29 -0400 Subject: [PATCH 10/61] [Bug] Python driver 2.0.918 causes SSL error. Need to revert back to 2.0.917 (#731) * cap redshift-connector at 2.0.917 to avoid ssl issue in 2.0.918 * changie --- .changes/unreleased/Fixes-20240317-113447.yaml | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20240317-113447.yaml diff --git a/.changes/unreleased/Fixes-20240317-113447.yaml b/.changes/unreleased/Fixes-20240317-113447.yaml new file mode 100644 index 000000000..4338fc95b --- /dev/null +++ b/.changes/unreleased/Fixes-20240317-113447.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 +time: 2024-03-17T11:34:47.873169-04:00 +custom: + Author: mikealfare + Issue: "730" diff --git a/setup.py b/setup.py index beffa3348..234e7c97d 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def _plugin_version_trim() -> str: f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<=2.0.918, >=2.0.913, !=2.0.914", + "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core "sqlparse>=0.2.3,<0.5", "agate", From c99c73bd0386e2641c825b8e36c5bc0229cd3258 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:54:54 -0700 Subject: [PATCH 11/61] Fix the renamed relations code (#723) * Add test * Add FrozenSet to imports * removed extraneous semicolons * update alter table to alter view for rename --------- Co-authored-by: Mila Page Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Mike Alfare --- .../Under the Hood-20240227-002713.yaml | 6 +++++ dbt/adapters/redshift/relation.py | 26 +++++++++++-------- .../relations/materialized_view/create.sql | 2 +- .../redshift/macros/relations/view/rename.sql | 2 +- .../macros/relations/view/replace.sql | 2 +- tests/unit/test_renamed_relations.py | 17 ++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240227-002713.yaml create mode 100644 tests/unit/test_renamed_relations.py diff --git a/.changes/unreleased/Under the Hood-20240227-002713.yaml b/.changes/unreleased/Under the Hood-20240227-002713.yaml new file mode 100644 index 000000000..48b8de8e4 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240227-002713.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add unit test for transaction semantics. +time: 2024-02-27T00:27:13.299107-08:00 +custom: + Author: versusfacit + Issue: "722" diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index 7255288e2..e120a2fac 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional +from typing import FrozenSet, Optional from dbt.adapters.base.relation import BaseRelation from dbt.adapters.relation_configs import ( @@ -30,16 +30,20 @@ class RedshiftRelation(BaseRelation): relation_configs = { RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, } - renameable_relations = frozenset( - { - RelationType.View, - RelationType.Table, - } + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) ) - replaceable_relations = frozenset( - { - RelationType.View, - } + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + } + ) ) def __post_init__(self): diff --git a/dbt/include/redshift/macros/relations/materialized_view/create.sql b/dbt/include/redshift/macros/relations/materialized_view/create.sql index 06fe2b6b5..1b81992e4 100644 --- a/dbt/include/redshift/macros/relations/materialized_view/create.sql +++ b/dbt/include/redshift/macros/relations/materialized_view/create.sql @@ -10,6 +10,6 @@ auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} as ( {{ materialized_view.query }} - ); + ) {% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/rename.sql b/dbt/include/redshift/macros/relations/view/rename.sql index 0c6cdcdfa..a96b04451 100644 --- a/dbt/include/redshift/macros/relations/view/rename.sql +++ b/dbt/include/redshift/macros/relations/view/rename.sql @@ -1,3 +1,3 @@ {% macro redshift__get_rename_view_sql(relation, new_name) %} - alter view {{ relation }} rename to {{ new_name }} + alter table {{ relation }} rename to {{ new_name }} {% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/replace.sql b/dbt/include/redshift/macros/relations/view/replace.sql index 25a9d8b38..7ae89ab45 100644 --- a/dbt/include/redshift/macros/relations/view/replace.sql +++ b/dbt/include/redshift/macros/relations/view/replace.sql @@ -13,6 +13,6 @@ {{ get_assert_columns_equivalent(sql) }} {%- endif %} as ( {{ sql }} - ) {{ bind_qualifier }}; + ) {{ bind_qualifier }} {%- endmacro %} diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py new file mode 100644 index 000000000..4817ab100 --- /dev/null +++ b/tests/unit/test_renamed_relations.py @@ -0,0 +1,17 @@ +from dbt.adapters.redshift.relation import RedshiftRelation +from dbt.adapters.contracts.relation import RelationType + + +def test_renameable_relation(): + relation = RedshiftRelation.create( + database="my_db", + schema="my_schema", + identifier="my_table", + type=RelationType.Table, + ) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + } + ) From be1a252cd51cf4a314999935734ef6ad9c9c87b5 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:53:21 -0700 Subject: [PATCH 12/61] Finish internal build workflow (#735) * Finish internal build. * Add comments. * Add different branch reference for testing with that unmerged workflow. * Debug setup * Revert setup.py * Adjust comments * Tweaks to input params. * Change python version to match Cloud. * Finalize workflow. * Change branch reference of workflow to main. --------- Co-authored-by: Mila Page --- .github/workflows/release-internal.yml | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index e9063a53f..0ce0f9adf 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -1,4 +1,16 @@ -name: Release internal patch +# What? +# +# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. +# +# How? +# +# After checking out and testing the provided ref, the image is built and uploaded. +# +# When? +# +# Manual trigger. + +name: "Release internal patch" on: workflow_dispatch: @@ -7,10 +19,11 @@ on: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - sha: - description: "The sha to use (leave empty to use latest on main)" + ref: + description: "The ref (sha or branch name) to use" type: string - required: false + default: "main" + required: true package_test_command: description: "Package test command" type: string @@ -21,19 +34,16 @@ defaults: run: shell: "bash" -env: - PYTHON_TARGET_VERSION: 3.11 - jobs: invoke-reusable-workflow: name: "Build and Release Internally" - uses: "VersusFacit/dbt-release/.github/workflows/internal-archive-release.yml@main" + uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" with: version_number: "${{ inputs.version_number }}" package_test_command: "${{ inputs.package_test_command }}" dbms_name: "redshift" - sha: "${{ inputs.sha }}" + ref: "${{ inputs.ref }}" secrets: "inherit" From 3483ad1a56f2680b18df00eeb470e66b85d6677a Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:43:51 -0400 Subject: [PATCH 13/61] Pin `black>=24.3` (#743) --- .changes/unreleased/Security-20240327-191927.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Security-20240327-191927.yaml diff --git a/.changes/unreleased/Security-20240327-191927.yaml b/.changes/unreleased/Security-20240327-191927.yaml new file mode 100644 index 000000000..763273c79 --- /dev/null +++ b/.changes/unreleased/Security-20240327-191927.yaml @@ -0,0 +1,6 @@ +kind: Security +body: Pin `black>=24.3` in `dev-requirements.txt` +time: 2024-03-27T19:19:27.243854-04:00 +custom: + Author: mikealfare + PR: "743" diff --git a/dev-requirements.txt b/dev-requirements.txt index 079a56062..85edead99 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,7 +7,7 @@ git+https://github.com/dbt-labs/dbt-postgres.git # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=23.12 +black>=24.3 bumpversion~=0.6.0 click~=8.1 ddtrace==2.3.0 From 3e9e9486c76de187e844fd674ffcd761e95e7d72 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:00:28 -0400 Subject: [PATCH 14/61] Add `dbt-core~=1.8.0a1` as convenience dep (#756) * add `dbt-core~=1.8.0a1` as convenience dep * skip inapplicable version comparison test --- .changes/unreleased/Dependencies-20240403-134607.yaml | 6 ++++++ setup.py | 2 ++ tests/functional/adapter/test_query_comment.py | 8 +++++++- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240403-134607.yaml diff --git a/.changes/unreleased/Dependencies-20240403-134607.yaml b/.changes/unreleased/Dependencies-20240403-134607.yaml new file mode 100644 index 000000000..ccc80c5e9 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240403-134607.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Add `dbt-core` as a dependency to preserve backwards compatibility for installation +time: 2024-04-03T13:46:07.335865-04:00 +custom: + Author: mikealfare + PR: "756" diff --git a/setup.py b/setup.py index 234e7c97d..4673657b8 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ def _plugin_version_trim() -> str: # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0a1", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core "sqlparse>=0.2.3,<0.5", "agate", diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py index db6a440d7..75c87ee38 100644 --- a/tests/functional/adapter/test_query_comment.py +++ b/tests/functional/adapter/test_query_comment.py @@ -6,6 +6,7 @@ BaseNullQueryComments, BaseEmptyQueryComments, ) +import pytest class TestQueryCommentsRedshift(BaseQueryComments): @@ -17,7 +18,12 @@ class TestMacroQueryCommentsRedshift(BaseMacroQueryComments): class TestMacroArgsQueryCommentsRedshift(BaseMacroArgsQueryComments): - pass + @pytest.mark.skip( + "This test is incorrectly comparing the version of `dbt-core`" + "to the version of `dbt-postgres`, which is not always the same." + ) + def test_matches_comment(self, project, get_package_version): + pass class TestMacroInvalidQueryCommentsRedshift(BaseMacroInvalidQueryComments): From afc79ad2c081938924e65b5da26619ff51edc655 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 3 Apr 2024 21:04:10 +0000 Subject: [PATCH 15/61] Bumping version to 1.8.0b2 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.0-b2.md | 22 ++++++++++++++++ .../Dependencies-20240319-101832.yaml | 0 .../Dependencies-20240403-134607.yaml | 0 .../Features-20240318-030018.yaml | 0 .../Fixes-20240317-113447.yaml | 0 .../Security-20240327-191927.yaml | 0 .../Under the Hood-20240227-002713.yaml | 0 CHANGELOG.md | 26 ++++++++++++++++++- dbt/adapters/redshift/__version__.py | 2 +- 10 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 .changes/1.8.0-b2.md rename .changes/{unreleased => 1.8.0}/Dependencies-20240319-101832.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240403-134607.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240318-030018.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240317-113447.yaml (100%) rename .changes/{unreleased => 1.8.0}/Security-20240327-191927.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240227-002713.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 54264b726..7cccf8ede 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0b1 +current_version = 1.8.0b2 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b2.md b/.changes/1.8.0-b2.md new file mode 100644 index 000000000..47996a7ce --- /dev/null +++ b/.changes/1.8.0-b2.md @@ -0,0 +1,22 @@ +## dbt-redshift 1.8.0-b2 - April 03, 2024 + +### Features + +- Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-redshift/issues/38)) + +### Fixes + +- Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 ([#730](https://github.com/dbt-labs/dbt-redshift/issues/730)) + +### Under the Hood + +- Add unit test for transaction semantics. ([#722](https://github.com/dbt-labs/dbt-redshift/issues/722)) + +### Dependencies + +- hard pin ddtrace ([#733](https://github.com/dbt-labs/dbt-redshift/pull/733)) +- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#756](https://github.com/dbt-labs/dbt-redshift/pull/756)) + +### Security + +- Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) diff --git a/.changes/unreleased/Dependencies-20240319-101832.yaml b/.changes/1.8.0/Dependencies-20240319-101832.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240319-101832.yaml rename to .changes/1.8.0/Dependencies-20240319-101832.yaml diff --git a/.changes/unreleased/Dependencies-20240403-134607.yaml b/.changes/1.8.0/Dependencies-20240403-134607.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240403-134607.yaml rename to .changes/1.8.0/Dependencies-20240403-134607.yaml diff --git a/.changes/unreleased/Features-20240318-030018.yaml b/.changes/1.8.0/Features-20240318-030018.yaml similarity index 100% rename from .changes/unreleased/Features-20240318-030018.yaml rename to .changes/1.8.0/Features-20240318-030018.yaml diff --git a/.changes/unreleased/Fixes-20240317-113447.yaml b/.changes/1.8.0/Fixes-20240317-113447.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240317-113447.yaml rename to .changes/1.8.0/Fixes-20240317-113447.yaml diff --git a/.changes/unreleased/Security-20240327-191927.yaml b/.changes/1.8.0/Security-20240327-191927.yaml similarity index 100% rename from .changes/unreleased/Security-20240327-191927.yaml rename to .changes/1.8.0/Security-20240327-191927.yaml diff --git a/.changes/unreleased/Under the Hood-20240227-002713.yaml b/.changes/1.8.0/Under the Hood-20240227-002713.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240227-002713.yaml rename to .changes/1.8.0/Under the Hood-20240227-002713.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aebed2a5..200815006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.8.0-b2 - April 03, 2024 + +### Features + +- Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-redshift/issues/38)) + +### Fixes + +- Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 ([#730](https://github.com/dbt-labs/dbt-redshift/issues/730)) + +### Under the Hood + +- Add unit test for transaction semantics. ([#722](https://github.com/dbt-labs/dbt-redshift/issues/722)) + +### Dependencies + +- hard pin ddtrace ([#733](https://github.com/dbt-labs/dbt-redshift/pull/733)) +- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#756](https://github.com/dbt-labs/dbt-redshift/pull/756)) + +### Security + +- Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) + + + ## dbt-redshift 1.8.0-b1 - March 01, 2024 ### Features @@ -55,7 +80,6 @@ - [@reptillicus](https://github.com/reptillicus) ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) - [@soksamnanglim](https://github.com/soksamnanglim) ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 6496f3e22..7d16c28f0 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0b1" +version = "1.8.0b2" From deb92e5cf0082dfcdd8238683e842329f70f9096 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:53:45 -0400 Subject: [PATCH 16/61] Update dependabot config to cover GHA (#759) * Update dependabot config to cover GHA --- .changes/unreleased/Under the Hood-20240410-182912.yaml | 6 ++++++ .github/dependabot.yml | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20240410-182912.yaml diff --git a/.changes/unreleased/Under the Hood-20240410-182912.yaml b/.changes/unreleased/Under the Hood-20240410-182912.yaml new file mode 100644 index 000000000..e6826c4d5 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240410-182912.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update dependabot config to cover GHA +time: 2024-04-10T18:29:12.026746-04:00 +custom: + Author: mikealfare + Issue: "759" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f34492..4673f47cf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,12 @@ version: 2 updates: - # python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From fdad756e99b8ba9b514bad0a0d52aa37e4b54847 Mon Sep 17 00:00:00 2001 From: Holly Evans <39742776+holly-evans@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:06:13 -0500 Subject: [PATCH 17/61] Allow dbt to cancel connections (#718) * Save backend_pid on initiation * Adjust OnConfigurationChangeOption import * Use MagicMock to accomodate __getitem__ call on cursor results * Test backend_pid usage * Store backend_pid on connection directly --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Teresa Martyny <135771777+martynydbt@users.noreply.github.com> --- .../unreleased/Fixes-20240326-123703.yaml | 6 ++ dbt/adapters/redshift/connections.py | 29 +++---- tests/unit/test_redshift_adapter.py | 76 ++++++++++++++----- 3 files changed, 79 insertions(+), 32 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240326-123703.yaml diff --git a/.changes/unreleased/Fixes-20240326-123703.yaml b/.changes/unreleased/Fixes-20240326-123703.yaml new file mode 100644 index 000000000..5d9bee694 --- /dev/null +++ b/.changes/unreleased/Fixes-20240326-123703.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: dbt can cancel open queries upon interrupt +time: 2024-03-26T12:37:03.17481-05:00 +custom: + Author: holly-evans + Issue: "705" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index b0fc0825d..cc58c02a6 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -233,27 +233,26 @@ def connect(): class RedshiftConnectionManager(SQLConnectionManager): TYPE = "redshift" - def _get_backend_pid(self): - sql = "select pg_backend_pid()" - _, cursor = self.add_query(sql) - - res = cursor.fetchone() - return res[0] - def cancel(self, connection: Connection): + pid = connection.backend_pid # type: ignore + sql = f"select pg_terminate_backend({pid})" + logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") + logger.debug(sql) + try: - pid = self._get_backend_pid() + self.add_query(sql) except redshift_connector.InterfaceError as e: if "is closed" in str(e): logger.debug(f"Connection {connection.name} was already closed") return raise - sql = f"select pg_terminate_backend({pid})" - cursor = connection.handle.cursor() - logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") - logger.debug(sql) - cursor.execute(sql) + @classmethod + def _get_backend_pid(cls, connection): + with connection.handle.cursor() as c: + sql = "select pg_backend_pid()" + res = c.execute(sql).fetchone() + return res[0] @classmethod def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse: @@ -325,7 +324,7 @@ def exponential_backoff(attempt: int): redshift_connector.DataError, ] - return cls.retry_connection( + open_connection = cls.retry_connection( connection, connect=connect_method_factory.get_connect_method(), logger=logger, @@ -333,6 +332,8 @@ def exponential_backoff(attempt: int): retry_timeout=exponential_backoff, retryable_exceptions=retryable_exceptions, ) + open_connection.backend_pid = cls._get_backend_pid(open_connection) # type: ignore + return open_connection def execute( self, diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index 671e47032..0bd5f8e99 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -4,7 +4,7 @@ from unittest import mock from dbt_common.exceptions import DbtRuntimeError -from unittest.mock import Mock, call +from unittest.mock import MagicMock, call import agate import dbt @@ -67,7 +67,7 @@ def adapter(self): inject_adapter(self._adapter, RedshiftPlugin) return self._adapter - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_implicit_database_conn(self): connection = self.adapter.acquire_connection("dummy") connection.handle @@ -84,7 +84,7 @@ def test_implicit_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region_with_database_conn(self): self.config.method = "database" @@ -103,7 +103,7 @@ def test_explicit_region_with_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_conn_without_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -129,7 +129,7 @@ def test_explicit_iam_conn_without_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_conn_timeout_30(self): self.config.credentials = self.config.credentials.replace(connect_timeout=30) connection = self.adapter.acquire_connection("dummy") @@ -147,7 +147,7 @@ def test_conn_timeout_30(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_conn_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -175,7 +175,7 @@ def test_explicit_iam_conn_with_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_serverless_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -201,7 +201,7 @@ def test_explicit_iam_serverless_with_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region(self): # Successful test self.config.credentials = self.config.credentials.replace( @@ -229,7 +229,7 @@ def test_explicit_region(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region_failure(self): # Failure test with no region self.config.credentials = self.config.credentials.replace( @@ -258,7 +258,7 @@ def test_explicit_region_failure(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_invalid_region(self): # Invalid region test self.config.credentials = self.config.credentials.replace( @@ -287,7 +287,7 @@ def test_explicit_invalid_region(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_disable(self): self.config.credentials.sslmode = "disable" connection = self.adapter.acquire_connection("dummy") @@ -306,7 +306,7 @@ def test_sslmode_disable(self): sslmode=None, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_allow(self): self.config.credentials.sslmode = "allow" connection = self.adapter.acquire_connection("dummy") @@ -325,7 +325,7 @@ def test_sslmode_allow(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_verify_full(self): self.config.credentials.sslmode = "verify-full" connection = self.adapter.acquire_connection("dummy") @@ -344,7 +344,7 @@ def test_sslmode_verify_full(self): sslmode="verify-full", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_verify_ca(self): self.config.credentials.sslmode = "verify-ca" connection = self.adapter.acquire_connection("dummy") @@ -363,7 +363,7 @@ def test_sslmode_verify_ca(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_prefer(self): self.config.credentials.sslmode = "prefer" connection = self.adapter.acquire_connection("dummy") @@ -382,7 +382,7 @@ def test_sslmode_prefer(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_serverless_iam_failure(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -447,6 +447,25 @@ def test_invalid_iam_no_cluster_id(self): self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_has_backend_pid(self): + backend_pid = 42 + + cursor = mock.MagicMock() + execute = cursor().__enter__().execute + execute().fetchone.return_value = (backend_pid,) + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + assert connection.backend_pid == backend_pid + + execute.assert_has_calls( + [ + call("select pg_backend_pid()"), + ] + ) + def test_cancel_open_connections_empty(self): self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) @@ -475,11 +494,32 @@ def test_cancel_open_connections_single(self): self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) add_query.assert_has_calls( [ - call("select pg_backend_pid()"), + call(f"select pg_terminate_backend({model.backend_pid})"), ] ) - master.handle.get_backend_pid.assert_not_called() + master.handle.backend_pid.assert_not_called() + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_backend_pid_used_in_pg_terminate_backend(self): + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + backend_pid = 42 + query_result = (backend_pid,) + + cursor = mock.MagicMock() + cursor().__enter__().execute().fetchone.return_value = query_result + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + + self.adapter.connections.cancel(connection) + + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({backend_pid})"), + ] + ) def test_dbname_verification_is_case_insensitive(self): # Override adapter settings from setUp() From 663b8edfb1d1f36e5014e9a2e9455dc771bdb722 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:55:21 -0500 Subject: [PATCH 18/61] update sqlparse to be in line with dbt-core (#768) * update sqlparse requirement to be in parity with dbt-core * add changelog * allow verify tests to pass (will need to patch) * pin core, update changelong, add ref to core security issue * revert core pin as not in pypi * update changelong to match core, reapply core new bound * pushing bound back up to be in line of what new core will be after b3 release --- .changes/unreleased/Security-20240416-195919.yaml | 6 ++++++ setup.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Security-20240416-195919.yaml diff --git a/.changes/unreleased/Security-20240416-195919.yaml b/.changes/unreleased/Security-20240416-195919.yaml new file mode 100644 index 000000000..af8fb6f1d --- /dev/null +++ b/.changes/unreleased/Security-20240416-195919.yaml @@ -0,0 +1,6 @@ +kind: Security +body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core +time: 2024-04-16T19:59:19.233806-05:00 +custom: + Author: McKnight-42 + PR: "768" diff --git a/setup.py b/setup.py index 4673657b8..dbb3913b9 100644 --- a/setup.py +++ b/setup.py @@ -66,9 +66,9 @@ def _plugin_version_trim() -> str: # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0a1", + "dbt-core>=1.8.0b3", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.2.3,<0.5", + "sqlparse>=0.5.0,<0.6.0", "agate", ], zip_safe=False, From 81d7a19b4faa1ca70beb36a05a8cb619aeb1a133 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 18 Apr 2024 19:54:44 +0000 Subject: [PATCH 19/61] Bumping version to 1.8.0b3 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.0-b3.md | 16 +++++++++++++++ .../Fixes-20240326-123703.yaml | 0 .../Security-20240416-195919.yaml | 0 .../Under the Hood-20240410-182912.yaml | 0 CHANGELOG.md | 20 +++++++++++++++++-- dbt/adapters/redshift/__version__.py | 2 +- 7 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 .changes/1.8.0-b3.md rename .changes/{unreleased => 1.8.0}/Fixes-20240326-123703.yaml (100%) rename .changes/{unreleased => 1.8.0}/Security-20240416-195919.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240410-182912.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7cccf8ede..52cc0acca 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0b2 +current_version = 1.8.0b3 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b3.md b/.changes/1.8.0-b3.md new file mode 100644 index 000000000..5f9d828c3 --- /dev/null +++ b/.changes/1.8.0-b3.md @@ -0,0 +1,16 @@ +## dbt-redshift 1.8.0-b3 - April 18, 2024 + +### Fixes + +- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + +### Under the Hood + +- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) + +### Contributors +- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) diff --git a/.changes/unreleased/Fixes-20240326-123703.yaml b/.changes/1.8.0/Fixes-20240326-123703.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240326-123703.yaml rename to .changes/1.8.0/Fixes-20240326-123703.yaml diff --git a/.changes/unreleased/Security-20240416-195919.yaml b/.changes/1.8.0/Security-20240416-195919.yaml similarity index 100% rename from .changes/unreleased/Security-20240416-195919.yaml rename to .changes/1.8.0/Security-20240416-195919.yaml diff --git a/.changes/unreleased/Under the Hood-20240410-182912.yaml b/.changes/1.8.0/Under the Hood-20240410-182912.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240410-182912.yaml rename to .changes/1.8.0/Under the Hood-20240410-182912.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 200815006..f45988f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.8.0-b3 - April 18, 2024 + +### Fixes + +- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + +### Under the Hood + +- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) + +### Contributors +- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + + ## dbt-redshift 1.8.0-b2 - April 03, 2024 ### Features @@ -28,8 +46,6 @@ - Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) - - ## dbt-redshift 1.8.0-b1 - March 01, 2024 ### Features diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 7d16c28f0..b0f82cbca 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0b2" +version = "1.8.0b3" From b8d330bf4b8c9a2d4e630c62ba8ddbd02a819578 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 18 Apr 2024 17:10:20 -0400 Subject: [PATCH 20/61] TableLastModifiedMetadataBatch capability (#744) * TableLastModifiedMetadataBatch capability * pin ddtrace * open connection before calculate_freshness_from_metadata_batch * remove calculate_freshness_from_metadata_batch override * changelog entry * TestGetLastRelationModifiedBatch * restore dev-requirements.txt --- .../unreleased/Features-20240404-171441.yaml | 6 + dbt/adapters/redshift/impl.py | 1 + .../test_get_relation_last_modified.py | 128 ++++++++++++++++-- 3 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 .changes/unreleased/Features-20240404-171441.yaml diff --git a/.changes/unreleased/Features-20240404-171441.yaml b/.changes/unreleased/Features-20240404-171441.yaml new file mode 100644 index 000000000..f1ac623c0 --- /dev/null +++ b/.changes/unreleased/Features-20240404-171441.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support TableLastModifiedMetadataBatch capability +time: 2024-04-04T17:14:41.313087-07:00 +custom: + Author: michelleark + Issue: "755" diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index a77601895..18faee48c 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -58,6 +58,7 @@ class RedshiftAdapter(SQLAdapter): { Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), } ) diff --git a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py index 6a77d22ae..c31e9ac61 100644 --- a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py +++ b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py @@ -1,12 +1,24 @@ import os +import pytest +from unittest import mock +from dbt.adapters.redshift.impl import RedshiftAdapter +from dbt.adapters.capability import Capability, CapabilityDict +from dbt.cli.main import dbtRunner from dbt.tests.util import run_dbt -import pytest from tests.functional.adapter.sources_freshness_tests import files -class TestGetLastRelationModified: +class SetupGetLastRelationModified: + @pytest.fixture(scope="class", autouse=True) + def set_env_vars(self, project): + os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema + yield + del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + + +class TestGetLastRelationModified(SetupGetLastRelationModified): @pytest.fixture(scope="class") def seeds(self): return { @@ -18,14 +30,6 @@ def seeds(self): def models(self): return {"schema.yml": files.SCHEMA_YML} - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - # we need the schema name for the sources section - os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema - run_dbt(["seed"]) - yield - del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] - @pytest.mark.parametrize( "source,status,expect_pass", [ @@ -34,9 +38,113 @@ def setup(self, project): ], ) def test_get_last_relation_modified(self, project, source, status, expect_pass): + run_dbt(["seed"]) + results = run_dbt( ["source", "freshness", "--select", f"source:{source}"], expect_pass=expect_pass ) assert len(results) == 1 result = results[0] assert result.status == status + + +freshness_metadata_schema_batch_yml = """ +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_table + - name: test_table2 + - name: test_table_with_loaded_at_field + loaded_at_field: my_loaded_at_field +""" + + +class TestGetLastRelationModifiedBatch(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def custom_schema(self, project, set_env_vars): + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + ) + project.adapter.drop_schema(relation) + project.adapter.create_schema(relation) + + yield relation.schema + + with project.adapter.connection_named("__test"): + project.adapter.drop_schema(relation) + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": freshness_metadata_schema_batch_yml} + + def get_freshness_result_for_table(self, table_name, results): + for result in results: + if result.node.name == table_name: + return result + return None + + def test_get_last_relation_modified_batch(self, project, custom_schema): + project.run_sql( + f"create table {custom_schema}.test_table as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table2 as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table_with_loaded_at_field as (select 1 as id, timestamp '2009-09-15 10:59:43' as my_loaded_at_field);" + ) + + runner = dbtRunner() + freshness_results_batch = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results_batch) == 3 + test_table_batch_result = self.get_freshness_result_for_table( + "test_table", freshness_results_batch + ) + test_table2_batch_result = self.get_freshness_result_for_table( + "test_table2", freshness_results_batch + ) + test_table_with_loaded_at_field_batch_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results_batch + ) + + # Remove TableLastModifiedMetadataBatch and run freshness on same input without batch strategy + capabilities_no_batch = CapabilityDict( + { + capability: support + for capability, support in RedshiftAdapter.capabilities().items() + if capability != Capability.TableLastModifiedMetadataBatch + } + ) + with mock.patch.object( + RedshiftAdapter, "capabilities", return_value=capabilities_no_batch + ): + freshness_results = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results) == 3 + test_table_result = self.get_freshness_result_for_table("test_table", freshness_results) + test_table2_result = self.get_freshness_result_for_table("test_table2", freshness_results) + test_table_with_loaded_at_field_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results + ) + + # assert results between batch vs non-batch freshness strategy are equivalent + assert test_table_result.status == test_table_batch_result.status + assert test_table_result.max_loaded_at == test_table_batch_result.max_loaded_at + + assert test_table2_result.status == test_table2_batch_result.status + assert test_table2_result.max_loaded_at == test_table2_batch_result.max_loaded_at + + assert ( + test_table_with_loaded_at_field_batch_result.status + == test_table_with_loaded_at_field_result.status + ) + assert ( + test_table_with_loaded_at_field_batch_result.max_loaded_at + == test_table_with_loaded_at_field_result.max_loaded_at + ) From 71acee99796f85f63ec99889d773fc8adc39b669 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:46:38 -0400 Subject: [PATCH 21/61] add pre-commit check for dbt-core (#775) --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d80b955c..b89d0df27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,10 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-case-conflict +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check - repo: https://github.com/psf/black rev: 23.1.0 hooks: From eb0c74b8334dabda468caa9c280cc8d3f4cc7515 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:14:52 -0400 Subject: [PATCH 22/61] Move to `pre-commit` only (#777) * moving linting config into pre-commit-config * remove explicit call to mypy * run linting * fix enum reference --- .flake8 | 16 --- .github/workflows/main.yml | 1 - .pre-commit-config.yaml | 114 ++++++++---------- dbt/adapters/redshift/connections.py | 12 +- .../redshift/relation_configs/dist.py | 4 +- .../relation_configs/materialized_view.py | 2 +- .../redshift/relation_configs/sort.py | 4 +- dev-requirements.txt | 24 ++-- mypy.ini | 2 - .../test_materialized_view.py | 12 +- tests/unit/utils.py | 1 + 11 files changed, 78 insertions(+), 114 deletions(-) delete mode 100644 .flake8 delete mode 100644 mypy.ini diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b08ffcd53..000000000 --- a/.flake8 +++ /dev/null @@ -1,16 +0,0 @@ -[flake8] -select = - E - W - F -ignore = - # makes Flake8 work like black - W503, - W504, - # makes Flake8 work like black - E203, - E741, - E501, -exclude = test -per-file-ignores = - */__init__.py: F401 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c361cb987..389f55a88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,6 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip --version pre-commit --version - mypy --version - name: pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b89d0df27..ae249943d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,67 +1,55 @@ -# For more on configuring pre-commit hooks (see https://pre-commit.com/) - -# Force all unspecified python hooks to run python 3.8 default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: https://github.com/dbt-labs/pre-commit-hooks - rev: v0.1.0a1 - hooks: - - id: dbt-core-in-adapters-check -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - id: black - alias: black-check - stages: [manual] - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - "--check" - - "--diff" -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - id: flake8 - alias: flake8-check - stages: [manual] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 - hooks: - - id: mypy - # N.B.: Mypy is... a bit fragile. - # - # By using `language: system` we run this hook in the local - # environment instead of a pre-commit isolated one. This is needed - # to ensure mypy correctly parses the project. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + additional_dependencies: [flaky] - # It may cause trouble in that it adds environmental variables out - # of our control to the mix. Unfortunately, there's nothing we can - # do about per pre-commit's author. - # See https://github.com/pre-commit/pre-commit/issues/730 for details. - args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters/.* - language: system - - id: mypy - alias: mypy-check - stages: [manual] - args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters - language: system +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + args: + - --show-error-codes + - --pretty + - --ignore-missing-imports + - --explicit-package-bases + files: ^dbt/adapters + additional_dependencies: + - types-pytz + - types-requests diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index cc58c02a6..b890127c6 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -50,7 +50,7 @@ class UserSSLMode(StrEnum): @classmethod def default(cls) -> "UserSSLMode": # default for `psycopg2`, which aligns with dbt-redshift 1.4 and provides backwards compatibility - return cls.prefer + return cls("prefer") class RedshiftSSLMode(StrEnum): @@ -60,11 +60,11 @@ class RedshiftSSLMode(StrEnum): SSL_MODE_TRANSLATION = { UserSSLMode.disable: None, - UserSSLMode.allow: RedshiftSSLMode.verify_ca, - UserSSLMode.prefer: RedshiftSSLMode.verify_ca, - UserSSLMode.require: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_ca: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_full: RedshiftSSLMode.verify_full, + UserSSLMode.allow: RedshiftSSLMode("verify-ca"), + UserSSLMode.prefer: RedshiftSSLMode("verify-ca"), + UserSSLMode.require: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_ca: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_full: RedshiftSSLMode("verify-full"), } diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/relation_configs/dist.py index c41eda578..0104d8db4 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/relation_configs/dist.py @@ -24,7 +24,7 @@ class RedshiftDistStyle(StrEnum): @classmethod def default(cls) -> "RedshiftDistStyle": - return cls.auto + return cls("auto") @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -103,7 +103,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> dict: config = {"diststyle": diststyle} else: - config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} + config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} # type: ignore return config diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index 05f4b170d..f6d93754e 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -57,7 +57,7 @@ class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigV database_name: str query: str backup: bool = field(default=True, compare=False, hash=False) - dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) + dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle("even")) sort: RedshiftSortConfig = RedshiftSortConfig() autorefresh: bool = False diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/relation_configs/sort.py index e44784c2f..91152615e 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/relation_configs/sort.py @@ -23,11 +23,11 @@ class RedshiftSortStyle(StrEnum): @classmethod def default(cls) -> "RedshiftSortStyle": - return cls.auto + return cls("auto") @classmethod def default_with_columns(cls) -> "RedshiftSortStyle": - return cls.compound + return cls("compound") @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/dev-requirements.txt b/dev-requirements.txt index 85edead99..d02863ae0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,28 +5,22 @@ git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core git+https://github.com/dbt-labs/dbt-postgres.git -# if version 1.x or greater -> pin to major version -# if version 0.x -> pin to minor -black>=24.3 -bumpversion~=0.6.0 -click~=8.1 +# dev +ipdb~=0.13.13 +pre-commit==3.7.0;python_version >="3.9" +pre-commit==3.5.0;python_version <"3.9" + +# test ddtrace==2.3.0 -flake8~=6.1 -flaky~=3.7 freezegun~=1.3 -ipdb~=0.13.13 -mypy==1.7.1 # patch updates have historically introduced breaking changes -pip-tools~=7.3 -pre-commit~=3.5 -pre-commit-hooks~=4.5 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 -pytz~=2023.3 tox~=4.11 -types-pytz~=2023.3 -types-requests~=2.31 + +# build +bumpversion~=0.6.0 twine~=4.0 wheel~=0.42 diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index b6e603581..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -namespace_packages = True diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py index 5e454fe5e..8e4f6ca3e 100644 --- a/tests/unit/relation_configs/test_materialized_view.py +++ b/tests/unit/relation_configs/test_materialized_view.py @@ -14,8 +14,8 @@ def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) config_dict = config.parse_relation_config(model_node) assert isinstance(config_dict["autorefresh"], bool) @@ -33,8 +33,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(TypeError): config.parse_relation_config(model_node) @@ -48,8 +48,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(ValueError): config.parse_relation_config(model_node) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 3fc1d7ec6..4de03ec80 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -2,6 +2,7 @@ Note that all imports should be inside the functions to avoid import/mocking issues. """ + import string import os from unittest import mock From 9cc7a7c105a26cad783bfa0eb7787329ba3fff19 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:08:35 -0400 Subject: [PATCH 23/61] workflow updates to support iam user auth testing and new variables (#778) --- .github/workflows/integration.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9d1fe0807..3fac0b0db 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -174,6 +174,17 @@ jobs: pip install bumpversion ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + - name: Create AWS IAM profile + run: | + aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure set region $AWS_REGION + aws configure set output json + env: + AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + - name: Run tox (redshift) if: matrix.adapter == 'redshift' env: @@ -182,6 +193,10 @@ jobs: REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} + REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 From e1cc700e0cefd04cd8dff79020693a05cd9bce68 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:11:34 -0400 Subject: [PATCH 24/61] Make space for both user and role in testing, add role environment variables (#783) * make space for both iam user and iam role in testing --- .github/workflows/integration.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3fac0b0db..21f013a7f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -176,14 +176,20 @@ jobs: - name: Create AWS IAM profile run: | - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region $AWS_REGION - aws configure set output json + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_USER_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN env: - AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} - name: Run tox (redshift) if: matrix.adapter == 'redshift' @@ -194,9 +200,11 @@ jobs: REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} - REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + REDSHIFT_TEST_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 From 557dbb655ae53952be81bf847dd625753683c4f7 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 23 Apr 2024 23:46:09 -0400 Subject: [PATCH 25/61] add region to CI env (#784) --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 21f013a7f..e0113dcbc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -199,6 +199,7 @@ jobs: REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} From 9b4e62aa1dc7d4deff46ebc0166f00a3e27c7b47 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:17:13 -0400 Subject: [PATCH 26/61] Update workflow to support iam user and iam role tests (#786) * update workflow to support iam user and iam role tests --- .github/workflows/integration.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e0113dcbc..ad29fef1e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -174,22 +174,32 @@ jobs: pip install bumpversion ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Create AWS IAM profile + - name: Create AWS IAM profiles run: | - aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION aws configure --profile $AWS_USER_PROFILE set output json - aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_USER_PROFILE + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json env: - AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} - AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - name: Run tox (redshift) if: matrix.adapter == 'redshift' @@ -205,7 +215,6 @@ jobs: REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} - REDSHIFT_TEST_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 From ddfc36c3806ab0e370bda68a4b84b45143516b03 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:20:14 -0400 Subject: [PATCH 27/61] pin macos-runners to macos-12 (#787) --- .github/scripts/integration-test-matrix.js | 4 ++-- .github/workflows/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 7db445d9e..ccb6d949d 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -37,7 +37,7 @@ module.exports = ({ context }) => { if (labels.includes("test macos") || testAllLabel) { include.push({ - os: "macos-latest", + os: "macos-12", adapter, "python-version": pythonVersion, }); @@ -70,7 +70,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { + for (const operatingSystem of ["windows-latest", "macos-12"]) { include.push({ os: operatingSystem, adapter: adapter, diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 389f55a88..d8c126301 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-12, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11'] steps: From 1afa51e11e2e5f10b4f256a4702790dae8da767f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 23:59:10 +0000 Subject: [PATCH 28/61] Bump actions/download-artifact from 3 to 4 (#765) * Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240412-155549.yaml | 6 ++++++ .github/workflows/main.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240412-155549.yaml diff --git a/.changes/unreleased/Dependencies-20240412-155549.yaml b/.changes/unreleased/Dependencies-20240412-155549.yaml new file mode 100644 index 000000000..b50bbd4fa --- /dev/null +++ b/.changes/unreleased/Dependencies-20240412-155549.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump actions/download-artifact from 3 to 4" +time: 2024-04-12T15:55:49.00000Z +custom: + Author: dependabot[bot] + PR: 765 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d8c126301..5527b568f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -188,7 +188,7 @@ jobs: python -m pip install --upgrade wheel python -m pip --version - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: dist path: dist/ From 32410a605817589056b683d0d6ce2fccd8126ad3 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:32:00 -0400 Subject: [PATCH 29/61] Revert "Bump actions/download-artifact from 3 to 4 (#765)" (#796) This reverts commit 1afa51e11e2e5f10b4f256a4702790dae8da767f. --- .changes/unreleased/Dependencies-20240412-155549.yaml | 6 ------ .github/workflows/main.yml | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 .changes/unreleased/Dependencies-20240412-155549.yaml diff --git a/.changes/unreleased/Dependencies-20240412-155549.yaml b/.changes/unreleased/Dependencies-20240412-155549.yaml deleted file mode 100644 index b50bbd4fa..000000000 --- a/.changes/unreleased/Dependencies-20240412-155549.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump actions/download-artifact from 3 to 4" -time: 2024-04-12T15:55:49.00000Z -custom: - Author: dependabot[bot] - PR: 765 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5527b568f..d8c126301 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -188,7 +188,7 @@ jobs: python -m pip install --upgrade wheel python -m pip --version - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: dist path: dist/ From 6031b238afaf232a173b3ca5f672d8bd191b57ed Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 2 May 2024 14:29:25 -0700 Subject: [PATCH 30/61] Stop adding aliases to render_limited output (#782) * Stop adding aliases to render_limited output * add changie --- .changes/unreleased/Fixes-20240423-131503.yaml | 6 ++++++ dbt/adapters/redshift/relation.py | 1 + tests/functional/adapter/empty/test_empty.py | 6 +++++- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20240423-131503.yaml diff --git a/.changes/unreleased/Fixes-20240423-131503.yaml b/.changes/unreleased/Fixes-20240423-131503.yaml new file mode 100644 index 000000000..8b5b82b38 --- /dev/null +++ b/.changes/unreleased/Fixes-20240423-131503.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Stop adding aliases to subqueries when calling with `--empty` +time: 2024-04-23T13:15:03.118968-07:00 +custom: + Author: colin-rogers-dbt + Issue: "782" diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index e120a2fac..eaf60f54c 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -27,6 +27,7 @@ class RedshiftRelation(BaseRelation): include_policy = RedshiftIncludePolicy # type: ignore quote_policy = RedshiftQuotePolicy # type: ignore + require_alias: bool = False relation_configs = { RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, } diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 3368a90f9..27f36f1df 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -1,5 +1,9 @@ -from dbt.tests.adapter.empty.test_empty import BaseTestEmpty +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef class TestRedshiftEmpty(BaseTestEmpty): pass + + +class TestRedshiftEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + pass From aaeec6d9bb062e31ea44b3b8f0f5901caa644f5b Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Fri, 3 May 2024 12:58:58 -0400 Subject: [PATCH 31/61] Cleanup main after cutting new 1.8.latest branch (#810) * Clean up changelog on main * Bumping version to 1.9.0a1 * Code quality cleanup --- .bumpversion.cfg | 2 +- .changes/1.8.0-b1.md | 49 ---------- .changes/1.8.0-b2.md | 22 ----- .changes/1.8.0-b3.md | 16 ---- .../1.8.0/Dependencies-20230912-120620.yaml | 6 -- .../1.8.0/Dependencies-20231002-164037.yaml | 6 -- .../1.8.0/Dependencies-20231009-192801.yaml | 6 -- .../1.8.0/Dependencies-20231010-195348.yaml | 6 -- .../1.8.0/Dependencies-20231013-190517.yaml | 6 -- .../1.8.0/Dependencies-20231017-191545.yaml | 6 -- .../1.8.0/Dependencies-20231027-173152.yaml | 6 -- .../1.8.0/Dependencies-20231030-193514.yaml | 6 -- .../1.8.0/Dependencies-20231108-190800.yaml | 6 -- .../1.8.0/Dependencies-20231110-192349.yaml | 6 -- .../1.8.0/Dependencies-20231113-195504.yaml | 6 -- .../1.8.0/Dependencies-20231116-194405.yaml | 6 -- .../1.8.0/Dependencies-20231127-201640.yaml | 6 -- .../1.8.0/Dependencies-20231127-201942.yaml | 6 -- .../1.8.0/Dependencies-20231128-194822.yaml | 6 -- .../1.8.0/Dependencies-20231129-195044.yaml | 6 -- .../1.8.0/Dependencies-20231130-044332.yaml | 6 -- .../1.8.0/Dependencies-20231204-193730.yaml | 6 -- .../1.8.0/Dependencies-20231212-195417.yaml | 6 -- .../1.8.0/Dependencies-20240118-095025.yaml | 6 -- .../1.8.0/Dependencies-20240124-111727.yaml | 6 -- .../1.8.0/Dependencies-20240319-101832.yaml | 6 -- .../1.8.0/Dependencies-20240403-134607.yaml | 6 -- .changes/1.8.0/Features-20231030-101055.yaml | 6 -- .changes/1.8.0/Features-20231214-195655.yaml | 6 -- .changes/1.8.0/Features-20231219-120533.yaml | 6 -- .changes/1.8.0/Features-20240318-030018.yaml | 6 -- .changes/1.8.0/Fixes-20231025-203732.yaml | 6 -- .changes/1.8.0/Fixes-20231026-164623.yaml | 6 -- .changes/1.8.0/Fixes-20231030-234315.yaml | 6 -- .changes/1.8.0/Fixes-20231103-181357.yaml | 6 -- .changes/1.8.0/Fixes-20240206-132326.yaml | 7 -- .changes/1.8.0/Fixes-20240317-113447.yaml | 6 -- .changes/1.8.0/Fixes-20240326-123703.yaml | 6 -- .changes/1.8.0/Security-20240327-191927.yaml | 6 -- .changes/1.8.0/Security-20240416-195919.yaml | 6 -- .../1.8.0/Under the Hood-20231119-132157.yaml | 6 -- .../1.8.0/Under the Hood-20240102-152425.yaml | 6 -- .../1.8.0/Under the Hood-20240227-002713.yaml | 6 -- .../1.8.0/Under the Hood-20240410-182912.yaml | 6 -- .../unreleased/Features-20240404-171441.yaml | 6 -- CHANGELOG.md | 91 ------------------- dbt/adapters/redshift/__version__.py | 2 +- 47 files changed, 2 insertions(+), 427 deletions(-) delete mode 100644 .changes/1.8.0-b1.md delete mode 100644 .changes/1.8.0-b2.md delete mode 100644 .changes/1.8.0-b3.md delete mode 100644 .changes/1.8.0/Dependencies-20230912-120620.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231002-164037.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231009-192801.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231010-195348.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231013-190517.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231017-191545.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231027-173152.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231030-193514.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231108-190800.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231110-192349.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231113-195504.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231116-194405.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231127-201640.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231127-201942.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231128-194822.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231129-195044.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231130-044332.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231204-193730.yaml delete mode 100644 .changes/1.8.0/Dependencies-20231212-195417.yaml delete mode 100644 .changes/1.8.0/Dependencies-20240118-095025.yaml delete mode 100644 .changes/1.8.0/Dependencies-20240124-111727.yaml delete mode 100644 .changes/1.8.0/Dependencies-20240319-101832.yaml delete mode 100644 .changes/1.8.0/Dependencies-20240403-134607.yaml delete mode 100644 .changes/1.8.0/Features-20231030-101055.yaml delete mode 100644 .changes/1.8.0/Features-20231214-195655.yaml delete mode 100644 .changes/1.8.0/Features-20231219-120533.yaml delete mode 100644 .changes/1.8.0/Features-20240318-030018.yaml delete mode 100644 .changes/1.8.0/Fixes-20231025-203732.yaml delete mode 100644 .changes/1.8.0/Fixes-20231026-164623.yaml delete mode 100644 .changes/1.8.0/Fixes-20231030-234315.yaml delete mode 100644 .changes/1.8.0/Fixes-20231103-181357.yaml delete mode 100644 .changes/1.8.0/Fixes-20240206-132326.yaml delete mode 100644 .changes/1.8.0/Fixes-20240317-113447.yaml delete mode 100644 .changes/1.8.0/Fixes-20240326-123703.yaml delete mode 100644 .changes/1.8.0/Security-20240327-191927.yaml delete mode 100644 .changes/1.8.0/Security-20240416-195919.yaml delete mode 100644 .changes/1.8.0/Under the Hood-20231119-132157.yaml delete mode 100644 .changes/1.8.0/Under the Hood-20240102-152425.yaml delete mode 100644 .changes/1.8.0/Under the Hood-20240227-002713.yaml delete mode 100644 .changes/1.8.0/Under the Hood-20240410-182912.yaml delete mode 100644 .changes/unreleased/Features-20240404-171441.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 52cc0acca..5baa94d26 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0b3 +current_version = 1.9.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b1.md b/.changes/1.8.0-b1.md deleted file mode 100644 index e8e4cbd2b..000000000 --- a/.changes/1.8.0-b1.md +++ /dev/null @@ -1,49 +0,0 @@ -## dbt-redshift 1.8.0-b1 - March 01, 2024 - -### Features - -- allow user to set debug level for redshift-connector via env var ([#650](https://github.com/dbt-labs/dbt-redshift/issues/650)) -- Support limiting get_catalog by object name ([#625](https://github.com/dbt-labs/dbt-redshift/issues/625)) -- Add support for checking table-last-modified by metadata ([#615](https://github.com/dbt-labs/dbt-redshift/issues/615)) - -### Fixes - -- Fix parsing of database results for materialized view auto refresh ([#643](https://github.com/dbt-labs/dbt-redshift/issues/643)) -- Fix describe_materialized_view for Redshift Serverless ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) -- Catalog queries assign the appropriate type for materialized views ([#652](https://github.com/dbt-labs/dbt-redshift/issues/652)) -- Fix handling of `backup` parameter during config change monitoring ([#621](https://github.com/dbt-labs/dbt-redshift/issues/621)) -- Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes external tables ([#710](https://github.com/dbt-labs/dbt-redshift/issues/710)) - -### Under the Hood - -- Add tests for --empty flag ([#667](https://github.com/dbt-labs/dbt-redshift/issues/667)) -- Update base adapter references as part of decoupling migration ([#698](https://github.com/dbt-labs/dbt-redshift/issues/698)) - -### Dependencies - -- Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) -- Update ddtrace requirement from ~=1.19 to ~=1.20 ([#622](https://github.com/dbt-labs/dbt-redshift/pull/622)) -- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#627](https://github.com/dbt-labs/dbt-redshift/pull/627)) -- Bump mypy from 1.5.1 to 1.6.0 ([#629](https://github.com/dbt-labs/dbt-redshift/pull/629)) -- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#634](https://github.com/dbt-labs/dbt-redshift/pull/634)) -- Update black requirement from ~=23.9 to ~=23.10 ([#636](https://github.com/dbt-labs/dbt-redshift/pull/636)) -- Bump mypy from 1.6.0 to 1.6.1 ([#648](https://github.com/dbt-labs/dbt-redshift/pull/648)) -- Update ddtrace requirement from ~=1.20 to ~=2.1 ([#651](https://github.com/dbt-labs/dbt-redshift/pull/651)) -- Update black requirement from ~=23.10 to ~=23.11 ([#660](https://github.com/dbt-labs/dbt-redshift/pull/660)) -- Bump mypy from 1.6.1 to 1.7.0 ([#662](https://github.com/dbt-labs/dbt-redshift/pull/662)) -- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#664](https://github.com/dbt-labs/dbt-redshift/pull/664)) -- Update ddtrace requirement from ~=2.1 to ~=2.2 ([#665](https://github.com/dbt-labs/dbt-redshift/pull/665)) -- Update ddtrace requirement from ~=2.2 to ~=2.3 ([#669](https://github.com/dbt-labs/dbt-redshift/pull/669)) -- Update wheel requirement from ~=0.41 to ~=0.42 ([#670](https://github.com/dbt-labs/dbt-redshift/pull/670)) -- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#672](https://github.com/dbt-labs/dbt-redshift/pull/672)) -- Remove direct boto3 dependency ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) -- Bump mypy from 1.7.0 to 1.7.1 ([#676](https://github.com/dbt-labs/dbt-redshift/pull/676)) -- Update freezegun requirement from ~=1.2 to ~=1.3 ([#681](https://github.com/dbt-labs/dbt-redshift/pull/681)) -- Update black requirement from ~=23.11 to ~=23.12 ([#688](https://github.com/dbt-labs/dbt-redshift/pull/688)) -- upgrade redshift driver to 2.0.918 ([#700](https://github.com/dbt-labs/dbt-redshift/pull/700)) -- Migrate to dbt-common and dbt-adapters ([#706](https://github.com/dbt-labs/dbt-redshift/pull/706)) - -### Contributors -- [@hexDoor](https://github.com/hexDoor) ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) -- [@reptillicus](https://github.com/reptillicus) ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) -- [@soksamnanglim](https://github.com/soksamnanglim) ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) diff --git a/.changes/1.8.0-b2.md b/.changes/1.8.0-b2.md deleted file mode 100644 index 47996a7ce..000000000 --- a/.changes/1.8.0-b2.md +++ /dev/null @@ -1,22 +0,0 @@ -## dbt-redshift 1.8.0-b2 - April 03, 2024 - -### Features - -- Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-redshift/issues/38)) - -### Fixes - -- Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 ([#730](https://github.com/dbt-labs/dbt-redshift/issues/730)) - -### Under the Hood - -- Add unit test for transaction semantics. ([#722](https://github.com/dbt-labs/dbt-redshift/issues/722)) - -### Dependencies - -- hard pin ddtrace ([#733](https://github.com/dbt-labs/dbt-redshift/pull/733)) -- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#756](https://github.com/dbt-labs/dbt-redshift/pull/756)) - -### Security - -- Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) diff --git a/.changes/1.8.0-b3.md b/.changes/1.8.0-b3.md deleted file mode 100644 index 5f9d828c3..000000000 --- a/.changes/1.8.0-b3.md +++ /dev/null @@ -1,16 +0,0 @@ -## dbt-redshift 1.8.0-b3 - April 18, 2024 - -### Fixes - -- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) - -### Under the Hood - -- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) - -### Security - -- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) - -### Contributors -- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) diff --git a/.changes/1.8.0/Dependencies-20230912-120620.yaml b/.changes/1.8.0/Dependencies-20230912-120620.yaml deleted file mode 100644 index be139237d..000000000 --- a/.changes/1.8.0/Dependencies-20230912-120620.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 -time: 2023-09-12T12:06:20.401643-07:00 -custom: - Author: soksamnanglim - PR: "601" diff --git a/.changes/1.8.0/Dependencies-20231002-164037.yaml b/.changes/1.8.0/Dependencies-20231002-164037.yaml deleted file mode 100644 index 54f51a402..000000000 --- a/.changes/1.8.0/Dependencies-20231002-164037.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=1.19 to ~=1.20" -time: 2023-10-02T16:40:37.00000Z -custom: - Author: dependabot[bot] - PR: 622 diff --git a/.changes/1.8.0/Dependencies-20231009-192801.yaml b/.changes/1.8.0/Dependencies-20231009-192801.yaml deleted file mode 100644 index 2a53a074f..000000000 --- a/.changes/1.8.0/Dependencies-20231009-192801.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit-hooks requirement from ~=4.4 to ~=4.5" -time: 2023-10-09T19:28:01.00000Z -custom: - Author: dependabot[bot] - PR: 627 diff --git a/.changes/1.8.0/Dependencies-20231010-195348.yaml b/.changes/1.8.0/Dependencies-20231010-195348.yaml deleted file mode 100644 index 4e8af02c1..000000000 --- a/.changes/1.8.0/Dependencies-20231010-195348.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.5.1 to 1.6.0" -time: 2023-10-10T19:53:48.00000Z -custom: - Author: dependabot[bot] - PR: 629 diff --git a/.changes/1.8.0/Dependencies-20231013-190517.yaml b/.changes/1.8.0/Dependencies-20231013-190517.yaml deleted file mode 100644 index 8db17538a..000000000 --- a/.changes/1.8.0/Dependencies-20231013-190517.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit requirement from ~=3.4 to ~=3.5" -time: 2023-10-13T19:05:17.00000Z -custom: - Author: dependabot[bot] - PR: 634 diff --git a/.changes/1.8.0/Dependencies-20231017-191545.yaml b/.changes/1.8.0/Dependencies-20231017-191545.yaml deleted file mode 100644 index 407615e32..000000000 --- a/.changes/1.8.0/Dependencies-20231017-191545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.9 to ~=23.10" -time: 2023-10-17T19:15:45.00000Z -custom: - Author: dependabot[bot] - PR: 636 diff --git a/.changes/1.8.0/Dependencies-20231027-173152.yaml b/.changes/1.8.0/Dependencies-20231027-173152.yaml deleted file mode 100644 index 8ed464977..000000000 --- a/.changes/1.8.0/Dependencies-20231027-173152.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.6.0 to 1.6.1" -time: 2023-10-27T17:31:52.00000Z -custom: - Author: dependabot[bot] - PR: 648 diff --git a/.changes/1.8.0/Dependencies-20231030-193514.yaml b/.changes/1.8.0/Dependencies-20231030-193514.yaml deleted file mode 100644 index e2771051b..000000000 --- a/.changes/1.8.0/Dependencies-20231030-193514.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=1.20 to ~=2.1" -time: 2023-10-30T19:35:14.00000Z -custom: - Author: dependabot[bot] - PR: 651 diff --git a/.changes/1.8.0/Dependencies-20231108-190800.yaml b/.changes/1.8.0/Dependencies-20231108-190800.yaml deleted file mode 100644 index e8ce0c5b4..000000000 --- a/.changes/1.8.0/Dependencies-20231108-190800.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.10 to ~=23.11" -time: 2023-11-08T19:08:00.00000Z -custom: - Author: dependabot[bot] - PR: 660 diff --git a/.changes/1.8.0/Dependencies-20231110-192349.yaml b/.changes/1.8.0/Dependencies-20231110-192349.yaml deleted file mode 100644 index 90b285065..000000000 --- a/.changes/1.8.0/Dependencies-20231110-192349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.6.1 to 1.7.0" -time: 2023-11-10T19:23:49.00000Z -custom: - Author: dependabot[bot] - PR: 662 diff --git a/.changes/1.8.0/Dependencies-20231113-195504.yaml b/.changes/1.8.0/Dependencies-20231113-195504.yaml deleted file mode 100644 index 39b81e3e8..000000000 --- a/.changes/1.8.0/Dependencies-20231113-195504.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.3 to ~=3.4" -time: 2023-11-13T19:55:04.00000Z -custom: - Author: dependabot[bot] - PR: 664 diff --git a/.changes/1.8.0/Dependencies-20231116-194405.yaml b/.changes/1.8.0/Dependencies-20231116-194405.yaml deleted file mode 100644 index 608ade10b..000000000 --- a/.changes/1.8.0/Dependencies-20231116-194405.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=2.1 to ~=2.2" -time: 2023-11-16T19:44:05.00000Z -custom: - Author: dependabot[bot] - PR: 665 diff --git a/.changes/1.8.0/Dependencies-20231127-201640.yaml b/.changes/1.8.0/Dependencies-20231127-201640.yaml deleted file mode 100644 index bbcdb6ff9..000000000 --- a/.changes/1.8.0/Dependencies-20231127-201640.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=2.2 to ~=2.3" -time: 2023-11-27T20:16:40.00000Z -custom: - Author: dependabot[bot] - PR: 669 diff --git a/.changes/1.8.0/Dependencies-20231127-201942.yaml b/.changes/1.8.0/Dependencies-20231127-201942.yaml deleted file mode 100644 index ac6c15d53..000000000 --- a/.changes/1.8.0/Dependencies-20231127-201942.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.41 to ~=0.42" -time: 2023-11-27T20:19:42.00000Z -custom: - Author: dependabot[bot] - PR: 670 diff --git a/.changes/1.8.0/Dependencies-20231128-194822.yaml b/.changes/1.8.0/Dependencies-20231128-194822.yaml deleted file mode 100644 index 820ba25d1..000000000 --- a/.changes/1.8.0/Dependencies-20231128-194822.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.4 to ~=3.5" -time: 2023-11-28T19:48:22.00000Z -custom: - Author: dependabot[bot] - PR: 672 diff --git a/.changes/1.8.0/Dependencies-20231129-195044.yaml b/.changes/1.8.0/Dependencies-20231129-195044.yaml deleted file mode 100644 index 235a89777..000000000 --- a/.changes/1.8.0/Dependencies-20231129-195044.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.7.0 to 1.7.1" -time: 2023-11-29T19:50:44.00000Z -custom: - Author: dependabot[bot] - PR: 676 diff --git a/.changes/1.8.0/Dependencies-20231130-044332.yaml b/.changes/1.8.0/Dependencies-20231130-044332.yaml deleted file mode 100644 index 0fbeb9a18..000000000 --- a/.changes/1.8.0/Dependencies-20231130-044332.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Remove direct boto3 dependency -time: 2023-11-30T04:43:32.872452+11:00 -custom: - Author: hexDoor - PR: "674" diff --git a/.changes/1.8.0/Dependencies-20231204-193730.yaml b/.changes/1.8.0/Dependencies-20231204-193730.yaml deleted file mode 100644 index b1d803288..000000000 --- a/.changes/1.8.0/Dependencies-20231204-193730.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update freezegun requirement from ~=1.2 to ~=1.3" -time: 2023-12-04T19:37:30.00000Z -custom: - Author: dependabot[bot] - PR: 681 diff --git a/.changes/1.8.0/Dependencies-20231212-195417.yaml b/.changes/1.8.0/Dependencies-20231212-195417.yaml deleted file mode 100644 index 2117f6ead..000000000 --- a/.changes/1.8.0/Dependencies-20231212-195417.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.11 to ~=23.12" -time: 2023-12-12T19:54:17.00000Z -custom: - Author: dependabot[bot] - PR: 688 diff --git a/.changes/1.8.0/Dependencies-20240118-095025.yaml b/.changes/1.8.0/Dependencies-20240118-095025.yaml deleted file mode 100644 index ba61481ba..000000000 --- a/.changes/1.8.0/Dependencies-20240118-095025.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: upgrade redshift driver to 2.0.918 -time: 2024-01-18T09:50:25.871857-08:00 -custom: - Author: colin-rogers-dbt - PR: "700" diff --git a/.changes/1.8.0/Dependencies-20240124-111727.yaml b/.changes/1.8.0/Dependencies-20240124-111727.yaml deleted file mode 100644 index f5874984a..000000000 --- a/.changes/1.8.0/Dependencies-20240124-111727.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Migrate to dbt-common and dbt-adapters -time: 2024-01-24T11:17:27.580348-08:00 -custom: - Author: colin-rogers-dbt - PR: "706" diff --git a/.changes/1.8.0/Dependencies-20240319-101832.yaml b/.changes/1.8.0/Dependencies-20240319-101832.yaml deleted file mode 100644 index 7b5f0eb3a..000000000 --- a/.changes/1.8.0/Dependencies-20240319-101832.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: hard pin ddtrace -time: 2024-03-19T10:18:32.426032-05:00 -custom: - Author: McKnight-42 - PR: "733" diff --git a/.changes/1.8.0/Dependencies-20240403-134607.yaml b/.changes/1.8.0/Dependencies-20240403-134607.yaml deleted file mode 100644 index ccc80c5e9..000000000 --- a/.changes/1.8.0/Dependencies-20240403-134607.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Add `dbt-core` as a dependency to preserve backwards compatibility for installation -time: 2024-04-03T13:46:07.335865-04:00 -custom: - Author: mikealfare - PR: "756" diff --git a/.changes/1.8.0/Features-20231030-101055.yaml b/.changes/1.8.0/Features-20231030-101055.yaml deleted file mode 100644 index 8648762e2..000000000 --- a/.changes/1.8.0/Features-20231030-101055.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: allow user to set debug level for redshift-connector via env var -time: 2023-10-30T10:10:55.976191-07:00 -custom: - Author: colin-rogers-dbt - Issue: "650" diff --git a/.changes/1.8.0/Features-20231214-195655.yaml b/.changes/1.8.0/Features-20231214-195655.yaml deleted file mode 100644 index 93f372c67..000000000 --- a/.changes/1.8.0/Features-20231214-195655.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support limiting get_catalog by object name -time: 2023-12-14T19:56:55.124051-05:00 -custom: - Author: mikealfare - Issue: "625" diff --git a/.changes/1.8.0/Features-20231219-120533.yaml b/.changes/1.8.0/Features-20231219-120533.yaml deleted file mode 100644 index 15b5ba1f1..000000000 --- a/.changes/1.8.0/Features-20231219-120533.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for checking table-last-modified by metadata -time: 2023-12-19T12:05:33.784649-05:00 -custom: - Author: mikealfare - Issue: "615" diff --git a/.changes/1.8.0/Features-20240318-030018.yaml b/.changes/1.8.0/Features-20240318-030018.yaml deleted file mode 100644 index badab8088..000000000 --- a/.changes/1.8.0/Features-20240318-030018.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add new workflow for internal patch releases -time: 2024-03-18T03:00:18.845618-07:00 -custom: - Author: versusfacit - Issue: "38" diff --git a/.changes/1.8.0/Fixes-20231025-203732.yaml b/.changes/1.8.0/Fixes-20231025-203732.yaml deleted file mode 100644 index 9e6bf1af7..000000000 --- a/.changes/1.8.0/Fixes-20231025-203732.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix parsing of database results for materialized view auto refresh -time: 2023-10-25T20:37:32.191259-04:00 -custom: - Author: mikealfare - Issue: "643" diff --git a/.changes/1.8.0/Fixes-20231026-164623.yaml b/.changes/1.8.0/Fixes-20231026-164623.yaml deleted file mode 100644 index f06eff381..000000000 --- a/.changes/1.8.0/Fixes-20231026-164623.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix describe_materialized_view for Redshift Serverless -time: 2023-10-26T16:46:23.253837-06:00 -custom: - Author: reptillicus - Issue: "641" diff --git a/.changes/1.8.0/Fixes-20231030-234315.yaml b/.changes/1.8.0/Fixes-20231030-234315.yaml deleted file mode 100644 index 823e252dd..000000000 --- a/.changes/1.8.0/Fixes-20231030-234315.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Catalog queries assign the appropriate type for materialized views -time: 2023-10-30T23:43:15.281772-04:00 -custom: - Author: mikealfare - Issue: "652" diff --git a/.changes/1.8.0/Fixes-20231103-181357.yaml b/.changes/1.8.0/Fixes-20231103-181357.yaml deleted file mode 100644 index 2995560d2..000000000 --- a/.changes/1.8.0/Fixes-20231103-181357.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix handling of `backup` parameter during config change monitoring -time: 2023-11-03T18:13:57.401994-04:00 -custom: - Author: mikealfare - Issue: "621" diff --git a/.changes/1.8.0/Fixes-20240206-132326.yaml b/.changes/1.8.0/Fixes-20240206-132326.yaml deleted file mode 100644 index 45b3a74c8..000000000 --- a/.changes/1.8.0/Fixes-20240206-132326.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes - external tables -time: 2024-02-06T13:23:26.061133-05:00 -custom: - Author: mikealfare - Issue: "710" diff --git a/.changes/1.8.0/Fixes-20240317-113447.yaml b/.changes/1.8.0/Fixes-20240317-113447.yaml deleted file mode 100644 index 4338fc95b..000000000 --- a/.changes/1.8.0/Fixes-20240317-113447.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 -time: 2024-03-17T11:34:47.873169-04:00 -custom: - Author: mikealfare - Issue: "730" diff --git a/.changes/1.8.0/Fixes-20240326-123703.yaml b/.changes/1.8.0/Fixes-20240326-123703.yaml deleted file mode 100644 index 5d9bee694..000000000 --- a/.changes/1.8.0/Fixes-20240326-123703.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: dbt can cancel open queries upon interrupt -time: 2024-03-26T12:37:03.17481-05:00 -custom: - Author: holly-evans - Issue: "705" diff --git a/.changes/1.8.0/Security-20240327-191927.yaml b/.changes/1.8.0/Security-20240327-191927.yaml deleted file mode 100644 index 763273c79..000000000 --- a/.changes/1.8.0/Security-20240327-191927.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Security -body: Pin `black>=24.3` in `dev-requirements.txt` -time: 2024-03-27T19:19:27.243854-04:00 -custom: - Author: mikealfare - PR: "743" diff --git a/.changes/1.8.0/Security-20240416-195919.yaml b/.changes/1.8.0/Security-20240416-195919.yaml deleted file mode 100644 index af8fb6f1d..000000000 --- a/.changes/1.8.0/Security-20240416-195919.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Security -body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core -time: 2024-04-16T19:59:19.233806-05:00 -custom: - Author: McKnight-42 - PR: "768" diff --git a/.changes/1.8.0/Under the Hood-20231119-132157.yaml b/.changes/1.8.0/Under the Hood-20231119-132157.yaml deleted file mode 100644 index 760c08ccf..000000000 --- a/.changes/1.8.0/Under the Hood-20231119-132157.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add tests for --empty flag -time: 2023-11-19T13:21:57.588501-05:00 -custom: - Author: michelleark - Issue: "667" diff --git a/.changes/1.8.0/Under the Hood-20240102-152425.yaml b/.changes/1.8.0/Under the Hood-20240102-152425.yaml deleted file mode 100644 index 23a3eeb46..000000000 --- a/.changes/1.8.0/Under the Hood-20240102-152425.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update base adapter references as part of decoupling migration -time: 2024-01-02T15:24:25.890421-08:00 -custom: - Author: colin-rogers-dbt VersusFacit - Issue: "698" diff --git a/.changes/1.8.0/Under the Hood-20240227-002713.yaml b/.changes/1.8.0/Under the Hood-20240227-002713.yaml deleted file mode 100644 index 48b8de8e4..000000000 --- a/.changes/1.8.0/Under the Hood-20240227-002713.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add unit test for transaction semantics. -time: 2024-02-27T00:27:13.299107-08:00 -custom: - Author: versusfacit - Issue: "722" diff --git a/.changes/1.8.0/Under the Hood-20240410-182912.yaml b/.changes/1.8.0/Under the Hood-20240410-182912.yaml deleted file mode 100644 index e6826c4d5..000000000 --- a/.changes/1.8.0/Under the Hood-20240410-182912.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update dependabot config to cover GHA -time: 2024-04-10T18:29:12.026746-04:00 -custom: - Author: mikealfare - Issue: "759" diff --git a/.changes/unreleased/Features-20240404-171441.yaml b/.changes/unreleased/Features-20240404-171441.yaml deleted file mode 100644 index f1ac623c0..000000000 --- a/.changes/unreleased/Features-20240404-171441.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support TableLastModifiedMetadataBatch capability -time: 2024-04-04T17:14:41.313087-07:00 -custom: - Author: michelleark - Issue: "755" diff --git a/CHANGELOG.md b/CHANGELOG.md index f45988f03..04a8e7db1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,97 +5,6 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-redshift 1.8.0-b3 - April 18, 2024 - -### Fixes - -- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) - -### Under the Hood - -- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) - -### Security - -- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) - -### Contributors -- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) - - -## dbt-redshift 1.8.0-b2 - April 03, 2024 - -### Features - -- Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-redshift/issues/38)) - -### Fixes - -- Pin `redshift-connector` to <2.0.918 to avoid SSL error introduced in 2.0.918 ([#730](https://github.com/dbt-labs/dbt-redshift/issues/730)) - -### Under the Hood - -- Add unit test for transaction semantics. ([#722](https://github.com/dbt-labs/dbt-redshift/issues/722)) - -### Dependencies - -- hard pin ddtrace ([#733](https://github.com/dbt-labs/dbt-redshift/pull/733)) -- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#756](https://github.com/dbt-labs/dbt-redshift/pull/756)) - -### Security - -- Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) - -## dbt-redshift 1.8.0-b1 - March 01, 2024 - -### Features - -- allow user to set debug level for redshift-connector via env var ([#650](https://github.com/dbt-labs/dbt-redshift/issues/650)) -- Support limiting get_catalog by object name ([#625](https://github.com/dbt-labs/dbt-redshift/issues/625)) -- Add support for checking table-last-modified by metadata ([#615](https://github.com/dbt-labs/dbt-redshift/issues/615)) - -### Fixes - -- Fix parsing of database results for materialized view auto refresh ([#643](https://github.com/dbt-labs/dbt-redshift/issues/643)) -- Fix describe_materialized_view for Redshift Serverless ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) -- Catalog queries assign the appropriate type for materialized views ([#652](https://github.com/dbt-labs/dbt-redshift/issues/652)) -- Fix handling of `backup` parameter during config change monitoring ([#621](https://github.com/dbt-labs/dbt-redshift/issues/621)) -- Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes external tables ([#710](https://github.com/dbt-labs/dbt-redshift/issues/710)) - -### Under the Hood - -- Add tests for --empty flag ([#667](https://github.com/dbt-labs/dbt-redshift/issues/667)) -- Update base adapter references as part of decoupling migration ([#698](https://github.com/dbt-labs/dbt-redshift/issues/698)) - -### Dependencies - -- Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) -- Update ddtrace requirement from ~=1.19 to ~=1.20 ([#622](https://github.com/dbt-labs/dbt-redshift/pull/622)) -- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#627](https://github.com/dbt-labs/dbt-redshift/pull/627)) -- Bump mypy from 1.5.1 to 1.6.0 ([#629](https://github.com/dbt-labs/dbt-redshift/pull/629)) -- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#634](https://github.com/dbt-labs/dbt-redshift/pull/634)) -- Update black requirement from ~=23.9 to ~=23.10 ([#636](https://github.com/dbt-labs/dbt-redshift/pull/636)) -- Bump mypy from 1.6.0 to 1.6.1 ([#648](https://github.com/dbt-labs/dbt-redshift/pull/648)) -- Update ddtrace requirement from ~=1.20 to ~=2.1 ([#651](https://github.com/dbt-labs/dbt-redshift/pull/651)) -- Update black requirement from ~=23.10 to ~=23.11 ([#660](https://github.com/dbt-labs/dbt-redshift/pull/660)) -- Bump mypy from 1.6.1 to 1.7.0 ([#662](https://github.com/dbt-labs/dbt-redshift/pull/662)) -- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#664](https://github.com/dbt-labs/dbt-redshift/pull/664)) -- Update ddtrace requirement from ~=2.1 to ~=2.2 ([#665](https://github.com/dbt-labs/dbt-redshift/pull/665)) -- Update ddtrace requirement from ~=2.2 to ~=2.3 ([#669](https://github.com/dbt-labs/dbt-redshift/pull/669)) -- Update wheel requirement from ~=0.41 to ~=0.42 ([#670](https://github.com/dbt-labs/dbt-redshift/pull/670)) -- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#672](https://github.com/dbt-labs/dbt-redshift/pull/672)) -- Remove direct boto3 dependency ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) -- Bump mypy from 1.7.0 to 1.7.1 ([#676](https://github.com/dbt-labs/dbt-redshift/pull/676)) -- Update freezegun requirement from ~=1.2 to ~=1.3 ([#681](https://github.com/dbt-labs/dbt-redshift/pull/681)) -- Update black requirement from ~=23.11 to ~=23.12 ([#688](https://github.com/dbt-labs/dbt-redshift/pull/688)) -- upgrade redshift driver to 2.0.918 ([#700](https://github.com/dbt-labs/dbt-redshift/pull/700)) -- Migrate to dbt-common and dbt-adapters ([#706](https://github.com/dbt-labs/dbt-redshift/pull/706)) - -### Contributors -- [@hexDoor](https://github.com/hexDoor) ([#674](https://github.com/dbt-labs/dbt-redshift/pull/674)) -- [@reptillicus](https://github.com/reptillicus) ([#641](https://github.com/dbt-labs/dbt-redshift/issues/641)) -- [@soksamnanglim](https://github.com/soksamnanglim) ([#601](https://github.com/dbt-labs/dbt-redshift/pull/601)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index b0f82cbca..6698ed64c 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0b3" +version = "1.9.0a1" From 12b5cd70480079a666802ce8fb289901098c3cd5 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 7 May 2024 08:08:29 -0700 Subject: [PATCH 32/61] Add missing changelog backlinks. (#814) Co-authored-by: Mila Page --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04a8e7db1..218a3de96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ## Previous Releases For information on prior major and minor releases, see their changelogs: +- [1.8](https://github.com/dbt-labs/dbt-redshift/blob/1.8.latest/CHANGELOG.md) +- [1.7](https://github.com/dbt-labs/dbt-redshift/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md) From 2d653c6262e23b75de8d35c6ef15561e72c37b59 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 7 May 2024 19:26:47 -0400 Subject: [PATCH 33/61] Add integration tests for IAM User auth (#774) * move connection fixtures into the functional scope * add iam user creds to the test.env template * add test for database connection method * add iam user auth test * maintain existing behavior when not providing profile * add AWS IAM profile in CI * pull in new env vars in CI * updates to make space for iam role --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Features-20240419-145208.yaml | 7 + dbt/adapters/redshift/connections.py | 155 ++-- test.env.example | 23 +- tests/conftest.py | 21 - tests/functional/conftest.py | 19 + tests/functional/test_auth_method.py | 87 +++ tests/unit/mock_adapter.py | 2 +- tests/unit/test_auth_method.py | 395 ++++++++++ tests/unit/test_connection.py | 130 ++++ tests/unit/test_conversion.py | 80 ++ .../test_materialized_view.py | 0 tests/unit/test_query.py | 116 +++ tests/unit/test_redshift_adapter.py | 698 ------------------ ..._renamed_relations.py => test_relation.py} | 0 tests/unit/test_ssl_mode.py | 168 +++++ tests/unit/utils.py | 7 +- 16 files changed, 1114 insertions(+), 794 deletions(-) create mode 100644 .changes/unreleased/Features-20240419-145208.yaml create mode 100644 tests/functional/conftest.py create mode 100644 tests/functional/test_auth_method.py create mode 100644 tests/unit/test_auth_method.py create mode 100644 tests/unit/test_connection.py create mode 100644 tests/unit/test_conversion.py rename tests/unit/{relation_configs => }/test_materialized_view.py (100%) create mode 100644 tests/unit/test_query.py delete mode 100644 tests/unit/test_redshift_adapter.py rename tests/unit/{test_renamed_relations.py => test_relation.py} (100%) create mode 100644 tests/unit/test_ssl_mode.py diff --git a/.changes/unreleased/Features-20240419-145208.yaml b/.changes/unreleased/Features-20240419-145208.yaml new file mode 100644 index 000000000..3a7a3468b --- /dev/null +++ b/.changes/unreleased/Features-20240419-145208.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Support IAM user auth via direct parameters, in addition to the existing profile + method +time: 2024-04-19T14:52:08.086607-04:00 +custom: + Author: mikealfare + Issue: "760" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index b890127c6..3b54717f3 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -1,7 +1,7 @@ import re from multiprocessing import Lock from contextlib import contextmanager -from typing import Tuple, Union, Optional, List +from typing import Any, Callable, Dict, Tuple, Union, Optional, List from dataclasses import dataclass, field import agate @@ -116,11 +116,13 @@ class RedshiftCredentials(Credentials): ra3_node: Optional[bool] = False connect_timeout: Optional[int] = None role: Optional[str] = None - sslmode: Optional[UserSSLMode] = field(default_factory=UserSSLMode.default) + sslmode: UserSSLMode = field(default_factory=UserSSLMode.default) retries: int = 1 region: Optional[str] = None # opt-in by default per team deliberation on https://peps.python.org/pep-0249/#autocommit autocommit: Optional[bool] = True + access_key_id: Optional[str] = None + secret_access_key: Optional[str] = None _ALIASES = {"dbname": "database", "pass": "password"} @@ -142,7 +144,6 @@ def _connection_keys(self): "region", "sslmode", "region", - "iam_profile", "autocreate", "db_groups", "ra3_node", @@ -150,6 +151,7 @@ def _connection_keys(self): "role", "retries", "autocommit", + "access_key_id", ) @property @@ -160,74 +162,107 @@ def unique_field(self) -> str: class RedshiftConnectMethodFactory: credentials: RedshiftCredentials - def __init__(self, credentials): + def __init__(self, credentials) -> None: self.credentials = credentials - def get_connect_method(self): - method = self.credentials.method + def get_connect_method(self) -> Callable[[], redshift_connector.Connection]: + + # Support missing 'method' for backwards compatibility + method = self.credentials.method or RedshiftConnectionMethod.DATABASE + if method == RedshiftConnectionMethod.DATABASE: + kwargs = self._database_kwargs + elif method == RedshiftConnectionMethod.IAM: + kwargs = self._iam_user_kwargs + else: + raise FailedToConnectError(f"Invalid 'method' in profile: '{method}'") + + def connect() -> redshift_connector.Connection: + c = redshift_connector.connect(**kwargs) + if self.credentials.autocommit: + c.autocommit = True + if self.credentials.role: + c.cursor().execute(f"set role {self.credentials.role}") + return c + + return connect + + @property + def _database_kwargs(self) -> Dict[str, Any]: + logger.debug("Connecting to redshift with 'database' credentials method") + kwargs = self._base_kwargs + + if self.credentials.user and self.credentials.password: + kwargs.update( + user=self.credentials.user, + password=self.credentials.password, + ) + else: + raise FailedToConnectError( + "'user' and 'password' fields are required for 'database' credentials method" + ) + + return kwargs + + @property + def _iam_user_kwargs(self) -> Dict[str, Any]: + logger.debug("Connecting to redshift with 'iam' credentials method") + kwargs = self._iam_kwargs + + if self.credentials.access_key_id and self.credentials.secret_access_key: + kwargs.update( + access_key_id=self.credentials.access_key_id, + secret_access_key=self.credentials.secret_access_key, + ) + elif self.credentials.access_key_id or self.credentials.secret_access_key: + raise FailedToConnectError( + "'access_key_id' and 'secret_access_key' are both needed if providing explicit credentials" + ) + else: + kwargs.update(profile=self.credentials.iam_profile) + + if user := self.credentials.user: + kwargs.update(db_user=user) + else: + raise FailedToConnectError("'user' field is required for 'iam' credentials method") + + return kwargs + + @property + def _iam_kwargs(self) -> Dict[str, Any]: + kwargs = self._base_kwargs + kwargs.update( + iam=True, + user="", + password="", + ) + + if cluster_id := self.credentials.cluster_id: + kwargs.update(cluster_identifier=cluster_id) + elif "serverless" in self.credentials.host: + kwargs.update(cluster_identifier=None) + else: + raise FailedToConnectError( + "Failed to use IAM method:" + " 'cluster_id' must be provided for provisioned cluster" + " 'host' must be provided for serverless endpoint" + ) + + return kwargs + + @property + def _base_kwargs(self) -> Dict[str, Any]: kwargs = { "host": self.credentials.host, - "database": self.credentials.database, "port": int(self.credentials.port) if self.credentials.port else int(5439), + "database": self.credentials.database, + "region": self.credentials.region, "auto_create": self.credentials.autocreate, "db_groups": self.credentials.db_groups, - "region": self.credentials.region, "timeout": self.credentials.connect_timeout, } - redshift_ssl_config = RedshiftSSLConfig.parse(self.credentials.sslmode) kwargs.update(redshift_ssl_config.to_dict()) - - # Support missing 'method' for backwards compatibility - if method == RedshiftConnectionMethod.DATABASE or method is None: - # this requirement is really annoying to encode into json schema, - # so validate it here - if self.credentials.password is None: - raise FailedToConnectError( - "'password' field is required for 'database' credentials" - ) - - def connect(): - logger.debug("Connecting to redshift with username/password based auth...") - c = redshift_connector.connect( - user=self.credentials.user, - password=self.credentials.password, - **kwargs, - ) - if self.credentials.autocommit: - c.autocommit = True - if self.credentials.role: - c.cursor().execute("set role {}".format(self.credentials.role)) - return c - - elif method == RedshiftConnectionMethod.IAM: - if not self.credentials.cluster_id and "serverless" not in self.credentials.host: - raise FailedToConnectError( - "Failed to use IAM method. 'cluster_id' must be provided for provisioned cluster. " - "'host' must be provided for serverless endpoint." - ) - - def connect(): - logger.debug("Connecting to redshift with IAM based auth...") - c = redshift_connector.connect( - iam=True, - db_user=self.credentials.user, - password="", - user="", - cluster_identifier=self.credentials.cluster_id, - profile=self.credentials.iam_profile, - **kwargs, - ) - if self.credentials.autocommit: - c.autocommit = True - if self.credentials.role: - c.cursor().execute("set role {}".format(self.credentials.role)) - return c - - else: - raise FailedToConnectError("Invalid 'method' in profile: '{}'".format(method)) - - return connect + return kwargs class RedshiftConnectionManager(SQLConnectionManager): diff --git a/test.env.example b/test.env.example index 4de05edab..83c682036 100644 --- a/test.env.example +++ b/test.env.example @@ -1,18 +1,21 @@ # Note: Make sure you have a Redshift account that is set up so these fields are easy to complete. - -### Test Environment field definitions # These will all be gathered from account information or created by you. -# Endpoint for Redshift connection + +# Database Authentication Method REDSHIFT_TEST_HOST= -# Username on your account -REDSHIFT_TEST_USER= -# Password for Redshift account -REDSHIFT_TEST_PASS= -# Local port to connect on REDSHIFT_TEST_PORT= -# Name of Redshift database in your account to test against REDSHIFT_TEST_DBNAME= -# Users for testing +REDSHIFT_TEST_USER= +REDSHIFT_TEST_PASS= +REDSHIFT_TEST_REGION= + +# IAM User Authentication Method +REDSHIFT_TEST_CLUSTER_ID= +REDSHIFT_TEST_IAM_USER_PROFILE= +REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID= +REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY= + +# Database users for testing DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 diff --git a/tests/conftest.py b/tests/conftest.py index 96f0d43e4..652100af5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1 @@ -import pytest -import os - -# Import the functional fixtures as a plugin -# Note: fixtures with session scope need to be local - pytest_plugins = ["dbt.tests.fixtures.project"] - - -# The profile dictionary, used to write out profiles.yml -@pytest.fixture(scope="class") -def dbt_profile_target(): - return { - "type": "redshift", - "threads": 1, - "retries": 6, - "host": os.getenv("REDSHIFT_TEST_HOST"), - "port": int(os.getenv("REDSHIFT_TEST_PORT")), - "user": os.getenv("REDSHIFT_TEST_USER"), - "pass": os.getenv("REDSHIFT_TEST_PASS"), - "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), - } diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 000000000..73329936d --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,19 @@ +import os + +import pytest + + +# The profile dictionary, used to write out profiles.yml +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "redshift", + "host": os.getenv("REDSHIFT_TEST_HOST"), + "port": int(os.getenv("REDSHIFT_TEST_PORT")), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "pass": os.getenv("REDSHIFT_TEST_PASS"), + "region": os.getenv("REDSHIFT_TEST_REGION"), + "threads": 1, + "retries": 6, + } diff --git a/tests/functional/test_auth_method.py b/tests/functional/test_auth_method.py new file mode 100644 index 000000000..0eb33c0fa --- /dev/null +++ b/tests/functional/test_auth_method.py @@ -0,0 +1,87 @@ +import os + +import pytest + +from dbt.adapters.redshift.connections import RedshiftConnectionMethod +from dbt.tests.util import run_dbt + + +MY_SEED = """ +id,name +1,apple +2,banana +3,cherry +""".strip() + + +MY_VIEW = """ +select * from {{ ref("my_seed") }} +""" + + +class AuthMethod: + + @pytest.fixture(scope="class") + def seeds(self): + yield {"my_seed.csv": MY_SEED} + + @pytest.fixture(scope="class") + def models(self): + yield {"my_view.sql": MY_VIEW} + + def test_connection(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 1 + + +class TestDatabaseMethod(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.DATABASE.value, + "host": os.getenv("REDSHIFT_TEST_HOST"), + "port": int(os.getenv("REDSHIFT_TEST_PORT")), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "pass": os.getenv("REDSHIFT_TEST_PASS"), + "threads": 1, + "retries": 6, + } + + +class TestIAMUserMethodProfile(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "iam_profile": os.getenv("REDSHIFT_TEST_IAM_USER_PROFILE"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } + + +class TestIAMUserMethodExplicit(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "access_key_id": os.getenv("REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID"), + "secret_access_key": os.getenv("REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY"), + "region": os.getenv("REDSHIFT_TEST_REGION"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index 8547480d1..6e4143b9c 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -1,5 +1,5 @@ -from unittest import mock from contextlib import contextmanager +from unittest import mock from dbt.adapters.base import BaseAdapter diff --git a/tests/unit/test_auth_method.py b/tests/unit/test_auth_method.py new file mode 100644 index 000000000..5b39db354 --- /dev/null +++ b/tests/unit/test_auth_method.py @@ -0,0 +1,395 @@ +from multiprocessing import get_context +from unittest import TestCase, mock +from unittest.mock import MagicMock + +from dbt.adapters.exceptions import FailedToConnectError +import redshift_connector + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from dbt.adapters.redshift.connections import RedshiftConnectMethodFactory, RedshiftSSLConfig +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter + + +DEFAULT_SSL_CONFIG = RedshiftSSLConfig().to_dict() + + +class AuthMethod(TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + +class TestInvalidMethod(AuthMethod): + def test_invalid_auth_method(self): + # we have to set method this way, otherwise it won't validate + self.config.credentials.method = "badmethod" + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() + self.assertTrue("badmethod" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_missing_region_failure(self): + # Failure test with no region + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233_no_region", + region=None, + ) + + with self.assertRaises(FailedToConnectError): + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233_no_region", + database="redshift", + cluster_identifier=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_invalid_region_failure(self): + # Invalid region test + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233_no_region.us-not-a-region-1", + region=None, + ) + + with self.assertRaises(FailedToConnectError): + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233_no_region", + database="redshift", + cluster_identifier=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + +class TestDatabaseMethod(AuthMethod): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + timeout=None, + region=None, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_explicit_auth_method(self): + self.config.method = "database" + + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + **DEFAULT_SSL_CONFIG, + ) + + def test_database_verification_is_case_insensitive(self): + # Override adapter settings from setUp() + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "Redshift", + "user": "root", + "host": "thishostshouldnotexist", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self.adapter.cleanup_connections() + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + self.adapter.verify_database("redshift") + + +class TestIAMUserMethod(AuthMethod): + + def test_iam_optionals(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist", + "port": 5439, + "schema": "public", + "method": "iam", + "cluster_id": "my_redshift", + "db_groups": ["my_dbgroup"], + "autocreate": True, + } + }, + "target": "test", + } + + config_from_parts_or_dicts(self.config, profile_cfg) + + def test_no_cluster_id(self): + self.config.credentials = self.config.credentials.replace(method="iam") + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() + + self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + cluster_id="my_redshift", + host="thishostshouldnotexist.test.us-east-1", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + db_user="root", + password="", + user="", + cluster_identifier="my_redshift", + region=None, + timeout=None, + auto_create=False, + db_groups=[], + profile=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + cluster_id="my_redshift", + iam_profile="test", + host="thishostshouldnotexist.test.us-east-1", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + + redshift_connector.connect.assert_called_once_with( + iam=True, + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + cluster_identifier="my_redshift", + region=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_explicit(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + cluster_id="my_redshift", + host="thishostshouldnotexist.test.us-east-1", + access_key_id="my_access_key_id", + secret_access_key="my_secret_access_key", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="thishostshouldnotexist.test.us-east-1", + access_key_id="my_access_key_id", + secret_access_key="my_secret_access_key", + database="redshift", + db_user="root", + password="", + user="", + cluster_identifier="my_redshift", + region=None, + timeout=None, + auto_create=False, + db_groups=[], + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + +class TestIAMUserMethodServerless(AuthMethod): + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_default_region(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + database="redshift", + cluster_identifier=None, + region=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_explicit_region(self): + # Successful test + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233.redshift-serverless.amazonaws.com", + region="us-east-2", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.redshift-serverless.amazonaws.com", + database="redshift", + cluster_identifier=None, + region="us-east-2", + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_invalid_serverless(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", + ) + with self.assertRaises(FailedToConnectError) as context: + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", + database="redshift", + cluster_identifier=None, + region=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + port=5439, + timeout=None, + **DEFAULT_SSL_CONFIG, + ) + self.assertTrue("'host' must be provided" in context.exception.msg) diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py new file mode 100644 index 000000000..17a676286 --- /dev/null +++ b/tests/unit/test_connection.py @@ -0,0 +1,130 @@ +from multiprocessing import get_context +from unittest import TestCase, mock +from unittest.mock import MagicMock, call + +import redshift_connector + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from tests.unit.utils import ( + config_from_parts_or_dicts, + inject_adapter, + mock_connection, +) + + +class TestConnection(TestCase): + + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + def test_cancel_open_connections_empty(self): + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_master(self): + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections[key] = mock_connection("master") + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_single(self): + master = mock_connection("master") + model = mock_connection("model") + + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections.update( + { + key: master, + 1: model, + } + ) + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + query_result = mock.MagicMock() + cursor = mock.Mock() + cursor.fetchone.return_value = (42,) + add_query.side_effect = [(None, cursor), (None, query_result)] + + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({model.backend_pid})"), + ] + ) + + master.handle.backend_pid.assert_not_called() + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_has_backend_pid(self): + backend_pid = 42 + + cursor = mock.MagicMock() + execute = cursor().__enter__().execute + execute().fetchone.return_value = (backend_pid,) + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + assert connection.backend_pid == backend_pid + + execute.assert_has_calls( + [ + call("select pg_backend_pid()"), + ] + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_backend_pid_used_in_pg_terminate_backend(self): + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + backend_pid = 42 + query_result = (backend_pid,) + + cursor = mock.MagicMock() + cursor().__enter__().execute().fetchone.return_value = query_result + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + + self.adapter.connections.cancel(connection) + + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({backend_pid})"), + ] + ) diff --git a/tests/unit/test_conversion.py b/tests/unit/test_conversion.py new file mode 100644 index 000000000..a375c4f6a --- /dev/null +++ b/tests/unit/test_conversion.py @@ -0,0 +1,80 @@ +import agate +from dbt_common.clients import agate_helper + +from dbt.adapters.redshift import RedshiftAdapter +from tests.unit.utils import TestAdapterConversions + + +class TestConversion(TestAdapterConversions): + def test_convert_text_type(self): + rows = [ + ["", "a1", "stringval1"], + ["", "a2", "stringvalasdfasdfasdfa"], + ["", "a3", "stringval3"], + ] + agate_table = self._make_table_of(rows, agate.Text) + expected = ["varchar(64)", "varchar(2)", "varchar(22)"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_text_type(agate_table, col_idx) == expect + + def test_convert_number_type(self): + rows = [ + ["", "23.98", "-1"], + ["", "12.78", "-2"], + ["", "79.41", "-3"], + ] + agate_table = self._make_table_of(rows, agate.Number) + expected = ["integer", "float8", "integer"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_number_type(agate_table, col_idx) == expect + + def test_convert_boolean_type(self): + rows = [ + ["", "false", "true"], + ["", "false", "false"], + ["", "false", "true"], + ] + agate_table = self._make_table_of(rows, agate.Boolean) + expected = ["boolean", "boolean", "boolean"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_boolean_type(agate_table, col_idx) == expect + + def test_convert_datetime_type(self): + rows = [ + ["", "20190101T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190102T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190103T01:01:01Z", "2019-01-01 01:01:01"], + ] + agate_table = self._make_table_of( + rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime] + ) + expected = [ + "timestamp without time zone", + "timestamp without time zone", + "timestamp without time zone", + ] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_datetime_type(agate_table, col_idx) == expect + + def test_convert_date_type(self): + rows = [ + ["", "2019-01-01", "2019-01-04"], + ["", "2019-01-02", "2019-01-04"], + ["", "2019-01-03", "2019-01-04"], + ] + agate_table = self._make_table_of(rows, agate.Date) + expected = ["date", "date", "date"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_date_type(agate_table, col_idx) == expect + + def test_convert_time_type(self): + # dbt's default type testers actually don't have a TimeDelta at all. + rows = [ + ["", "120s", "10s"], + ["", "3m", "11s"], + ["", "1h", "12s"], + ] + agate_table = self._make_table_of(rows, agate.TimeDelta) + expected = ["varchar(24)", "varchar(24)", "varchar(24)"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_time_type(agate_table, col_idx) == expect diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/test_materialized_view.py similarity index 100% rename from tests/unit/relation_configs/test_materialized_view.py rename to tests/unit/test_materialized_view.py diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py new file mode 100644 index 000000000..ff8076215 --- /dev/null +++ b/tests/unit/test_query.py @@ -0,0 +1,116 @@ +from multiprocessing import get_context +from unittest import TestCase, mock + +from dbt.adapters.sql.connections import SQLConnectionManager +from dbt_common.clients import agate_helper +from dbt_common.exceptions import DbtRuntimeError + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter + + +class TestQuery(TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + @mock.patch.object(SQLConnectionManager, "get_thread_connection") + def mock_cursor(self, mock_get_thread_conn): + conn = mock.MagicMock + mock_get_thread_conn.return_value = conn + mock_handle = mock.MagicMock + conn.return_value = mock_handle + mock_cursor = mock.MagicMock + mock_handle.return_value = mock_cursor + return mock_cursor + + def test_execute_with_fetch(self): + cursor = mock.Mock() + table = agate_helper.empty_table() + with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: + mock_add_query.return_value = ( + None, + cursor, + ) # when mock_add_query is called, it will always return None, cursor + with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: + mock_get_response.return_value = None + with mock.patch.object( + self.adapter.connections, "get_result_from_cursor" + ) as mock_get_result_from_cursor: + mock_get_result_from_cursor.return_value = table + self.adapter.connections.execute(sql="select * from test", fetch=True) + mock_add_query.assert_called_once_with("select * from test", False) + mock_get_result_from_cursor.assert_called_once_with(cursor, None) + mock_get_response.assert_called_once_with(cursor) + + def test_execute_without_fetch(self): + cursor = mock.Mock() + with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: + mock_add_query.return_value = ( + None, + cursor, + ) # when mock_add_query is called, it will always return None, cursor + with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: + mock_get_response.return_value = None + with mock.patch.object( + self.adapter.connections, "get_result_from_cursor" + ) as mock_get_result_from_cursor: + self.adapter.connections.execute(sql="select * from test2", fetch=False) + mock_add_query.assert_called_once_with("select * from test2", False) + mock_get_result_from_cursor.assert_not_called() + mock_get_response.assert_called_once_with(cursor) + + def test_add_query_success(self): + cursor = mock.Mock() + with mock.patch.object(SQLConnectionManager, "add_query") as mock_add_query: + mock_add_query.return_value = None, cursor + self.adapter.connections.add_query("select * from test3") + mock_add_query.assert_called_once_with( + "select * from test3", True, bindings=None, abridge_sql_log=False + ) + + def test_add_query_with_no_cursor(self): + with mock.patch.object( + self.adapter.connections, "get_thread_connection" + ) as mock_get_thread_connection: + mock_get_thread_connection.return_value = None + with self.assertRaisesRegex(DbtRuntimeError, "Tried to run invalid SQL: on "): + self.adapter.connections.add_query(sql="") + mock_get_thread_connection.assert_called_once() diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py deleted file mode 100644 index 0bd5f8e99..000000000 --- a/tests/unit/test_redshift_adapter.py +++ /dev/null @@ -1,698 +0,0 @@ -import unittest - -from multiprocessing import get_context -from unittest import mock - -from dbt_common.exceptions import DbtRuntimeError -from unittest.mock import MagicMock, call - -import agate -import dbt -import redshift_connector - -from dbt.adapters.redshift import ( - RedshiftAdapter, - Plugin as RedshiftPlugin, -) -from dbt_common.clients import agate_helper -from dbt.adapters.exceptions import FailedToConnectError -from dbt.adapters.redshift.connections import RedshiftConnectMethodFactory, RedshiftSSLConfig -from .utils import ( - config_from_parts_or_dicts, - mock_connection, - TestAdapterConversions, - inject_adapter, -) - - -DEFAULT_SSL_CONFIG = RedshiftSSLConfig().to_dict() - - -class TestRedshiftAdapter(unittest.TestCase): - def setUp(self): - profile_cfg = { - "outputs": { - "test": { - "type": "redshift", - "dbname": "redshift", - "user": "root", - "host": "thishostshouldnotexist.test.us-east-1", - "pass": "password", - "port": 5439, - "schema": "public", - } - }, - "target": "test", - } - - project_cfg = { - "name": "X", - "version": "0.1", - "profile": "test", - "project-root": "/tmp/dbt/does-not-exist", - "quoting": { - "identifier": False, - "schema": True, - }, - "config-version": 2, - } - - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self._adapter = None - - @property - def adapter(self): - if self._adapter is None: - self._adapter = RedshiftAdapter(self.config, get_context("spawn")) - inject_adapter(self._adapter, RedshiftPlugin) - return self._adapter - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_implicit_database_conn(self): - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - timeout=None, - region=None, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_region_with_database_conn(self): - self.config.method = "database" - - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_iam_conn_without_profile(self): - self.config.credentials = self.config.credentials.replace( - method="iam", - cluster_id="my_redshift", - host="thishostshouldnotexist.test.us-east-1", - ) - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - db_user="root", - password="", - user="", - cluster_identifier="my_redshift", - region=None, - timeout=None, - auto_create=False, - db_groups=[], - profile=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_conn_timeout_30(self): - self.config.credentials = self.config.credentials.replace(connect_timeout=30) - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=30, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_iam_conn_with_profile(self): - self.config.credentials = self.config.credentials.replace( - method="iam", - cluster_id="my_redshift", - iam_profile="test", - host="thishostshouldnotexist.test.us-east-1", - ) - connection = self.adapter.acquire_connection("dummy") - connection.handle - - redshift_connector.connect.assert_called_once_with( - iam=True, - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - cluster_identifier="my_redshift", - region=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_iam_serverless_with_profile(self): - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", - ) - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", - database="redshift", - cluster_identifier=None, - region=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_region(self): - # Successful test - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233.redshift-serverless.amazonaws.com", - region="us-east-2", - ) - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233.redshift-serverless.amazonaws.com", - database="redshift", - cluster_identifier=None, - region="us-east-2", - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_region_failure(self): - # Failure test with no region - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233_no_region", - region=None, - ) - - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233_no_region", - database="redshift", - cluster_identifier=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_explicit_invalid_region(self): - # Invalid region test - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233_no_region.us-not-a-region-1", - region=None, - ) - - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233_no_region", - database="redshift", - cluster_identifier=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_sslmode_disable(self): - self.config.credentials.sslmode = "disable" - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - ssl=False, - sslmode=None, - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_sslmode_allow(self): - self.config.credentials.sslmode = "allow" - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - ssl=True, - sslmode="verify-ca", - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_sslmode_verify_full(self): - self.config.credentials.sslmode = "verify-full" - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - ssl=True, - sslmode="verify-full", - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_sslmode_verify_ca(self): - self.config.credentials.sslmode = "verify-ca" - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - ssl=True, - sslmode="verify-ca", - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_sslmode_prefer(self): - self.config.credentials.sslmode = "prefer" - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", - database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], - region=None, - timeout=None, - ssl=True, - sslmode="verify-ca", - ) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_serverless_iam_failure(self): - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", - ) - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError) as context: - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", - database="redshift", - cluster_identifier=None, - region=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - port=5439, - timeout=None, - **DEFAULT_SSL_CONFIG, - ) - self.assertTrue("'host' must be provided" in context.exception.msg) - - def test_iam_conn_optionals(self): - profile_cfg = { - "outputs": { - "test": { - "type": "redshift", - "dbname": "redshift", - "user": "root", - "host": "thishostshouldnotexist", - "port": 5439, - "schema": "public", - "method": "iam", - "cluster_id": "my_redshift", - "db_groups": ["my_dbgroup"], - "autocreate": True, - } - }, - "target": "test", - } - - config_from_parts_or_dicts(self.config, profile_cfg) - - def test_invalid_auth_method(self): - # we have to set method this way, otherwise it won't validate - self.config.credentials.method = "badmethod" - with self.assertRaises(FailedToConnectError) as context: - connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) - connect_method_factory.get_connect_method() - self.assertTrue("badmethod" in context.exception.msg) - - def test_invalid_iam_no_cluster_id(self): - self.config.credentials = self.config.credentials.replace(method="iam") - with self.assertRaises(FailedToConnectError) as context: - connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) - connect_method_factory.get_connect_method() - - self.assertTrue("'cluster_id' must be provided" in context.exception.msg) - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_connection_has_backend_pid(self): - backend_pid = 42 - - cursor = mock.MagicMock() - execute = cursor().__enter__().execute - execute().fetchone.return_value = (backend_pid,) - redshift_connector.connect().cursor = cursor - - connection = self.adapter.acquire_connection("dummy") - connection.handle - assert connection.backend_pid == backend_pid - - execute.assert_has_calls( - [ - call("select pg_backend_pid()"), - ] - ) - - def test_cancel_open_connections_empty(self): - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_master(self): - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections[key] = mock_connection("master") - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_single(self): - master = mock_connection("master") - model = mock_connection("model") - - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections.update( - { - key: master, - 1: model, - } - ) - with mock.patch.object(self.adapter.connections, "add_query") as add_query: - query_result = mock.MagicMock() - cursor = mock.Mock() - cursor.fetchone.return_value = (42,) - add_query.side_effect = [(None, cursor), (None, query_result)] - - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) - add_query.assert_has_calls( - [ - call(f"select pg_terminate_backend({model.backend_pid})"), - ] - ) - - master.handle.backend_pid.assert_not_called() - - @mock.patch("redshift_connector.connect", MagicMock()) - def test_backend_pid_used_in_pg_terminate_backend(self): - with mock.patch.object(self.adapter.connections, "add_query") as add_query: - backend_pid = 42 - query_result = (backend_pid,) - - cursor = mock.MagicMock() - cursor().__enter__().execute().fetchone.return_value = query_result - redshift_connector.connect().cursor = cursor - - connection = self.adapter.acquire_connection("dummy") - connection.handle - - self.adapter.connections.cancel(connection) - - add_query.assert_has_calls( - [ - call(f"select pg_terminate_backend({backend_pid})"), - ] - ) - - def test_dbname_verification_is_case_insensitive(self): - # Override adapter settings from setUp() - profile_cfg = { - "outputs": { - "test": { - "type": "redshift", - "dbname": "Redshift", - "user": "root", - "host": "thishostshouldnotexist", - "pass": "password", - "port": 5439, - "schema": "public", - } - }, - "target": "test", - } - - project_cfg = { - "name": "X", - "version": "0.1", - "profile": "test", - "project-root": "/tmp/dbt/does-not-exist", - "quoting": { - "identifier": False, - "schema": True, - }, - "config-version": 2, - } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self.adapter.cleanup_connections() - self._adapter = RedshiftAdapter(self.config, get_context("spawn")) - self.adapter.verify_database("redshift") - - def test_execute_with_fetch(self): - cursor = mock.Mock() - table = agate_helper.empty_table() - with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: - mock_add_query.return_value = ( - None, - cursor, - ) # when mock_add_query is called, it will always return None, cursor - with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: - mock_get_response.return_value = None - with mock.patch.object( - self.adapter.connections, "get_result_from_cursor" - ) as mock_get_result_from_cursor: - mock_get_result_from_cursor.return_value = table - self.adapter.connections.execute(sql="select * from test", fetch=True) - mock_add_query.assert_called_once_with("select * from test", False) - mock_get_result_from_cursor.assert_called_once_with(cursor, None) - mock_get_response.assert_called_once_with(cursor) - - def test_execute_without_fetch(self): - cursor = mock.Mock() - with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: - mock_add_query.return_value = ( - None, - cursor, - ) # when mock_add_query is called, it will always return None, cursor - with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: - mock_get_response.return_value = None - with mock.patch.object( - self.adapter.connections, "get_result_from_cursor" - ) as mock_get_result_from_cursor: - self.adapter.connections.execute(sql="select * from test2", fetch=False) - mock_add_query.assert_called_once_with("select * from test2", False) - mock_get_result_from_cursor.assert_not_called() - mock_get_response.assert_called_once_with(cursor) - - def test_add_query_with_no_cursor(self): - with mock.patch.object( - self.adapter.connections, "get_thread_connection" - ) as mock_get_thread_connection: - mock_get_thread_connection.return_value = None - with self.assertRaisesRegex(DbtRuntimeError, "Tried to run invalid SQL: on "): - self.adapter.connections.add_query(sql="") - mock_get_thread_connection.assert_called_once() - - def test_add_query_success(self): - cursor = mock.Mock() - with mock.patch.object( - dbt.adapters.redshift.connections.SQLConnectionManager, "add_query" - ) as mock_add_query: - mock_add_query.return_value = None, cursor - self.adapter.connections.add_query("select * from test3") - mock_add_query.assert_called_once_with( - "select * from test3", True, bindings=None, abridge_sql_log=False - ) - - @mock.patch.object( - dbt.adapters.redshift.connections.SQLConnectionManager, "get_thread_connection" - ) - def mock_cursor(self, mock_get_thread_conn): - conn = mock.MagicMock - mock_get_thread_conn.return_value = conn - mock_handle = mock.MagicMock - conn.return_value = mock_handle - mock_cursor = mock.MagicMock - mock_handle.return_value = mock_cursor - return mock_cursor - - -class TestRedshiftAdapterConversions(TestAdapterConversions): - def test_convert_text_type(self): - rows = [ - ["", "a1", "stringval1"], - ["", "a2", "stringvalasdfasdfasdfa"], - ["", "a3", "stringval3"], - ] - agate_table = self._make_table_of(rows, agate.Text) - expected = ["varchar(64)", "varchar(2)", "varchar(22)"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_text_type(agate_table, col_idx) == expect - - def test_convert_number_type(self): - rows = [ - ["", "23.98", "-1"], - ["", "12.78", "-2"], - ["", "79.41", "-3"], - ] - agate_table = self._make_table_of(rows, agate.Number) - expected = ["integer", "float8", "integer"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_number_type(agate_table, col_idx) == expect - - def test_convert_boolean_type(self): - rows = [ - ["", "false", "true"], - ["", "false", "false"], - ["", "false", "true"], - ] - agate_table = self._make_table_of(rows, agate.Boolean) - expected = ["boolean", "boolean", "boolean"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_boolean_type(agate_table, col_idx) == expect - - def test_convert_datetime_type(self): - rows = [ - ["", "20190101T01:01:01Z", "2019-01-01 01:01:01"], - ["", "20190102T01:01:01Z", "2019-01-01 01:01:01"], - ["", "20190103T01:01:01Z", "2019-01-01 01:01:01"], - ] - agate_table = self._make_table_of( - rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime] - ) - expected = [ - "timestamp without time zone", - "timestamp without time zone", - "timestamp without time zone", - ] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_datetime_type(agate_table, col_idx) == expect - - def test_convert_date_type(self): - rows = [ - ["", "2019-01-01", "2019-01-04"], - ["", "2019-01-02", "2019-01-04"], - ["", "2019-01-03", "2019-01-04"], - ] - agate_table = self._make_table_of(rows, agate.Date) - expected = ["date", "date", "date"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_date_type(agate_table, col_idx) == expect - - def test_convert_time_type(self): - # dbt's default type testers actually don't have a TimeDelta at all. - rows = [ - ["", "120s", "10s"], - ["", "3m", "11s"], - ["", "1h", "12s"], - ] - agate_table = self._make_table_of(rows, agate.TimeDelta) - expected = ["varchar(24)", "varchar(24)", "varchar(24)"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_time_type(agate_table, col_idx) == expect diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_relation.py similarity index 100% rename from tests/unit/test_renamed_relations.py rename to tests/unit/test_relation.py diff --git a/tests/unit/test_ssl_mode.py b/tests/unit/test_ssl_mode.py new file mode 100644 index 000000000..832e0718b --- /dev/null +++ b/tests/unit/test_ssl_mode.py @@ -0,0 +1,168 @@ +from multiprocessing import get_context +from unittest import TestCase, mock +from unittest.mock import MagicMock + +import redshift_connector + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from dbt.adapters.redshift.connections import RedshiftSSLConfig +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter + + +DEFAULT_SSL_CONFIG = RedshiftSSLConfig().to_dict() + + +class TestSSLMode(TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_disable(self): + self.config.credentials.sslmode = "disable" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=False, + sslmode=None, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_allow(self): + self.config.credentials.sslmode = "allow" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_verify_full(self): + self.config.credentials.sslmode = "verify-full" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-full", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_verify_ca(self): + self.config.credentials.sslmode = "verify-ca" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_prefer(self): + self.config.credentials.sslmode = "prefer" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_timeout(self): + self.config.credentials = self.config.credentials.replace(connect_timeout=30) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=30, + **DEFAULT_SSL_CONFIG, + ) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 4de03ec80..ee580eb9a 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -5,13 +5,12 @@ import string import os -from unittest import mock -from unittest import TestCase +from unittest import TestCase, mock import agate -import pytest -from dbt_common.dataclass_schema import ValidationError from dbt.config.project import PartialProject +from dbt_common.dataclass_schema import ValidationError +import pytest def normalize(path): From 7e94b0aa8a145f50176e956b29a8916251b0061b Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 7 May 2024 20:46:40 -0400 Subject: [PATCH 34/61] Add IAM Role Authentication (#781) * adding SSO support for redshift Committer: Abby Whittier * ADAP-891: Support test results as views (#614) * implement store-failures-as tests * Use the PID to terminate the session (#568) * The first element of the result is the PID * Debug-level logging of high-level message + SQL * Using redshift_connector `cursor.fetchone()` returns `(,)` * Use cursor to call `select pg_terminate_backend({pid})` directly rather than using the `SQLConnectionManager` --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> * added error checking for new optional user field * black formatting * move connection fixtures into the functional scope * add iam user creds to the test.env template * add test for database connection method * add iam user auth test * add IAM User auth test and second user auth method * changie * maintain existing behavior when not providing profile * add AWS IAM profile * pull in new env vars * fixed env vars refs for CI * move all repo vars to secrets * split out connect method by connection method and provided information * condition to produce just kwargs, consolidate connect method * update .format to f-strings * incorporate feedback from pr#630 * update kwargs logic flow * updates to make space for iam role * revert type on user * revert test case decorator * revert test case decorator * revert error message * add integration tests * make space for both iam user and iam role in testing * add role arn * naming * try supplying region for CI * add region to CI env * we can only support role credentials by profile * move iam user specific config out of iam and into iam user * add type annotations * move iam defaults out of iam user * add required params to test profiles * add required params to test profiles * simplify test files * add expected fields back in * split out unit test files * split out unit test files * add unit tests for iam role auth method * standardize names * allow for the default profile * add unit tests for iam role access * changie * changie --------- Co-authored-by: Abby Whittier Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Anders --- .../unreleased/Features-20240425-011440.yaml | 6 ++ dbt/adapters/redshift/connections.py | 19 +++++- dbt/include/redshift/profile_template.yml | 2 + test.env.example | 7 ++- tests/functional/test_auth_method.py | 16 +++++ tests/unit/test_auth_method.py | 63 +++++++++++++++++++ 6 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Features-20240425-011440.yaml diff --git a/.changes/unreleased/Features-20240425-011440.yaml b/.changes/unreleased/Features-20240425-011440.yaml new file mode 100644 index 000000000..a8197dd6f --- /dev/null +++ b/.changes/unreleased/Features-20240425-011440.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for IAM Role auth +time: 2024-04-25T01:14:40.601575-04:00 +custom: + Author: mikealfare,abbywh + Issue: "623" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 3b54717f3..752c81e32 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -37,6 +37,7 @@ def get_message(self) -> str: class RedshiftConnectionMethod(StrEnum): DATABASE = "database" IAM = "iam" + IAM_ROLE = "iam_role" class UserSSLMode(StrEnum): @@ -102,9 +103,9 @@ def parse(cls, user_sslmode: UserSSLMode) -> "RedshiftSSLConfig": @dataclass class RedshiftCredentials(Credentials): host: str - user: str port: Port method: str = RedshiftConnectionMethod.DATABASE # type: ignore + user: Optional[str] = None password: Optional[str] = None # type: ignore cluster_id: Optional[str] = field( default=None, @@ -173,6 +174,8 @@ def get_connect_method(self) -> Callable[[], redshift_connector.Connection]: kwargs = self._database_kwargs elif method == RedshiftConnectionMethod.IAM: kwargs = self._iam_user_kwargs + elif method == RedshiftConnectionMethod.IAM_ROLE: + kwargs = self._iam_role_kwargs else: raise FailedToConnectError(f"Invalid 'method' in profile: '{method}'") @@ -227,6 +230,20 @@ def _iam_user_kwargs(self) -> Dict[str, Any]: return kwargs + @property + def _iam_role_kwargs(self) -> Dict[str, Optional[Any]]: + logger.debug("Connecting to redshift with 'iam_role' credentials method") + kwargs = self._iam_kwargs + kwargs.update( + group_federation=True, + db_user=None, + ) + + if iam_profile := self.credentials.iam_profile: + kwargs.update(profile=iam_profile) + + return kwargs + @property def _iam_kwargs(self) -> Dict[str, Any]: kwargs = self._base_kwargs diff --git a/dbt/include/redshift/profile_template.yml b/dbt/include/redshift/profile_template.yml index 41f33e87e..d78356923 100644 --- a/dbt/include/redshift/profile_template.yml +++ b/dbt/include/redshift/profile_template.yml @@ -15,6 +15,8 @@ prompts: hide_input: true iam: _fixed_method: iam + iam_role: + _fixed_method: iam_role dbname: hint: 'default database that dbt will build objects in' schema: diff --git a/test.env.example b/test.env.example index 83c682036..6816b4ec2 100644 --- a/test.env.example +++ b/test.env.example @@ -9,12 +9,17 @@ REDSHIFT_TEST_USER= REDSHIFT_TEST_PASS= REDSHIFT_TEST_REGION= -# IAM User Authentication Method +# IAM Methods REDSHIFT_TEST_CLUSTER_ID= + +# IAM User Authentication Method REDSHIFT_TEST_IAM_USER_PROFILE= REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID= REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY= +# IAM Role Authentication Method +REDSHIFT_TEST_IAM_ROLE_PROFILE= + # Database users for testing DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 diff --git a/tests/functional/test_auth_method.py b/tests/functional/test_auth_method.py index 0eb33c0fa..b2273e02c 100644 --- a/tests/functional/test_auth_method.py +++ b/tests/functional/test_auth_method.py @@ -85,3 +85,19 @@ def dbt_profile_target(self): "host": "", # host is a required field in dbt-core "port": 0, # port is a required field in dbt-core } + + +class TestIAMRoleAuthProfile(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM_ROLE.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "iam_profile": os.getenv("REDSHIFT_TEST_IAM_ROLE_PROFILE"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } diff --git a/tests/unit/test_auth_method.py b/tests/unit/test_auth_method.py index 5b39db354..bd9912d0c 100644 --- a/tests/unit/test_auth_method.py +++ b/tests/unit/test_auth_method.py @@ -393,3 +393,66 @@ def test_profile_invalid_serverless(self): **DEFAULT_SSL_CONFIG, ) self.assertTrue("'host' must be provided" in context.exception.msg) + + +class TestIAMRoleMethod(AuthMethod): + + def test_no_cluster_id(self): + self.config.credentials = self.config.credentials.replace(method="iam_role") + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() + + self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + cluster_id="my_redshift", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + cluster_identifier="my_redshift", + db_user=None, + password="", + user="", + region=None, + timeout=None, + auto_create=False, + db_groups=[], + port=5439, + group_federation=True, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + cluster_id="my_redshift", + iam_profile="test", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + cluster_identifier="my_redshift", + db_user=None, + password="", + user="", + region=None, + timeout=None, + auto_create=False, + db_groups=[], + profile="test", + port=5439, + group_federation=True, + **DEFAULT_SSL_CONFIG, + ) From 80a484bb3f2faada34921adbf7a3fa8289bcf687 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 8 May 2024 10:37:10 -0400 Subject: [PATCH 35/61] Update CODEOWNERS (#815) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6283d123..02ed72d45 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # This codeowners file is used to ensure all PRs require reviews from the adapters team -* @dbt-labs/core-adapters +* @dbt-labs/adapters From b57890e3503928ced7dfd75de6985b348f0e7307 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 10 May 2024 11:36:00 -0400 Subject: [PATCH 36/61] Mark flaky tests (#813) * mark flaky tests using pytest.mark * split integration test runs by flaky and non-flaky * fix the posargs syntax for tox * force flaky tests to run in series, fix python version getting truncated * revert the integration test job name to match the existing names * separate flaky tests as a separate matrix of jobs * recombine the integration tests * allow integration tests to run fully in parallel for non-flaky tests * remove windows from integration test matrix since we weren't originally testing on windows anyway * add windows back in * register the custom marker with pytest * revert the combination of integration tests * configure a pytest return of 5 to be 0 * fix test order to avoid unnecessary change * mark more flaky tests * mark more flaky tests * pin windows images to 2019 to avoid datadog traceport failures * pinning windows images to 2019 didn't solve datadog issue, updating pin to the most recent version * turn off telemetry warnings (and telemetry) * mark more tests as flaky * mark more tests as flaky * incorporate feedback * incorporate feedback --- .github/scripts/integration-test-matrix.js | 87 --------- .github/workflows/integration.yml | 172 ++++++++++-------- .github/workflows/main.yml | 2 +- pytest.ini | 2 + tests/conftest.py | 11 ++ .../adapter/catalog_tests/test_get_catalog.py | 2 + .../catalog_tests/test_relation_types.py | 1 + .../test_materialized_views.py | 15 +- tests/functional/adapter/test_basic.py | 12 +- tests/functional/adapter/test_persist_docs.py | 13 +- tox.ini | 1 + 11 files changed, 151 insertions(+), 167 deletions(-) delete mode 100644 .github/scripts/integration-test-matrix.js diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index ccb6d949d..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,87 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; - const supportedAdapters = ["redshift"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-12", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-12"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ad29fef1e..0620392ba 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -53,85 +53,32 @@ defaults: shell: bash jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: + test: + name: redshift / python ${{ matrix.python-version }} / ${{ matrix.os }} + # run if not a PR from a forked repository or has a label to mark as safe to test if: >- github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) runs-on: ${{ matrix.os }} - needs: test-metadata - strategy: fail-fast: false max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + matrix: + os: [ubuntu-22.04, macos12, windows-2022] + python-version: ["3.8"] + include: + - os: ubuntu-22.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" env: - TOXENV: integration-${{ matrix.adapter }} + TOXENV: integration-redshift PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true @@ -202,7 +149,6 @@ jobs: AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - name: Run tox (redshift) - if: matrix.adapter == 'redshift' env: REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} @@ -218,7 +164,7 @@ jobs: DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace + run: tox -- -m "not flaky" --ddtrace - uses: actions/upload-artifact@v3 if: always() @@ -235,11 +181,93 @@ jobs: - uses: actions/upload-artifact@v3 if: always() with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv + name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv path: integration_results.csv + test-flaky: + name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky + + # run this after the norm integration tests to avoid collisions + needs: test + + # run if not a PR from a forked repository or has a label to mark as safe to test + if: >- + github.event_name != 'pull_request_target' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-22.04 + + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_INSTRUMENTATION_TELEMETRY_ENABLED: false + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + steps: + - name: Check out the repository + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + + # explicity checkout the branch for the PR, + # this is necessary for the `pull_request_target` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Update dev_requirements.txt + if: inputs.dbt-core-branch != '' + run: | + pip install bumpversion + ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + + - name: Run tox (redshift) + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m flaky --ddtrace + require-label-comment: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: test @@ -262,7 +290,7 @@ jobs: check_for_duplicate_msg: true post-failure: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: test if: ${{ failure() }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d8c126301..c49b57385 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-22.04, macos-12, windows-2022] python-version: ['3.8', '3.9', '3.10', '3.11'] steps: diff --git a/pytest.ini b/pytest.ini index b3d74bc14..8c290dc14 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,3 +7,5 @@ env_files = testpaths = tests/unit tests/functional +markers = + flaky: marks tests as flaky so they run one at a time (de-select with '-m "not flaky"') diff --git a/tests/conftest.py b/tests/conftest.py index 652100af5..712bf047a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,12 @@ pytest_plugins = ["dbt.tests.fixtures.project"] + + +def pytest_sessionfinish(session, exitstatus): + """ + Configures pytest to treat a scenario with no tests as passing + + pytest returns a code 5 when it collects no tests in an effort to warn when tests are expected but not collected + We don't want this when running tox because some combinations of markers and test segments return nothing + """ + if exitstatus == 5: + session.exitstatus = 0 diff --git a/tests/functional/adapter/catalog_tests/test_get_catalog.py b/tests/functional/adapter/catalog_tests/test_get_catalog.py index 519c806c4..e0b512896 100644 --- a/tests/functional/adapter/catalog_tests/test_get_catalog.py +++ b/tests/functional/adapter/catalog_tests/test_get_catalog.py @@ -97,6 +97,7 @@ def my_information_schema(self, adapter, my_schema): identifier="INFORMATION_SCHEMA", ).information_schema() + @pytest.mark.flaky def test_get_one_catalog_by_relations( self, adapter, @@ -120,6 +121,7 @@ def test_get_one_catalog_by_relations( # note the underlying table is missing as it's not in `my_relations` assert len(catalog) == 12 + @pytest.mark.flaky def test_get_one_catalog_by_schemas( self, adapter, diff --git a/tests/functional/adapter/catalog_tests/test_relation_types.py b/tests/functional/adapter/catalog_tests/test_relation_types.py index 9b9156dec..657bf215b 100644 --- a/tests/functional/adapter/catalog_tests/test_relation_types.py +++ b/tests/functional/adapter/catalog_tests/test_relation_types.py @@ -24,6 +24,7 @@ def docs(self, project): run_dbt(["run"]) yield run_dbt(["docs", "generate"]) + @pytest.mark.flaky @pytest.mark.parametrize( "node_name,relation_type", [ diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index 64f697e77..f201196b4 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -75,6 +75,14 @@ def test_materialized_view_create_idempotent(self, project, my_materialized_view ) assert self.query_relation_type(project, my_materialized_view) == "materialized_view" + @pytest.mark.flaky + def test_table_replaces_materialized_view(self, project, my_materialized_view): + super().test_table_replaces_materialized_view(project, my_materialized_view) + + @pytest.mark.flaky + def test_view_replaces_materialized_view(self, project, my_materialized_view): + super().test_view_replaces_materialized_view(project, my_materialized_view) + class RedshiftMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) @@ -200,6 +208,7 @@ def test_change_is_not_applied_via_alter(self, project, my_materialized_view): assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs, False) assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) + @pytest.mark.flaky def test_change_is_not_applied_via_replace(self, project, my_materialized_view): self.check_start_state(project, my_materialized_view) @@ -224,7 +233,10 @@ class TestRedshiftMaterializedViewChangesFail( RedshiftMaterializedViewChanges, MaterializedViewChangesFailMixin ): # Note: using retries doesn't work when we expect `dbt_run` to fail - pass + + @pytest.mark.flaky + def test_change_is_not_applied_via_replace(self, project, my_materialized_view): + super().test_change_is_not_applied_via_replace(project, my_materialized_view) NO_BACKUP_MATERIALIZED_VIEW = """ @@ -259,6 +271,7 @@ def dbt_run_results(self, project): def test_running_mv_with_backup_false_succeeds(self, dbt_run_results): assert dbt_run_results[0].node.config_call_dict["backup"] is False + @pytest.mark.flaky def test_running_mv_with_backup_false_is_idempotent(self, project, dbt_run_results): """ Addresses: https://github.com/dbt-labs/dbt-redshift/issues/621 diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 64ab24e42..cd04bb1ba 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -38,7 +38,9 @@ # TODO: update these with test cases or remove them if not needed class TestSimpleMaterializationsRedshift(BaseSimpleMaterializations): - pass + @pytest.mark.flaky + def test_base(self, project): + super().test_base(project) class TestSingularTestsRedshift(BaseSingularTests): @@ -54,11 +56,15 @@ class TestEmptyRedshift(BaseEmpty): class TestEphemeralRedshift(BaseEphemeral): - pass + @pytest.mark.flaky + def test_ephemeral(self, project): + super().test_ephemeral(project) class TestIncrementalRedshift(BaseIncremental): - pass + @pytest.mark.flaky + def test_incremental(self, project): + super().test_incremental(project) class TestGenericTestsRedshift(BaseGenericTests): diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py index 61b8bd5a6..6eeaf881c 100644 --- a/tests/functional/adapter/test_persist_docs.py +++ b/tests/functional/adapter/test_persist_docs.py @@ -12,15 +12,21 @@ class TestPersistDocs(BasePersistDocs): - pass + @pytest.mark.flaky + def test_has_comments_pglike(self, project): + super().test_has_comments_pglike(project) class TestPersistDocsColumnMissing(BasePersistDocsColumnMissing): - pass + @pytest.mark.flaky + def test_missing_column(self, project): + super().test_missing_column(project) class TestPersistDocsCommentOnQuotedColumn(BasePersistDocsCommentOnQuotedColumn): - pass + @pytest.mark.flaky + def test_quoted_column_comments(self, run_has_comments): + super().test_quoted_column_comments(run_has_comments) class TestPersistDocsLateBinding(BasePersistDocsBase): @@ -40,6 +46,7 @@ def project_config_update(self): } } + @pytest.mark.flaky def test_comment_on_late_binding_view(self, project): run_dbt() run_dbt(["docs", "generate"]) diff --git a/tox.ini b/tox.ini index c490fed9a..462bc8f07 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ passenv = REDSHIFT_TEST_* PYTEST_ADDOPTS DD_CIVISIBILITY_AGENTLESS_ENABLED + DD_INSTRUMENTATION_TELEMETRY_ENABLED DD_API_KEY DD_SITE DD_ENV From 43968b5ea70254036a9f9549a65da76818cc2d01 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 13 May 2024 18:37:13 -0400 Subject: [PATCH 37/61] fix macos runner typo (#820) --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0620392ba..429d627fc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -67,7 +67,7 @@ jobs: fail-fast: false max-parallel: 3 matrix: - os: [ubuntu-22.04, macos12, windows-2022] + os: [ubuntu-22.04, macos-12, windows-2022] python-version: ["3.8"] include: - os: ubuntu-22.04 From 4b8add4944c76e25f0f4dd8bb26d6ec3e269d174 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Mon, 13 May 2024 18:13:44 -0600 Subject: [PATCH 38/61] Cross-database `date` macro (#806) * Cross-database `date` macro * Temporary dev requirements for testing * Add meaningless newline to bump CI * Update changelog entry * Revert "Add meaningless newline to bump CI" This reverts commit df3519e4e1527ca0f2a46ac3eec5e2f53a087af6. * Revert "Temporary dev requirements for testing" This reverts commit e77835ff94eb89bee94d1304125dae4267c23074. --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .changes/unreleased/Features-20240501-151859.yaml | 6 ++++++ tests/functional/adapter/utils/test_utils.py | 5 +++++ 2 files changed, 11 insertions(+) create mode 100644 .changes/unreleased/Features-20240501-151859.yaml diff --git a/.changes/unreleased/Features-20240501-151859.yaml b/.changes/unreleased/Features-20240501-151859.yaml new file mode 100644 index 000000000..e4791638c --- /dev/null +++ b/.changes/unreleased/Features-20240501-151859.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Cross-database `date` macro +time: 2024-05-01T15:18:59.292022-06:00 +custom: + Author: dbeatty10 + Issue: 808 diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 61a706f4d..174c44b6c 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -6,6 +6,7 @@ from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_concat import BaseConcat from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive +from dbt.tests.adapter.utils.test_date import BaseDate from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine @@ -61,6 +62,10 @@ class TestCurrentTimestamp(BaseCurrentTimestampNaive): pass +class TestDate(BaseDate): + pass + + class TestDateAdd(BaseDateAdd): pass From 248928065bd01e8457a9e47f0705cf98b8a5b055 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Tue, 14 May 2024 13:19:43 -0600 Subject: [PATCH 39/61] Import relevant pytest(s) for cross-database `cast` macro (#805) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .changes/unreleased/Features-20240430-185708.yaml | 6 ++++++ tests/functional/adapter/utils/test_utils.py | 5 +++++ 2 files changed, 11 insertions(+) create mode 100644 .changes/unreleased/Features-20240430-185708.yaml diff --git a/.changes/unreleased/Features-20240430-185708.yaml b/.changes/unreleased/Features-20240430-185708.yaml new file mode 100644 index 000000000..59407c196 --- /dev/null +++ b/.changes/unreleased/Features-20240430-185708.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add tests for cross-database `cast` macro +time: 2024-04-30T18:57:08.04492-06:00 +custom: + Author: dbeatty10 + Issue: "804" diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 174c44b6c..3a085712a 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -3,6 +3,7 @@ from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_concat import BaseConcat from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive @@ -49,6 +50,10 @@ class TestBoolOr(BaseBoolOr): pass +class TestCast(BaseCast): + pass + + class TestCastBoolToText(BaseCastBoolToText): pass From 3b6de5b3fbd4503e1ab12ea758f9745e8cec7eb0 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 20 May 2024 21:54:56 -0400 Subject: [PATCH 40/61] Add docker release to the full release process for final releases (#758) * add docker release to release pipeline * add docker to dependabot * update to align with other adapters * update build arg in test * removed defaulted input for docker package * add permissions for nightly test release * account for draft releases not havign docker, allow docker only release * Update docker/Dockerfile * pull release-prep into this repo to install aws cli, use env vars, and deal with flaky tests * remove unused test script --------- Co-authored-by: Emily Rockman --- .github/dependabot.yml | 5 + .github/workflows/nightly-release.yml | 1 + .github/workflows/release-prep.yml | 702 ++++++++++++++++++++++++++ .github/workflows/release.yml | 53 +- Makefile | 10 + docker/Dockerfile | 37 ++ docker/README.md | 58 +++ docker/dev.Dockerfile | 50 ++ 8 files changed, 890 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/release-prep.yml create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/dev.Dockerfile diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4673f47cf..ae2be43aa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,8 @@ updates: schedule: interval: "weekly" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 46db5b749..3c36361d5 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -20,6 +20,7 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write # this is the permission that allows Docker release defaults: run: diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml new file mode 100644 index 000000000..5c5ff0cd7 --- /dev/null +++ b/.github/workflows/release-prep.yml @@ -0,0 +1,702 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# sha: The commit to attach to this release +# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# target_branch: The branch that we will release from +# env_setup_script_path: Path to the environment setup script +# test_run: Test run (The temp branch will be used for release) +# nightly_release: Identifier that this is nightly release +# +# Outputs: +# final_sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For test runs: prep-release/test-run/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +# + +name: Version Bump and Changelog Generation + +on: + workflow_call: + inputs: + sha: + required: true + type: string + version_number: + required: true + type: string + target_branch: + required: true + type: string + env_setup_script_path: + required: false + type: string + default: "" + test_run: + required: false + default: true + type: boolean + nightly_release: + type: boolean + default: false + required: false + outputs: + final_sha: + description: The new commit that includes the changelog and version bump. + value: ${{ jobs.determine-release-sha.outputs.final_sha }} + changelog_path: + description: The path to the changelog for this version + value: ${{ jobs.audit-changelog.outputs.changelog_path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Release Preparation]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + + steps: + - name: "[DEBUG] Print Variables" + run: | + # WORKFLOW INPUTS + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo The branch that we will release from: ${{ inputs.target_branch }} + echo Path to the environment setup script: ${{ inputs.env_setup_script_path }} + echo Test run: ${{ inputs.test_run }} + echo Nightly release: ${{ inputs.nightly_release }} + # ENVIRONMENT VARIABLES + echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + runs-on: ubuntu-latest + + outputs: + changelog_path: ${{ steps.set_path.outputs.changelog_path }} + exists: ${{ steps.set_existence.outputs.exists }} + base_version: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + - name: "Set Changelog Path" + id: set_path + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + # Send notification + echo "changelog_path=$path" >> $GITHUB_OUTPUT + title="Changelog path" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" + + - name: "Set Changelog Existence For Subsequent Jobs" + id: set_existence + run: | + does_exist=false + if test -f ${{ steps.set_path.outputs.changelog_path }} + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[Notification] Set Changelog Existence For Subsequent Jobs" + run: | + title="Changelog exists" + if [[ ${{ steps.set_existence.outputs.exists }} == true ]] + then + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" + else + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} + echo exists: ${{ steps.set_existence.outputs.exists }} + echo base_version: ${{ steps.semver.outputs.base-version }} + echo prerelease: ${{ steps.semver.outputs.pre-release }} + echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + audit-version-in-code: + runs-on: ubuntu-latest + + outputs: + up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Check Current Version In Code" + id: version-check + run: | + is_updated=false + if grep -Fxq "current_version = ${{ inputs.version_number }}" .bumpversion.cfg + then + is_updated=true + fi + echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT + + - name: "[Notification] Check Current Version In Code" + run: | + title="Version check" + if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] + then + message="The version in the codebase is equal to the provided version" + else + message="The version in the codebase differs from the provided version" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + skip-generate-changelog: + runs-on: ubuntu-latest + needs: [audit-changelog] + if: needs.audit-changelog.outputs.exists == 'true' + + steps: + - name: "Changelog Exists, Skip Generating New Changelog" + run: | + # Send notification + title="Skip changelog generation" + message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + runs-on: ubuntu-latest + needs: [audit-version-in-code] + if: needs.audit-version-in-code.outputs.up_to_date == 'true' + + steps: + - name: "Version Already Bumped" + run: | + # Send notification + title="Skip version bump" + message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + create-temp-branch: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code] + if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' + + outputs: + branch_name: ${{ steps.variables.outputs.branch_name }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Generate Branch Name" + id: variables + run: | + name="prep-release/" + if [[ ${{ inputs.nightly_release }} == true ]] + then + name+="nightly-release/" + elif [[ ${{ inputs.test_run }} == true ]] + then + name+="test-run/" + fi + name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" + echo "branch_name=$name" >> $GITHUB_OUTPUT + + - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" + run: | + git checkout -b ${{ steps.variables.outputs.branch_name }} + git push -u origin ${{ steps.variables.outputs.branch_name }} + + - name: "[Notification] Temp branch created" + run: | + # Send notification + title="Temp branch generated" + message="The ${{ steps.variables.outputs.branch_name }} branch created" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo branch_name ${{ steps.variables.outputs.branch_name }} + + generate-changelog-bump-version: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code, create-temp-branch] + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Add Homebrew To PATH" + run: | + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install Homebrew Packages" + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set json File Name" + id: json_file + run: | + echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get Core Team Membership" + run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core Team Membership for Changie Contributors exclusion" + id: set_team_membership + run: | + team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) + echo $team_list + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV + + - name: "Delete the json File" + run: | + rm ${{ steps.json_file.outputs.name }} + + - name: "Generate Release Changelog" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + git status + + - name: "Check Changelog Created Successfully" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + title="Changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + python3 -m venv env + source env/bin/activate + python -m pip install --upgrade pip + + - name: "Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + # note: bumpversion is no longer supported, it actually points to bump2version now + run: | + source env/bin/activate + if [ -f "editable-requirements.txt" ] + then + python -m pip install -r dev-requirements.txt -r editable-requirements.txt + else + python -m pip install -r dev-requirements.txt + fi + env/bin/bumpversion --allow-dirty --new-version ${{ inputs.version_number }} major + git status + + - name: "[Notification] Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + title="Version bump" + message="Version successfully bumped in codebase to ${{ inputs.version_number }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + # this step will fail on whitespace errors but also correct them + - name: "Remove Trailing Whitespace Via Pre-commit" + continue-on-error: true + run: | + pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + # this step will fail on newline errors but also correct them + - name: "Removing Extra Newlines Via Pre-commit" + continue-on-error: true + run: | + pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + - name: "Commit & Push Changes" + run: | + #Data for commit + user="Github Build Bot" + email="buildbot@fishtownanalytics.com" + commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" + #Commit changes to branch + git config user.name "$user" + git config user.email "$email" + git pull + git add . + git commit -m "$commit_message" + git push + + run-unit-tests: + runs-on: ubuntu-latest + needs: [create-temp-branch, generate-changelog-bump-version] + + env: + TOXENV: unit + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + python -m tox --version + + - name: "Run Tox" + run: tox + + run-integration-tests: + runs-on: ubuntu-22.04 + needs: [create-temp-branch, generate-changelog-bump-version] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m "not flaky" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + run-integration-tests-flaky: + runs-on: ubuntu-22.04 + needs: [run-integration-tests, create-temp-branch] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m flaky + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + merge-changes-into-target-branch: + runs-on: ubuntu-latest + needs: [run-unit-tests, run-integration-tests-flaky, create-temp-branch, audit-version-in-code, audit-changelog] + if: | + !failure() && !cancelled() && + inputs.test_run == false && + ( + needs.audit-changelog.outputs.exists == 'false' || + needs.audit-version-in-code.outputs.up_to_date == 'false' + ) + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} + echo inputs.test_run: ${{ inputs.test_run }} + echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} + echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Checkout Repo ${{ github.repository }}" + uses: actions/checkout@v4 + + - name: "Merge Changes Into ${{ inputs.target_branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} + target_branch: ${{ inputs.target_branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[Notification] Changes Merged into ${{ inputs.target_branch }}" + run: | + title="Changelog and Version Bump Branch Merge" + message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into ${{ inputs.target_branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + determine-release-sha: + runs-on: ubuntu-latest + needs: + [ + create-temp-branch, + merge-changes-into-target-branch, + audit-changelog, + audit-version-in-code, + ] + # always run this job, regardless of if the dependant jobs were skipped + if: ${{ !failure() && !cancelled() }} + + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} + echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} + echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Resolve Branch To Checkout" + id: resolve_branch + run: | + branch="" + if [[ ${{ inputs.test_run == true }} ]] + then + branch=${{ needs.create-temp-branch.outputs.branch_name }} + else + branch=${{ inputs.target_branch }} + fi + echo "target_branch=$branch" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Branch To Checkout" + run: | + title="Branch pick" + message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.resolve_branch.outputs.target_branch }} + + - name: "[Debug] Log Branch" + run: git status + + - name: "Resolve Commit SHA For Release" + id: resolve_commit_sha + run: | + commit_sha="" + if [[ ${{ needs.audit-changelog.outputs.exists }} == false ]] || [[ ${{ needs.audit-version-in-code.outputs.up_to_date }} == false ]] + then + commit_sha=$(git rev-parse HEAD) + else + commit_sha=${{ inputs.sha }} + fi + echo "release_sha=$commit_sha" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Commit SHA For Release" + run: | + title="Release commit pick" + message="The ${{ steps.resolve_commit_sha.outputs.release_sha }} commit will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" + if: ${{ inputs.test_run == false && needs.create-temp-branch.outputs.branch_name != '' }} + run: | + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "[Debug] Print Outputs" + run: | + echo release_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68eb3320..e01b961f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: @@ -60,6 +61,11 @@ on: type: boolean default: false required: false + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false workflow_call: inputs: sha: @@ -131,9 +137,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - + uses: ./.github/workflows/release-prep.yml with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} @@ -141,17 +145,13 @@ jobs: env_setup_script_path: ${{ inputs.env_setup_script_path }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -160,11 +160,9 @@ jobs: build-test-package: name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -174,19 +172,15 @@ jobs: package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} github-release: name: GitHub Release - if: ${{ !failure() && !cancelled() }} - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -195,34 +189,41 @@ jobs: pypi-release: name: PyPI Release - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [github-release] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + docker-release: + name: "Docker Release" + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} + needs: [bump-version-generate-changelog, build-test-package, github-release] + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - bump-version-generate-changelog, - build-test-package, github-release, pypi-release, + docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/Makefile b/Makefile index efd23b806..f32c3ba8f 100644 --- a/Makefile +++ b/Makefile @@ -65,3 +65,13 @@ help: ## Show this help message. @echo @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + + +.PHONY: docker-dev +docker-dev: + docker build -f docker/dev.Dockerfile -t dbt-redshift-dev . + docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker/Dockerfile -t dbt-redshift . diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..5914e6396 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,37 @@ +# this image gets published to GHCR for production use +ARG py_version=3.11.2 + +FROM python:$py_version-slim-bullseye as base + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-redshift + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..5be9e56ef --- /dev/null +++ b/docker/README.md @@ -0,0 +1,58 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-redshift` + +In order to build a new image, run the following docker command. +```shell +docker build --tag --target dbt-redshift +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the image will be populated with the latest version of `dbt-redshift` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +```shell +docker build --tag \ + --target dbt-redshift \ + --build-arg commit_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +```shell +cd dbt-core/docker +docker build --tag my-dbt --target dbt-redshift . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +```shell +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-redshift \ + --build-arg commit_ref=v1.0.0b1 \ + . +``` + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +```shell +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 000000000..aac016320 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,50 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + + +FROM base as dbt-redshift-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv From cef201f95677dd2dec5f0875ea67a258f4d3a6b0 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 21 May 2024 00:43:58 -0400 Subject: [PATCH 41/61] Add aws cli setup to the internal PyPI release pipeline (#829) * vendor `internal-archive-release.yml` from `dbt-release`; remove unnecessary conditionals given the scope is now dbt-redshift * add aws cli setup for integration tests * replace secrets generation with secrets from existing integration test workflow * add breakout between flaky and non-flaky tests --- .github/workflows/release-internal.yml | 270 +++++++++++++++++++++++-- 1 file changed, 257 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 0ce0f9adf..3bba79a26 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -1,15 +1,11 @@ # What? -# # Tag and release an arbitrary ref. Uploads to an internal archive for further processing. # # How? -# # After checking out and testing the provided ref, the image is built and uploaded. # # When? -# # Manual trigger. - name: "Release internal patch" on: @@ -34,16 +30,264 @@ defaults: run: shell: "bash" +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Internal Archive Release]" + TEMP_PROFILE_NAME: "temp_aws_profile" + jobs: - invoke-reusable-workflow: - name: "Build and Release Internally" + job-setup: + name: Job Setup + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Print Variables" + run: | + echo The release version number: ${{ inputs.version_number }} + echo The release ref: ${{ inputs.ref }} + echo Package test command: ${{ inputs.package_test_command }} + + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Validate patch version input against patch version of ref" + id: validate_version + run: | + version_in_file="$(grep -E 'version(: str)? =' "dbt/adapters/redshift/__version__.py" | cut -d '"' -f2)" + + if [[ "${{ inputs.version_number }}" != "${version_in_file}" ]]; then + message="Error: patch version input to this job ${{ inputs.version_number }} and version of code at input ref ${version_in_file} are not equal. Exiting..." + echo "::error $title::$message" + exit 1 + fi + + unit-tests: + name: 'Unit Tests' + runs-on: ubuntu-latest + needs: job-setup + env: + TOXENV: unit + + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + python -m tox --version + + - name: "Run Tests" + run: tox + + integration-tests: + name: 'Integration Tests' + runs-on: ubuntu-latest + needs: unit-tests + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n4" + DBT_INVOCATION_ENV: github-actions + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: "Create AWS IAM profiles" + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: "Run tox" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m "not flaky" + + integration-tests-flaky: + name: 'Integration Tests (flaky)' + runs-on: ubuntu-latest + needs: integration-tests + + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n1" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: "Run tox" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m flaky + + create-internal-release: + name: Create release for internal archive + runs-on: ubuntu-latest + needs: integration-tests-flaky + + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install --upgrade setuptools wheel twine check-wheel-contents + python -m pip --version + + - name: "Configure AWS profile for upload" + run: | + aws configure set aws_access_key_id ${{ secrets.AWS_ARCHIVE_ACCESS_KEY_ID }} --profile ${{ env.TEMP_PROFILE_NAME }} + aws configure set aws_secret_access_key ${{ secrets.AWS_ARCHIVE_SECRET_ACCESS_KEY }} --profile ${{ env.TEMP_PROFILE_NAME }} + aws configure set region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} + aws configure set output text --profile ${{ env.TEMP_PROFILE_NAME }} + aws codeartifact login --tool twine --repository ${{ secrets.AWS_REPOSITORY }} --domain ${{ secrets.AWS_DOMAIN }} --domain-owner ${{ secrets.AWS_DOMAIN_OWNER }} --region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} + + - name: "Alter version in metadata of python package" + run: | + version_file="dbt/adapters/redshift/__version__.py" + setup_file="./setup.py" + version_in_file=$(grep -E 'version(: str)? =' "${version_file}" | cut -d '"' -f2) + + # check the latest build of adapter code in our archive + versions_on_aws="$(aws codeartifact list-package-versions --repository ${{ secrets.AWS_REPOSITORY }} --domain ${{ secrets.AWS_DOMAIN }} --domain-owner ${{ secrets.AWS_DOMAIN_OWNER }} --region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} --format pypi --package dbt-redshift --output json --query 'versions[*].version' | jq -r '.[]' | grep "^${{ inputs.version_number }}" || true )" # suppress pipefail only here + + current_latest_version="$(echo "${versions_on_aws}" | sort -V | tail -n 1 )" + + echo "[Debug] version_in_file: ${version_in_file}" + echo "[Debug] current_latest_version: ${current_latest_version}" + + echo ">>> Altering ${version_file}" + # Ensure a build+xxx where xxx is an integer is always present in versioning + # sed may be a no-op -- this is fine! + if [[ ${current_latest_version} =~ (.*build)([0-9]+)$ ]]; then + base="${BASH_REMATCH[1]}" + number="${BASH_REMATCH[2]}" + new_number=$((number + 1)) + v="${base}${new_number}" + tee <<< "version = \"${v}\"" "${version_file}" + if [ -f "${setup_file}" ]; then + sed -i "s/^package_version = .*$/package_version = \"${v}\"/" "${setup_file}" + fi + else + v="${version_in_file}+build1" + tee <<< "version = \"${v}\"" "${version_file}" + if [ -f "${setup_file}" ]; then + sed -i "s/^package_version = .*$/package_version = \"${v}\"/" "${setup_file}" + fi + fi + + - name: "Build Distributions - scripts/build-dist.sh" + run: scripts/build-dist.sh + + - name: "[DEBUG] Show Distributions" + run: ls -lh dist/ + + - name: "Check Distribution Descriptions" + run: twine check dist/* - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" + - name: "[DEBUG] Check Wheel Contents" + run: check-wheel-contents dist/*.whl --ignore W007,W008 - with: - version_number: "${{ inputs.version_number }}" - package_test_command: "${{ inputs.package_test_command }}" - dbms_name: "redshift" - ref: "${{ inputs.ref }}" + - name: "Upload Build Artifact - ${{ inputs.version_number }}" + run: | + twine upload --repository codeartifact dist/* - secrets: "inherit" + version_file="$(echo "dbt/adapters/redshift/__version__.py")" + version="$(grep 'version =' "${version_file}" | cut -d '"' -f2)" + message="-- Success -- released ${version}" + echo "::notice $NOTIFICATION_PREFIX::$message" From 0c3f514978814ec302d7f31cf3b7bf8e3fe0a98c Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:45:43 +0000 Subject: [PATCH 42/61] Adap 746/merge agate lazy load (#831) * lazy load agate * Add test and documentation. * Alter doc strings. * Lint. * Fix test. * Don't need a test for this. --------- Co-authored-by: dwreeves Co-authored-by: Mila Page --- .../unreleased/Under the Hood-20240331-103115.yaml | 6 ++++++ dbt/adapters/redshift/connections.py | 13 +++++++++---- dbt/adapters/redshift/impl.py | 9 ++++++--- dbt/adapters/redshift/relation_configs/base.py | 11 ++++++++--- dbt/adapters/redshift/relation_configs/dist.py | 8 +++++--- .../redshift/relation_configs/materialized_view.py | 10 ++++++---- dbt/adapters/redshift/relation_configs/sort.py | 8 +++++--- 7 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240331-103115.yaml diff --git a/.changes/unreleased/Under the Hood-20240331-103115.yaml b/.changes/unreleased/Under the Hood-20240331-103115.yaml new file mode 100644 index 000000000..ffb5a6489 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240331-103115.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Lazy load agate +time: 2024-03-31T10:31:15.65006-04:00 +custom: + Author: dwreeves + Issue: "745" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 752c81e32..dfd7ab30b 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -1,14 +1,12 @@ import re from multiprocessing import Lock from contextlib import contextmanager -from typing import Any, Callable, Dict, Tuple, Union, Optional, List +from typing import Any, Callable, Dict, Tuple, Union, Optional, List, TYPE_CHECKING from dataclasses import dataclass, field -import agate import sqlparse import redshift_connector from dbt.adapters.exceptions import FailedToConnectError -from dbt_common.clients import agate_helper from redshift_connector.utils.oids import get_datatype_name from dbt.adapters.sql import SQLConnectionManager @@ -19,6 +17,11 @@ from dbt_common.helper_types import Port from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError +if TYPE_CHECKING: + # Indirectly imported via agate_helper, which is lazy loaded further downfile. + # Used by mypy for earlier type hints. + import agate + class SSLConfigError(CompilationError): def __init__(self, exc: ValidationError): @@ -393,13 +396,15 @@ def execute( auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None, - ) -> Tuple[AdapterResponse, agate.Table]: + ) -> Tuple[AdapterResponse, "agate.Table"]: sql = self._add_query_comment(sql) _, cursor = self.add_query(sql, auto_begin) response = self.get_response(cursor) if fetch: table = self.get_result_from_cursor(cursor, limit) else: + from dbt_common.clients import agate_helper + table = agate_helper.empty_table() return response, table diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 18faee48c..d498685ed 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,7 +1,7 @@ import os from dataclasses import dataclass from dbt_common.contracts.constraints import ConstraintType -from typing import Optional, Set, Any, Dict, Type +from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING from collections import namedtuple from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport @@ -28,6 +28,9 @@ GET_RELATIONS_MACRO_NAME = "redshift__get_relations" +if TYPE_CHECKING: + import agate + @dataclass class RedshiftConfig(AdapterConfig): @@ -85,7 +88,7 @@ def drop_relation(self, relation): return super().drop_relation(relation) @classmethod - def convert_text_type(cls, agate_table, col_idx): + def convert_text_type(cls, agate_table: "agate.Table", col_idx): column = agate_table.columns[col_idx] # `lens` must be a list, so this can't be a generator expression, # because max() raises ane exception if its argument has no members. @@ -94,7 +97,7 @@ def convert_text_type(cls, agate_table, col_idx): return "varchar({})".format(max_len) @classmethod - def convert_time_type(cls, agate_table, col_idx): + def convert_time_type(cls, agate_table: "agate.Table", col_idx): return "varchar(24)" @available diff --git a/dbt/adapters/redshift/relation_configs/base.py b/dbt/adapters/redshift/relation_configs/base.py index c4faab664..6f1409659 100644 --- a/dbt/adapters/redshift/relation_configs/base.py +++ b/dbt/adapters/redshift/relation_configs/base.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from typing import Optional, Dict +from typing import Optional, Dict, TYPE_CHECKING -import agate from dbt.adapters.base.relation import Policy from dbt.adapters.contracts.relation import ComponentName, RelationConfig from dbt.adapters.relation_configs import ( @@ -15,6 +14,10 @@ RedshiftQuotePolicy, ) +if TYPE_CHECKING: + # Imported downfile for specific row gathering function. + import agate + @dataclass(frozen=True, eq=True, unsafe_hash=True) class RedshiftRelationConfigBase(RelationConfigBase): @@ -63,8 +66,10 @@ def _render_part(cls, component: ComponentName, value: Optional[str]) -> Optiona return None @classmethod - def _get_first_row(cls, results: agate.Table) -> agate.Row: + def _get_first_row(cls, results: "agate.Table") -> "agate.Row": try: return results.rows[0] except IndexError: + import agate + return agate.Row(values=set()) diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/relation_configs/dist.py index 0104d8db4..2bcdb9566 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/relation_configs/dist.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional, Set, Dict +from typing import Optional, Set, Dict, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationConfigChange, RelationConfigChangeAction, @@ -15,6 +14,9 @@ from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +if TYPE_CHECKING: + import agate + class RedshiftDistStyle(StrEnum): auto = "auto" @@ -108,7 +110,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> dict: return config @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> Dict: + def parse_relation_results(cls, relation_results_entry: "agate.Row") -> Dict: """ Translate agate objects from the database into a standard dictionary. diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index f6d93754e..48c04b554 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -1,7 +1,6 @@ from dataclasses import dataclass, field -from typing import Optional, Set, Dict, Any +from typing import Optional, Set, Dict, Any, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationResults, RelationConfigChange, @@ -25,6 +24,9 @@ ) from dbt.adapters.redshift.utility import evaluate_bool +if TYPE_CHECKING: + import agate + @dataclass(frozen=True, eq=True, unsafe_hash=True) class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigValidationMixin): @@ -173,10 +175,10 @@ def parse_relation_results(cls, relation_results: RelationResults) -> Dict: Returns: a standard dictionary describing this `RedshiftMaterializedViewConfig` instance """ - materialized_view: agate.Row = cls._get_first_row( + materialized_view: "agate.Row" = cls._get_first_row( relation_results.get("materialized_view") ) - query: agate.Row = cls._get_first_row(relation_results.get("query")) + query: "agate.Row" = cls._get_first_row(relation_results.get("query")) config_dict = { "mv_name": materialized_view.get("table"), diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/relation_configs/sort.py index 91152615e..be5b3627d 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/relation_configs/sort.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional, FrozenSet, Set, Dict, Any +from typing import Optional, FrozenSet, Set, Dict, Any, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationConfigChange, RelationConfigChangeAction, @@ -15,6 +14,9 @@ from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +if TYPE_CHECKING: + import agate + class RedshiftSortStyle(StrEnum): auto = "auto" @@ -136,7 +138,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any return config_dict @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_relation_results(cls, relation_results_entry: "agate.Row") -> dict: """ Translate agate objects from the database into a standard dictionary. From a83553b7f160f46f3c6840eb98a17a86525a7e08 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 13 Jun 2024 02:33:23 +0000 Subject: [PATCH 43/61] Pin to 2.1.1 and below to fix the cursor error. (#844) * Pin to 2.1.1 and below to fix the cursor error. * Update setup.py * add changie --------- Co-authored-by: Mila Page Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Colin --- .changes/unreleased/Fixes-20240612-131752.yaml | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20240612-131752.yaml diff --git a/.changes/unreleased/Fixes-20240612-131752.yaml b/.changes/unreleased/Fixes-20240612-131752.yaml new file mode 100644 index 000000000..36ea4d922 --- /dev/null +++ b/.changes/unreleased/Fixes-20240612-131752.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: update pin range for redshift-connector to allow 2.1.0 +time: 2024-06-12T13:17:52.460407-07:00 +custom: + Author: colin-rogers-dbt VersusFacit + Issue: "844" diff --git a/setup.py b/setup.py index dbb3913b9..c2649e66a 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def _plugin_version_trim() -> str: f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", + "redshift-connector<2.1.1,>=2.0.913,!=2.0.914", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0b3", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core From 885db2db7c049308340038efdaa0e83214219ded Mon Sep 17 00:00:00 2001 From: Florian Eiden Date: Wed, 12 Jun 2024 21:30:33 -0700 Subject: [PATCH 44/61] Remove group_federation for serverless (#836) * Remove grou_federation for serverless * Add changie log * Fix test on cluster_id * Add explicit role_arn support * Revert "Add explicit role_arn support" This reverts commit 00c53d6e5d1d4bb5433ea0fae4cebcb1fd7319a6. * Add tests * Linting * Linting * Tests should ignore group_federation, and not expect false * Pick the correct method for the test * db_user should line up with IAMRole on Provisionned * Align tests to spec * Explicitely set group federation for serverless * Solved the cluster id issue - should be null on serverless * Update changie desc --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Fixes-20240531-113620.yaml | 6 + dbt/adapters/redshift/connections.py | 19 +-- tests/unit/test_auth_method.py | 117 ++++++++++++++++++ 3 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240531-113620.yaml diff --git a/.changes/unreleased/Fixes-20240531-113620.yaml b/.changes/unreleased/Fixes-20240531-113620.yaml new file mode 100644 index 000000000..a020ca045 --- /dev/null +++ b/.changes/unreleased/Fixes-20240531-113620.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support IAM Role authentication for Redshift Serverless +time: 2024-05-31T11:36:20.397521-07:00 +custom: + Author: fleid + Issue: "835" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index dfd7ab30b..d3fbcafea 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -237,10 +237,15 @@ def _iam_user_kwargs(self) -> Dict[str, Any]: def _iam_role_kwargs(self) -> Dict[str, Optional[Any]]: logger.debug("Connecting to redshift with 'iam_role' credentials method") kwargs = self._iam_kwargs - kwargs.update( - group_federation=True, - db_user=None, - ) + + # It's a role, we're ignoring the user + kwargs.update(db_user=None) + + # Serverless shouldn't get group_federation, Provisoned clusters should + if "serverless" in self.credentials.host: + kwargs.update(group_federation=False) + else: + kwargs.update(group_federation=True) if iam_profile := self.credentials.iam_profile: kwargs.update(profile=iam_profile) @@ -256,10 +261,10 @@ def _iam_kwargs(self) -> Dict[str, Any]: password="", ) - if cluster_id := self.credentials.cluster_id: - kwargs.update(cluster_identifier=cluster_id) - elif "serverless" in self.credentials.host: + if "serverless" in self.credentials.host: kwargs.update(cluster_identifier=None) + elif cluster_id := self.credentials.cluster_id: + kwargs.update(cluster_identifier=cluster_id) else: raise FailedToConnectError( "Failed to use IAM method:" diff --git a/tests/unit/test_auth_method.py b/tests/unit/test_auth_method.py index bd9912d0c..55b1aad74 100644 --- a/tests/unit/test_auth_method.py +++ b/tests/unit/test_auth_method.py @@ -456,3 +456,120 @@ def test_profile(self): group_federation=True, **DEFAULT_SSL_CONFIG, ) + + +class TestIAMRoleMethodServerless(AuthMethod): + # Should behave like IAM Role provisioned, with the exception of not having group_federation set + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_default_region(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + database="redshift", + cluster_identifier=None, + region=None, + auto_create=False, + db_groups=[], + db_user=None, + password="", + user="", + profile="iam_profile_test", + timeout=None, + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_ignore_cluster(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + cluster_id="my_redshift", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + database="redshift", + cluster_identifier=None, + region=None, + auto_create=False, + db_groups=[], + db_user=None, + password="", + user="", + profile="iam_profile_test", + timeout=None, + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_explicit_region(self): + # Successful test + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.redshift-serverless.amazonaws.com", + region="us-east-2", + ) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.redshift-serverless.amazonaws.com", + database="redshift", + cluster_identifier=None, + region="us-east-2", + auto_create=False, + db_groups=[], + db_user=None, + password="", + user="", + profile="iam_profile_test", + timeout=None, + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_invalid_serverless(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", + ) + with self.assertRaises(FailedToConnectError) as context: + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", + database="redshift", + cluster_identifier=None, + region=None, + auto_create=False, + db_groups=[], + db_user=None, + password="", + user="", + profile="iam_profile_test", + port=5439, + timeout=None, + group_federation=False, + **DEFAULT_SSL_CONFIG, + ) + self.assertTrue("'host' must be provided" in context.exception.msg) From a1cd9dd6ffdcefa1e7fa0ba32e521d2789837e5b Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:31:28 -0500 Subject: [PATCH 45/61] update user docs-issue workflow (#847) * update user docs-issue workflow --- .github/workflows/docs-issues.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml index 00a098df8..f49cf517c 100644 --- a/.github/workflows/docs-issues.yml +++ b/.github/workflows/docs-issues.yml @@ -1,19 +1,18 @@ # **what?** -# Open an issue in docs.getdbt.com when a PR is labeled `user docs` +# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed # **why?** # To reduce barriers for keeping docs up to date # **when?** -# When a PR is labeled `user docs` and is merged. Runs on pull_request_target to run off the workflow already merged, -# not the workflow that existed on the PR branch. This allows old PRs to get comments. +# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. -name: Open issues in docs.getdbt.com repo when a PR is labeled -run-name: "Open an issue in docs.getdbt.com for PR #${{ github.event.pull_request.number }}" +name: Open issues in docs.getdbt.com repo when an issue is labeled +run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" on: - pull_request_target: + issues: types: [labeled, closed] defaults: @@ -21,23 +20,22 @@ defaults: shell: bash permissions: - issues: write # opens new issues - pull-requests: write # comments on PRs - + issues: write # comments on issues jobs: open_issues: - # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`. Otherwise it runs the + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we # decide if it should run or not. if: | - (github.event.pull_request.merged == true) && - ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) || + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || (github.event.action == 'labeled' && github.event.label.name == 'user docs')) uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main with: issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} PR #${{ github.event.pull_request.number }}" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." secrets: inherit From 38856994c52708f2c50b16e00dcea618c4e945fb Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 20 Jun 2024 18:34:56 -0700 Subject: [PATCH 46/61] Update internal release naming and arguments (#849) * Update internal release naming and arguments * add comment on skip_tests --- .github/workflows/release-internal.yml | 112 +++---------------------- 1 file changed, 13 insertions(+), 99 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 3bba79a26..30afe9111 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -6,15 +6,12 @@ # # When? # Manual trigger. -name: "Release internal patch" +name: "Release to Cloud" +run-name: "Release to Cloud off of ${{ inputs.ref }}" on: workflow_dispatch: inputs: - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true ref: description: "The ref (sha or branch name) to use" type: string @@ -42,7 +39,6 @@ jobs: steps: - name: "[DEBUG] Print Variables" run: | - echo The release version number: ${{ inputs.version_number }} echo The release ref: ${{ inputs.ref }} echo Package test command: ${{ inputs.package_test_command }} @@ -51,17 +47,6 @@ jobs: with: ref: "${{ inputs.ref }}" - - name: "Validate patch version input against patch version of ref" - id: validate_version - run: | - version_in_file="$(grep -E 'version(: str)? =' "dbt/adapters/redshift/__version__.py" | cut -d '"' -f2)" - - if [[ "${{ inputs.version_number }}" != "${version_in_file}" ]]; then - message="Error: patch version input to this job ${{ inputs.version_number }} and version of code at input ref ${version_in_file} are not equal. Exiting..." - echo "::error $title::$message" - exit 1 - fi - unit-tests: name: 'Unit Tests' runs-on: ubuntu-latest @@ -207,87 +192,16 @@ jobs: DBT_TEST_USER_3: dbt_test_user_3 run: tox -- -m flaky - create-internal-release: - name: Create release for internal archive - runs-on: ubuntu-latest - needs: integration-tests-flaky - - steps: - - name: "Checkout provided ref, default to branch main" - uses: actions/checkout@v4 - with: - ref: "${{ inputs.ref }}" - - - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} + call-release-workflow: + needs: [unit-tests, integration-tests, integration-tests-flaky] + name: "Create cloud release" + uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - - name: "Install Python Dependencies" - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: "Configure AWS profile for upload" - run: | - aws configure set aws_access_key_id ${{ secrets.AWS_ARCHIVE_ACCESS_KEY_ID }} --profile ${{ env.TEMP_PROFILE_NAME }} - aws configure set aws_secret_access_key ${{ secrets.AWS_ARCHIVE_SECRET_ACCESS_KEY }} --profile ${{ env.TEMP_PROFILE_NAME }} - aws configure set region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} - aws configure set output text --profile ${{ env.TEMP_PROFILE_NAME }} - aws codeartifact login --tool twine --repository ${{ secrets.AWS_REPOSITORY }} --domain ${{ secrets.AWS_DOMAIN }} --domain-owner ${{ secrets.AWS_DOMAIN_OWNER }} --region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} - - - name: "Alter version in metadata of python package" - run: | - version_file="dbt/adapters/redshift/__version__.py" - setup_file="./setup.py" - version_in_file=$(grep -E 'version(: str)? =' "${version_file}" | cut -d '"' -f2) - - # check the latest build of adapter code in our archive - versions_on_aws="$(aws codeartifact list-package-versions --repository ${{ secrets.AWS_REPOSITORY }} --domain ${{ secrets.AWS_DOMAIN }} --domain-owner ${{ secrets.AWS_DOMAIN_OWNER }} --region ${{ secrets.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }} --format pypi --package dbt-redshift --output json --query 'versions[*].version' | jq -r '.[]' | grep "^${{ inputs.version_number }}" || true )" # suppress pipefail only here - - current_latest_version="$(echo "${versions_on_aws}" | sort -V | tail -n 1 )" - - echo "[Debug] version_in_file: ${version_in_file}" - echo "[Debug] current_latest_version: ${current_latest_version}" - - echo ">>> Altering ${version_file}" - # Ensure a build+xxx where xxx is an integer is always present in versioning - # sed may be a no-op -- this is fine! - if [[ ${current_latest_version} =~ (.*build)([0-9]+)$ ]]; then - base="${BASH_REMATCH[1]}" - number="${BASH_REMATCH[2]}" - new_number=$((number + 1)) - v="${base}${new_number}" - tee <<< "version = \"${v}\"" "${version_file}" - if [ -f "${setup_file}" ]; then - sed -i "s/^package_version = .*$/package_version = \"${v}\"/" "${setup_file}" - fi - else - v="${version_in_file}+build1" - tee <<< "version = \"${v}\"" "${version_file}" - if [ -f "${setup_file}" ]; then - sed -i "s/^package_version = .*$/package_version = \"${v}\"/" "${setup_file}" - fi - fi - - - name: "Build Distributions - scripts/build-dist.sh" - run: scripts/build-dist.sh - - - name: "[DEBUG] Show Distributions" - run: ls -lh dist/ - - - name: "Check Distribution Descriptions" - run: twine check dist/* - - - name: "[DEBUG] Check Wheel Contents" - run: check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: "Upload Build Artifact - ${{ inputs.version_number }}" - run: | - twine upload --repository codeartifact dist/* + with: + package_test_command: "${{ inputs.package_test_command }}" + dbms_name: "redshift" + ref: "${{ inputs.ref }}" + # Always skip tests since we run them above + skip_tests: true - version_file="$(echo "dbt/adapters/redshift/__version__.py")" - version="$(grep 'version =' "${version_file}" | cut -d '"' -f2)" - message="-- Success -- released ${version}" - echo "::notice $NOTIFICATION_PREFIX::$message" + secrets: "inherit" From 3e8860a7fa4ac0d754325b37ff705bf03230e610 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 21 Jun 2024 13:56:46 -0400 Subject: [PATCH 47/61] remove calculated version for dbt-postgres dependency (#850) --- setup.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/setup.py b/setup.py index c2649e66a..036ac30fb 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import re import sys if sys.version_info < (3, 8): @@ -37,16 +36,6 @@ def _plugin_version() -> str: return attributes["version"] -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - setup( name="dbt-redshift", version=_plugin_version(), @@ -61,7 +50,7 @@ def _plugin_version_trim() -> str: install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", + "dbt-postgres>=1.8,<1.10", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.1.1,>=2.0.913,!=2.0.914", From e757692ca3ba02e30fd7b519f398b1ef2c872378 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:09:56 -0400 Subject: [PATCH 48/61] Base 207/add test (#851) * Add aws cli setup to the internal PyPI release pipeline (#829) * vendor `internal-archive-release.yml` from `dbt-release`; remove unnecessary conditionals given the scope is now dbt-redshift * add aws cli setup for integration tests * replace secrets generation with secrets from existing integration test workflow * add breakout between flaky and non-flaky tests * Add test for upstream change. --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Mila Page Co-authored-by: Mike Alfare --- tests/functional/adapter/dbt_show/test_dbt_show.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index 808a7733c..83cb399ca 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -1,4 +1,8 @@ -from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowSqlHeader, + BaseShowLimit, + BaseShowDoesNotHandleDoubleLimit, +) class TestRedshiftShowLimit(BaseShowLimit): @@ -7,3 +11,7 @@ class TestRedshiftShowLimit(BaseShowLimit): class TestRedshiftShowSqlHeader(BaseShowSqlHeader): pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass From b510298d752711455997c9050d03d8667fb29539 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Fri, 12 Jul 2024 16:14:02 -0700 Subject: [PATCH 49/61] Adap 821/unit test infer wrong datatype (#852) * Add test to confirm upstream handling * Add changelog. * Add another test. * standardize names to reflect python objects. * revert dev requirements. --------- Co-authored-by: Mila Page --- .../unreleased/Fixes-20240625-170324.yaml | 7 ++ .../adapter/unit_testing/fixtures.py | 73 +++++++++++++++++++ .../adapter/unit_testing/test_unit_testing.py | 53 ++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240625-170324.yaml create mode 100644 tests/functional/adapter/unit_testing/fixtures.py diff --git a/.changes/unreleased/Fixes-20240625-170324.yaml b/.changes/unreleased/Fixes-20240625-170324.yaml new file mode 100644 index 000000000..316a92b95 --- /dev/null +++ b/.changes/unreleased/Fixes-20240625-170324.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: 'Handle unit test fixtures where typing goes wrong from first value in column + being Null. ' +time: 2024-06-25T17:03:24.73937-07:00 +custom: + Author: versusfacit + Issue: "821" diff --git a/tests/functional/adapter/unit_testing/fixtures.py b/tests/functional/adapter/unit_testing/fixtures.py new file mode 100644 index 000000000..36212dff3 --- /dev/null +++ b/tests/functional/adapter/unit_testing/fixtures.py @@ -0,0 +1,73 @@ +model_none_value_base = """ +{{ config(materialized="table") }} + +select 1 as id, 'a' as col1 +""" + +model_none_value_model = """ +{{config(materialized="table")}} + +select * from {{ ref('none_value_base') }} +""" + + +test_none_column_value_doesnt_throw_error_csv = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + format: csv + rows: | + id,col1 + ,d + ,e + 6,f + + expect: + format: csv + rows: | + id,col1 + ,d + ,e + 6,f +""" + +test_none_column_value_doesnt_throw_error_dct = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": "f"} + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": "f"} +""" + +test_none_column_value_will_throw_error = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": } + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": } +""" diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 4d89c4b08..f54a22a88 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,7 +1,19 @@ import pytest + +from dbt.exceptions import ParsingError +from dbt.tests.fixtures.project import write_project_files +from dbt.tests.util import run_dbt + from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +from tests.functional.adapter.unit_testing.fixtures import ( + model_none_value_base, + model_none_value_model, + test_none_column_value_doesnt_throw_error_csv, + test_none_column_value_doesnt_throw_error_dct, + test_none_column_value_will_throw_error, +) class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): @@ -34,6 +46,47 @@ def data_types(self): ] +class RedshiftUnitTestingNone: + def test_nones_handled_dict(self, project): + run_dbt(["build"]) + + +class TestRedshiftUnitTestCsvNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_csv, + } + + +class TestRedshiftUnitTestDictNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_dct, + } + + +class TestRedshiftUnitTestingTooManyNonesFails: + @pytest.fixture(scope="class") + def models(self): + return { + "__properties.yml": test_none_column_value_will_throw_error, + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + } + + def test_invalid_input(self, project): + with pytest.raises(ParsingError) as e: + run_dbt(["build"]) + + assert "Unit Test fixtures require at least one row free of None" in str(e) + + class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass From f18447874210d7afa32a6f646c045333ed99e742 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Wed, 17 Jul 2024 11:54:48 -0700 Subject: [PATCH 50/61] Revert "Adap 821/unit test infer wrong datatype (#852)" (#860) This reverts commit b510298d752711455997c9050d03d8667fb29539. --- .../unreleased/Fixes-20240625-170324.yaml | 7 -- .../adapter/unit_testing/fixtures.py | 73 ------------------- .../adapter/unit_testing/test_unit_testing.py | 53 -------------- 3 files changed, 133 deletions(-) delete mode 100644 .changes/unreleased/Fixes-20240625-170324.yaml delete mode 100644 tests/functional/adapter/unit_testing/fixtures.py diff --git a/.changes/unreleased/Fixes-20240625-170324.yaml b/.changes/unreleased/Fixes-20240625-170324.yaml deleted file mode 100644 index 316a92b95..000000000 --- a/.changes/unreleased/Fixes-20240625-170324.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: 'Handle unit test fixtures where typing goes wrong from first value in column - being Null. ' -time: 2024-06-25T17:03:24.73937-07:00 -custom: - Author: versusfacit - Issue: "821" diff --git a/tests/functional/adapter/unit_testing/fixtures.py b/tests/functional/adapter/unit_testing/fixtures.py deleted file mode 100644 index 36212dff3..000000000 --- a/tests/functional/adapter/unit_testing/fixtures.py +++ /dev/null @@ -1,73 +0,0 @@ -model_none_value_base = """ -{{ config(materialized="table") }} - -select 1 as id, 'a' as col1 -""" - -model_none_value_model = """ -{{config(materialized="table")}} - -select * from {{ ref('none_value_base') }} -""" - - -test_none_column_value_doesnt_throw_error_csv = """ -unit_tests: - - name: test_simple - - model: none_value_model - given: - - input: ref('none_value_base') - format: csv - rows: | - id,col1 - ,d - ,e - 6,f - - expect: - format: csv - rows: | - id,col1 - ,d - ,e - 6,f -""" - -test_none_column_value_doesnt_throw_error_dct = """ -unit_tests: - - name: test_simple - - model: none_value_model - given: - - input: ref('none_value_base') - rows: - - { "id": , "col1": "d"} - - { "id": , "col1": "e"} - - { "id": 6, "col1": "f"} - - expect: - rows: - - {id: , "col1": "d"} - - {id: , "col1": "e"} - - {id: 6, "col1": "f"} -""" - -test_none_column_value_will_throw_error = """ -unit_tests: - - name: test_simple - - model: none_value_model - given: - - input: ref('none_value_base') - rows: - - { "id": , "col1": "d"} - - { "id": , "col1": "e"} - - { "id": 6, "col1": } - - expect: - rows: - - {id: , "col1": "d"} - - {id: , "col1": "e"} - - {id: 6, "col1": } -""" diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index f54a22a88..4d89c4b08 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,19 +1,7 @@ import pytest - -from dbt.exceptions import ParsingError -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import run_dbt - from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput -from tests.functional.adapter.unit_testing.fixtures import ( - model_none_value_base, - model_none_value_model, - test_none_column_value_doesnt_throw_error_csv, - test_none_column_value_doesnt_throw_error_dct, - test_none_column_value_will_throw_error, -) class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): @@ -46,47 +34,6 @@ def data_types(self): ] -class RedshiftUnitTestingNone: - def test_nones_handled_dict(self, project): - run_dbt(["build"]) - - -class TestRedshiftUnitTestCsvNone(RedshiftUnitTestingNone): - @pytest.fixture(scope="class") - def models(self): - return { - "none_value_base.sql": model_none_value_base, - "none_value_model.sql": model_none_value_model, - "__properties.yml": test_none_column_value_doesnt_throw_error_csv, - } - - -class TestRedshiftUnitTestDictNone(RedshiftUnitTestingNone): - @pytest.fixture(scope="class") - def models(self): - return { - "none_value_base.sql": model_none_value_base, - "none_value_model.sql": model_none_value_model, - "__properties.yml": test_none_column_value_doesnt_throw_error_dct, - } - - -class TestRedshiftUnitTestingTooManyNonesFails: - @pytest.fixture(scope="class") - def models(self): - return { - "__properties.yml": test_none_column_value_will_throw_error, - "none_value_base.sql": model_none_value_base, - "none_value_model.sql": model_none_value_model, - } - - def test_invalid_input(self, project): - with pytest.raises(ParsingError) as e: - run_dbt(["build"]) - - assert "Unit Test fixtures require at least one row free of None" in str(e) - - class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass From 4f1f7c88cbc2cd05170966104139c65a307aee48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 00:05:13 +0000 Subject: [PATCH 51/61] Update freezegun requirement from ~=1.3 to ~=1.4 (#695) * Update freezegun requirement from ~=1.3 to ~=1.4 Updates the requirements on [freezegun](https://github.com/spulec/freezegun) to permit the latest version. - [Release notes](https://github.com/spulec/freezegun/releases) - [Changelog](https://github.com/spulec/freezegun/blob/master/CHANGELOG) - [Commits](https://github.com/spulec/freezegun/compare/1.3.0...1.4.0) --- updated-dependencies: - dependency-name: freezegun dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20231219-192336.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20231219-192336.yaml diff --git a/.changes/unreleased/Dependencies-20231219-192336.yaml b/.changes/unreleased/Dependencies-20231219-192336.yaml new file mode 100644 index 000000000..1d2d4bb51 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231219-192336.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update freezegun requirement from ~=1.3 to ~=1.4" +time: 2023-12-19T19:23:36.00000Z +custom: + Author: dependabot[bot] + PR: 695 diff --git a/dev-requirements.txt b/dev-requirements.txt index d02863ae0..ce327ab86 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -12,7 +12,7 @@ pre-commit==3.5.0;python_version <"3.9" # test ddtrace==2.3.0 -freezegun~=1.3 +freezegun~=1.4 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 From 18e2271f692a75c35ba229ccb0820e77c600e251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 03:59:55 +0000 Subject: [PATCH 52/61] Bump actions/checkout from 3 to 4 (#802) * Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240429-192949.yaml | 6 ++++++ .github/workflows/integration.yml | 4 ++-- .github/workflows/main.yml | 6 +++--- .github/workflows/nightly-release.yml | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Dependencies-20240429-192949.yaml diff --git a/.changes/unreleased/Dependencies-20240429-192949.yaml b/.changes/unreleased/Dependencies-20240429-192949.yaml new file mode 100644 index 000000000..861eed359 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240429-192949.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump actions/checkout from 3 to 4" +time: 2024-04-29T19:29:49.00000Z +custom: + Author: dependabot[bot] + PR: 802 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 429d627fc..b05fcb00d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -90,7 +90,7 @@ jobs: steps: - name: Check out the repository if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -98,7 +98,7 @@ jobs: # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c49b57385..e21d453fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -78,7 +78,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -120,7 +120,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 3c36361d5..e7d1da6ec 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -40,7 +40,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} From 47fce6aaa2391fbc521955175a75ba59b50992d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:14:19 +0000 Subject: [PATCH 53/61] Bump dbt-labs/actions from 1.1.0 to 1.1.1 (#762) * Bump dbt-labs/actions from 1.1.0 to 1.1.1 Bumps [dbt-labs/actions](https://github.com/dbt-labs/actions) from 1.1.0 to 1.1.1. - [Commits](https://github.com/dbt-labs/actions/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: dbt-labs/actions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240412-155445.yaml | 6 ++++++ .github/workflows/nightly-release.yml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Dependencies-20240412-155445.yaml diff --git a/.changes/unreleased/Dependencies-20240412-155445.yaml b/.changes/unreleased/Dependencies-20240412-155445.yaml new file mode 100644 index 000000000..c6c90f0f9 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240412-155445.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump dbt-labs/actions from 1.1.0 to 1.1.1" +time: 2024-04-12T15:54:45.00000Z +custom: + Author: dependabot[bot] + PR: 762 diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index e7d1da6ec..b38929417 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -58,7 +58,7 @@ jobs: - name: "Audit Version And Parse Into Parts" id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ steps.version-number-sources.outputs.current_version }} @@ -80,7 +80,7 @@ jobs: echo "number=$number" >> $GITHUB_OUTPUT - name: "Audit Nightly Release Version And Parse Into Parts" - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ steps.nightly-release-version.outputs.number }} From 19f9e56ad2f5452e532edfc923e8dfcaa483b201 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 19:56:36 +0000 Subject: [PATCH 54/61] Update tox requirement from ~=4.11 to ~=4.16 (#870) * Update tox requirement from ~=4.11 to ~=4.16 Updates the requirements on [tox](https://github.com/tox-dev/tox) to permit the latest version. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.0...4.16.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240718-191741.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240718-191741.yaml diff --git a/.changes/unreleased/Dependencies-20240718-191741.yaml b/.changes/unreleased/Dependencies-20240718-191741.yaml new file mode 100644 index 000000000..457737507 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240718-191741.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update tox requirement from ~=4.11 to ~=4.16" +time: 2024-07-18T19:17:41.00000Z +custom: + Author: dependabot[bot] + PR: 870 diff --git a/dev-requirements.txt b/dev-requirements.txt index ce327ab86..7ea647491 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -18,7 +18,7 @@ pytest-csv~=3.0 pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 -tox~=4.11 +tox~=4.16 # build bumpversion~=0.6.0 From 6674d55080a5f82a92f2f826f3b3148aa4659632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 22:26:54 +0000 Subject: [PATCH 55/61] Update twine requirement from ~=4.0 to ~=5.1 (#876) * Update twine requirement from ~=4.0 to ~=5.1 Updates the requirements on [twine](https://github.com/pypa/twine) to permit the latest version. - [Release notes](https://github.com/pypa/twine/releases) - [Changelog](https://github.com/pypa/twine/blob/main/docs/changelog.rst) - [Commits](https://github.com/pypa/twine/compare/4.0.0...v5.1.1) --- updated-dependencies: - dependency-name: twine dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot --- .changes/unreleased/Dependencies-20240719-195946.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240719-195946.yaml diff --git a/.changes/unreleased/Dependencies-20240719-195946.yaml b/.changes/unreleased/Dependencies-20240719-195946.yaml new file mode 100644 index 000000000..45a061646 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240719-195946.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update twine requirement from ~=4.0 to ~=5.1" +time: 2024-07-19T19:59:46.00000Z +custom: + Author: dependabot[bot] + PR: 876 diff --git a/dev-requirements.txt b/dev-requirements.txt index 7ea647491..09c6e3a42 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -22,5 +22,5 @@ tox~=4.16 # build bumpversion~=0.6.0 -twine~=4.0 +twine~=5.1 wheel~=0.42 From 725b45e4343ca189bc0338477ffdd4dd07f26cdb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jul 2024 00:13:12 +0000 Subject: [PATCH 56/61] Bump pre-commit from 3.7.0 to 3.7.1 (#867) * Bump pre-commit from 3.7.0 to 3.7.1 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.7.0 to 3.7.1. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.7.0...v3.7.1) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR * loosen pin on pre-commit --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Dependencies-20240718-191611.yaml | 6 ++++++ dev-requirements.txt | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Dependencies-20240718-191611.yaml diff --git a/.changes/unreleased/Dependencies-20240718-191611.yaml b/.changes/unreleased/Dependencies-20240718-191611.yaml new file mode 100644 index 000000000..bdf87eecd --- /dev/null +++ b/.changes/unreleased/Dependencies-20240718-191611.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump pre-commit from 3.7.0 to 3.7.1" +time: 2024-07-18T19:16:11.00000Z +custom: + Author: dependabot[bot] + PR: 867 diff --git a/dev-requirements.txt b/dev-requirements.txt index 09c6e3a42..d7e570f92 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,8 +7,8 @@ git+https://github.com/dbt-labs/dbt-postgres.git # dev ipdb~=0.13.13 -pre-commit==3.7.0;python_version >="3.9" -pre-commit==3.5.0;python_version <"3.9" +pre-commit~=3.7.0;python_version >="3.9" +pre-commit~=3.5.0;python_version <"3.9" # test ddtrace==2.3.0 From f261d17d62f7fd435879d001506095a9dee5f504 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 23 Jul 2024 13:20:51 -0400 Subject: [PATCH 57/61] Remove `freezegun` as a test dependency since it is not used (#875) * remove `freezegun` as a test dependency since it is not used --- .changes/unreleased/Under the Hood-20240719-133151.yaml | 6 ++++++ dev-requirements.txt | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20240719-133151.yaml diff --git a/.changes/unreleased/Under the Hood-20240719-133151.yaml b/.changes/unreleased/Under the Hood-20240719-133151.yaml new file mode 100644 index 000000000..b67ac6cea --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240719-133151.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Remove `freezegun` as a testing dependency; this package is no longer used +time: 2024-07-19T13:31:51.503575-04:00 +custom: + Author: mikealfare + Issue: "1136" diff --git a/dev-requirements.txt b/dev-requirements.txt index d7e570f92..c82ea37ca 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -12,7 +12,6 @@ pre-commit~=3.5.0;python_version <"3.9" # test ddtrace==2.3.0 -freezegun~=1.4 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 From a1f33db1806a24993df071859497f918d2c6767e Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:21:25 -0700 Subject: [PATCH 58/61] Adap 821/unit tests infer wrong datatype for None values in fixtures (#885) * Add test to confirm upstream handling * Add changelog. * Add another test. * standardize names to reflect python objects. * revert dev requirements. * Add the macro for the row checking. * Test for user facing exception thrown in base adapter with new redshift macor. * revert the dev requirements --------- Co-authored-by: Mila Page Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20240625-170324.yaml | 7 ++ .../redshift/macros/adapters/unit_testing.sql | 11 +++ .../adapter/unit_testing/fixtures.py | 73 +++++++++++++++++++ .../adapter/unit_testing/test_unit_testing.py | 64 ++++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240625-170324.yaml create mode 100644 dbt/include/redshift/macros/adapters/unit_testing.sql create mode 100644 tests/functional/adapter/unit_testing/fixtures.py diff --git a/.changes/unreleased/Fixes-20240625-170324.yaml b/.changes/unreleased/Fixes-20240625-170324.yaml new file mode 100644 index 000000000..316a92b95 --- /dev/null +++ b/.changes/unreleased/Fixes-20240625-170324.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: 'Handle unit test fixtures where typing goes wrong from first value in column + being Null. ' +time: 2024-06-25T17:03:24.73937-07:00 +custom: + Author: versusfacit + Issue: "821" diff --git a/dbt/include/redshift/macros/adapters/unit_testing.sql b/dbt/include/redshift/macros/adapters/unit_testing.sql new file mode 100644 index 000000000..5463f4e2b --- /dev/null +++ b/dbt/include/redshift/macros/adapters/unit_testing.sql @@ -0,0 +1,11 @@ +{%- macro redshift__validate_fixture_rows(rows, row_number) -%} + {%- if rows is not none and rows|length > 0 -%} + {%- set row = rows[0] -%} + {%- for key, value in row.items() -%} + {%- if value is none -%} + {%- set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") -%} + {{ exceptions.raise_compiler_error("Unit test fixture " ~ fixture_name ~ " in " ~ model.name ~ " does not have any row free of null values, which may cause type mismatch errors during unit test execution.") }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} diff --git a/tests/functional/adapter/unit_testing/fixtures.py b/tests/functional/adapter/unit_testing/fixtures.py new file mode 100644 index 000000000..36212dff3 --- /dev/null +++ b/tests/functional/adapter/unit_testing/fixtures.py @@ -0,0 +1,73 @@ +model_none_value_base = """ +{{ config(materialized="table") }} + +select 1 as id, 'a' as col1 +""" + +model_none_value_model = """ +{{config(materialized="table")}} + +select * from {{ ref('none_value_base') }} +""" + + +test_none_column_value_doesnt_throw_error_csv = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + format: csv + rows: | + id,col1 + ,d + ,e + 6,f + + expect: + format: csv + rows: | + id,col1 + ,d + ,e + 6,f +""" + +test_none_column_value_doesnt_throw_error_dct = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": "f"} + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": "f"} +""" + +test_none_column_value_will_throw_error = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": } + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": } +""" diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 4d89c4b08..27ed54cb6 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,7 +1,21 @@ import pytest + +from dbt.artifacts.schemas.results import RunStatus +from dbt.tests.fixtures.project import write_project_files +from dbt.tests.util import run_dbt + from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +from tests.functional.adapter.unit_testing.fixtures import ( + model_none_value_base, + model_none_value_model, + test_none_column_value_doesnt_throw_error_csv, + test_none_column_value_doesnt_throw_error_dct, + test_none_column_value_will_throw_error, +) + +from dbt_common.exceptions import CompilationError class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): @@ -34,6 +48,56 @@ def data_types(self): ] +class RedshiftUnitTestingNone: + def test_nones_handled_dict(self, project): + run_dbt(["build"]) + + +class TestRedshiftUnitTestCsvNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_csv, + } + + +class TestRedshiftUnitTestDictNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_dct, + } + + +class TestRedshiftUnitTestingTooManyNonesFails: + @pytest.fixture(scope="class") + def models(self): + return { + "__properties.yml": test_none_column_value_will_throw_error, + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + } + + def test_invalid_input(self, project): + """This is a user-facing exception, so we can't pytest.raise(CompilationError)""" + + def _find_first_error(items): + return next((item for item in items if item.status == RunStatus.Error), None) + + run_result = run_dbt(["build"], expect_pass=False) + first_item = _find_first_error(run_result) + + assert first_item is not None + assert ( + "does not have any row free of null values, which may cause type mismatch errors during unit test execution" + in str(first_item.message) + ) + + class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass From 5c6c2bbae197a021c779a5c3a6a5d237c93cb867 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 25 Jul 2024 06:09:23 -0700 Subject: [PATCH 59/61] Add support for 3.12 in workflow files and configs. (#886) Co-authored-by: Mila Page --- .github/workflows/integration.yml | 4 +++- .github/workflows/main.yml | 4 ++-- .pre-commit-config.yaml | 1 + setup.py | 1 + tox.ini | 6 +++--- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b05fcb00d..5a320e258 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -76,6 +76,8 @@ jobs: python-version: "3.10" - os: ubuntu-22.04 python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" env: TOXENV: integration-redshift @@ -201,7 +203,7 @@ jobs: fail-fast: false max-parallel: 1 matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] env: TOXENV: integration-redshift diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e21d453fe..5acb4f7b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -174,7 +174,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04, macos-12, windows-2022] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae249943d..a46d0f050 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,7 @@ repos: - --target-version=py39 - --target-version=py310 - --target-version=py311 + - --target-version=py312 - repo: https://github.com/pycqa/flake8 rev: 7.0.0 diff --git a/setup.py b/setup.py index 036ac30fb..712da3870 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ def _plugin_version() -> str: "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], python_requires=">=3.8", ) diff --git a/tox.ini b/tox.ini index 462bc8f07..fafa867a3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311 +envlist = py38,py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py}] +[testenv:{unit,py38,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py}-{redshift}] +[testenv:{integration,py38,py39,py310,py311,py312,py}-{redshift}] description = adapter plugin integration testing skip_install = true passenv = From b18b5556a64524c452df9ef10874763ff82dcdf8 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 25 Jul 2024 21:42:08 -0400 Subject: [PATCH 60/61] make dependabot less aggressive (#878) --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ae2be43aa..746dcae22 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,13 +5,25 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "docker" directory: "/docker" schedule: interval: "weekly" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch From 38fd49a2c8abacae285c3eb4102a11444cd9fd9a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 31 Jul 2024 21:54:08 -0400 Subject: [PATCH 61/61] Integration workflow update to support all-in-one adapter testing (#889) --- .github/scripts/update_dbt_core_branch.sh | 20 ----- .../scripts/update_dev_dependency_branches.sh | 21 +++++ .github/workflows/integration.yml | 80 ++++++++++++++----- dev-requirements.txt | 2 +- 4 files changed, 84 insertions(+), 39 deletions(-) delete mode 100755 .github/scripts/update_dbt_core_branch.sh create mode 100755 .github/scripts/update_dev_dependency_branches.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh new file mode 100755 index 000000000..022df6a8a --- /dev/null +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +set -e + + +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file +else + sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5a320e258..4d6bbd8d9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,8 @@ name: Adapter Integration Tests +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" + on: # pushes to release branches push: @@ -34,10 +36,31 @@ on: # manual trigger workflow_dispatch: inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" + name: + description: "Name to associate with run (example: 'dbt-adapters-242')" + required: false + type: string + default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: "main" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string required: false + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" type: string + required: false + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -90,13 +113,20 @@ jobs: DD_SERVICE: ${{ github.event.repository.name }} steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' + - name: Check out the repository (push) + if: github.event_name == 'push' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false + ref: ${{ inputs.adapter_branch }} - # explicity checkout the branch for the PR, + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -110,6 +140,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: github.event_name == 'workflow_dispatch' + run: | + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + - name: Install python dependencies run: | python -m pip install --user --upgrade pip @@ -117,12 +156,6 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Create AWS IAM profiles run: | aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID @@ -217,13 +250,20 @@ jobs: DD_SERVICE: ${{ github.event.repository.name }} steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' + - name: Check out the repository (push) + if: github.event_name == 'push' uses: actions/checkout@v3 with: persist-credentials: false - # explicity checkout the branch for the PR, + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ inputs.adapter_branch }} + + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -244,11 +284,15 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: github.event_name == 'workflow_dispatch' run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + - name: Run tox (redshift) env: diff --git a/dev-requirements.txt b/dev-requirements.txt index c82ea37ca..52c26c936 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # install latest changes in dbt-core + dbt-postgres +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core git+https://github.com/dbt-labs/dbt-postgres.git # dev