diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f54032556..5baa94d26 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0a1 +current_version = 1.9.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/unreleased/Dependencies-20230912-120620.yaml b/.changes/unreleased/Dependencies-20230912-120620.yaml deleted file mode 100644 index be139237d..000000000 --- a/.changes/unreleased/Dependencies-20230912-120620.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 -time: 2023-09-12T12:06:20.401643-07:00 -custom: - Author: soksamnanglim - PR: "601" diff --git a/.changes/unreleased/Dependencies-20231002-164037.yaml b/.changes/unreleased/Dependencies-20231002-164037.yaml deleted file mode 100644 index 54f51a402..000000000 --- a/.changes/unreleased/Dependencies-20231002-164037.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=1.19 to ~=1.20" -time: 2023-10-02T16:40:37.00000Z -custom: - Author: dependabot[bot] - PR: 622 diff --git a/.changes/unreleased/Dependencies-20231009-192801.yaml b/.changes/unreleased/Dependencies-20231009-192801.yaml deleted file mode 100644 index 2a53a074f..000000000 --- a/.changes/unreleased/Dependencies-20231009-192801.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit-hooks requirement from ~=4.4 to ~=4.5" -time: 2023-10-09T19:28:01.00000Z -custom: - Author: dependabot[bot] - PR: 627 diff --git a/.changes/unreleased/Dependencies-20231010-195348.yaml b/.changes/unreleased/Dependencies-20231010-195348.yaml deleted file mode 100644 index 4e8af02c1..000000000 --- a/.changes/unreleased/Dependencies-20231010-195348.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.5.1 to 1.6.0" -time: 2023-10-10T19:53:48.00000Z -custom: - Author: dependabot[bot] - PR: 629 diff --git a/.changes/unreleased/Dependencies-20231013-190517.yaml b/.changes/unreleased/Dependencies-20231013-190517.yaml deleted file mode 100644 index 8db17538a..000000000 --- a/.changes/unreleased/Dependencies-20231013-190517.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit requirement from ~=3.4 to ~=3.5" -time: 2023-10-13T19:05:17.00000Z -custom: - Author: dependabot[bot] - PR: 634 diff --git a/.changes/unreleased/Dependencies-20231017-191545.yaml b/.changes/unreleased/Dependencies-20231017-191545.yaml deleted file mode 100644 index 407615e32..000000000 --- a/.changes/unreleased/Dependencies-20231017-191545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.9 to ~=23.10" -time: 2023-10-17T19:15:45.00000Z -custom: - Author: dependabot[bot] - PR: 636 diff --git a/.changes/unreleased/Dependencies-20231027-173152.yaml b/.changes/unreleased/Dependencies-20231027-173152.yaml deleted file mode 100644 index 8ed464977..000000000 --- a/.changes/unreleased/Dependencies-20231027-173152.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.6.0 to 1.6.1" -time: 2023-10-27T17:31:52.00000Z -custom: - Author: dependabot[bot] - PR: 648 diff --git a/.changes/unreleased/Dependencies-20231030-193514.yaml b/.changes/unreleased/Dependencies-20231030-193514.yaml deleted file mode 100644 index e2771051b..000000000 --- a/.changes/unreleased/Dependencies-20231030-193514.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=1.20 to ~=2.1" -time: 2023-10-30T19:35:14.00000Z -custom: - Author: dependabot[bot] - PR: 651 diff --git a/.changes/unreleased/Dependencies-20231108-190800.yaml b/.changes/unreleased/Dependencies-20231108-190800.yaml deleted file mode 100644 index e8ce0c5b4..000000000 --- a/.changes/unreleased/Dependencies-20231108-190800.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.10 to ~=23.11" -time: 2023-11-08T19:08:00.00000Z -custom: - Author: dependabot[bot] - PR: 660 diff --git a/.changes/unreleased/Dependencies-20231110-192349.yaml b/.changes/unreleased/Dependencies-20231110-192349.yaml deleted file mode 100644 index 90b285065..000000000 --- a/.changes/unreleased/Dependencies-20231110-192349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.6.1 to 1.7.0" -time: 2023-11-10T19:23:49.00000Z -custom: - Author: dependabot[bot] - PR: 662 diff --git a/.changes/unreleased/Dependencies-20231113-195504.yaml b/.changes/unreleased/Dependencies-20231113-195504.yaml deleted file mode 100644 index 39b81e3e8..000000000 --- a/.changes/unreleased/Dependencies-20231113-195504.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.3 to ~=3.4" -time: 2023-11-13T19:55:04.00000Z -custom: - Author: dependabot[bot] - PR: 664 diff --git a/.changes/unreleased/Dependencies-20231116-194405.yaml b/.changes/unreleased/Dependencies-20231116-194405.yaml deleted file mode 100644 index 608ade10b..000000000 --- a/.changes/unreleased/Dependencies-20231116-194405.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=2.1 to ~=2.2" -time: 2023-11-16T19:44:05.00000Z -custom: - Author: dependabot[bot] - PR: 665 diff --git a/.changes/unreleased/Dependencies-20231127-201640.yaml b/.changes/unreleased/Dependencies-20231127-201640.yaml deleted file mode 100644 index bbcdb6ff9..000000000 --- a/.changes/unreleased/Dependencies-20231127-201640.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=2.2 to ~=2.3" -time: 2023-11-27T20:16:40.00000Z -custom: - Author: dependabot[bot] - PR: 669 diff --git a/.changes/unreleased/Dependencies-20231127-201942.yaml b/.changes/unreleased/Dependencies-20231127-201942.yaml deleted file mode 100644 index ac6c15d53..000000000 --- a/.changes/unreleased/Dependencies-20231127-201942.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.41 to ~=0.42" -time: 2023-11-27T20:19:42.00000Z -custom: - Author: dependabot[bot] - PR: 670 diff --git a/.changes/unreleased/Dependencies-20231128-194822.yaml b/.changes/unreleased/Dependencies-20231128-194822.yaml deleted file mode 100644 index 820ba25d1..000000000 --- a/.changes/unreleased/Dependencies-20231128-194822.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.4 to ~=3.5" -time: 2023-11-28T19:48:22.00000Z -custom: - Author: dependabot[bot] - PR: 672 diff --git a/.changes/unreleased/Dependencies-20231129-195044.yaml b/.changes/unreleased/Dependencies-20231129-195044.yaml deleted file mode 100644 index 235a89777..000000000 --- a/.changes/unreleased/Dependencies-20231129-195044.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.7.0 to 1.7.1" -time: 2023-11-29T19:50:44.00000Z -custom: - Author: dependabot[bot] - PR: 676 diff --git a/.changes/unreleased/Dependencies-20231130-044332.yaml b/.changes/unreleased/Dependencies-20231130-044332.yaml deleted file mode 100644 index 0fbeb9a18..000000000 --- a/.changes/unreleased/Dependencies-20231130-044332.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Remove direct boto3 dependency -time: 2023-11-30T04:43:32.872452+11:00 -custom: - Author: hexDoor - PR: "674" diff --git a/.changes/unreleased/Dependencies-20231204-193730.yaml b/.changes/unreleased/Dependencies-20231204-193730.yaml deleted file mode 100644 index b1d803288..000000000 --- a/.changes/unreleased/Dependencies-20231204-193730.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update freezegun requirement from ~=1.2 to ~=1.3" -time: 2023-12-04T19:37:30.00000Z -custom: - Author: dependabot[bot] - PR: 681 diff --git a/.changes/unreleased/Dependencies-20231212-195417.yaml b/.changes/unreleased/Dependencies-20231212-195417.yaml deleted file mode 100644 index 2117f6ead..000000000 --- a/.changes/unreleased/Dependencies-20231212-195417.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.11 to ~=23.12" -time: 2023-12-12T19:54:17.00000Z -custom: - Author: dependabot[bot] - PR: 688 diff --git a/.changes/unreleased/Dependencies-20231219-192336.yaml b/.changes/unreleased/Dependencies-20231219-192336.yaml new file mode 100644 index 000000000..1d2d4bb51 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231219-192336.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update freezegun requirement from ~=1.3 to ~=1.4" +time: 2023-12-19T19:23:36.00000Z +custom: + Author: dependabot[bot] + PR: 695 diff --git a/.changes/unreleased/Dependencies-20240118-095025.yaml b/.changes/unreleased/Dependencies-20240118-095025.yaml deleted file mode 100644 index ba61481ba..000000000 --- a/.changes/unreleased/Dependencies-20240118-095025.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: upgrade redshift driver to 2.0.918 -time: 2024-01-18T09:50:25.871857-08:00 -custom: - Author: colin-rogers-dbt - PR: "700" diff --git a/.changes/unreleased/Dependencies-20240124-111727.yaml b/.changes/unreleased/Dependencies-20240124-111727.yaml deleted file mode 100644 index f5874984a..000000000 --- a/.changes/unreleased/Dependencies-20240124-111727.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Migrate to dbt-common and dbt-adapters -time: 2024-01-24T11:17:27.580348-08:00 -custom: - Author: colin-rogers-dbt - PR: "706" diff --git a/.changes/unreleased/Dependencies-20240412-155445.yaml b/.changes/unreleased/Dependencies-20240412-155445.yaml new file mode 100644 index 000000000..c6c90f0f9 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240412-155445.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump dbt-labs/actions from 1.1.0 to 1.1.1" +time: 2024-04-12T15:54:45.00000Z +custom: + Author: dependabot[bot] + PR: 762 diff --git a/.changes/unreleased/Dependencies-20240429-192949.yaml b/.changes/unreleased/Dependencies-20240429-192949.yaml new file mode 100644 index 000000000..861eed359 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240429-192949.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump actions/checkout from 3 to 4" +time: 2024-04-29T19:29:49.00000Z +custom: + Author: dependabot[bot] + PR: 802 diff --git a/.changes/unreleased/Dependencies-20240718-191611.yaml b/.changes/unreleased/Dependencies-20240718-191611.yaml new file mode 100644 index 000000000..bdf87eecd --- /dev/null +++ b/.changes/unreleased/Dependencies-20240718-191611.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump pre-commit from 3.7.0 to 3.7.1" +time: 2024-07-18T19:16:11.00000Z +custom: + Author: dependabot[bot] + PR: 867 diff --git a/.changes/unreleased/Dependencies-20240718-191741.yaml b/.changes/unreleased/Dependencies-20240718-191741.yaml new file mode 100644 index 000000000..457737507 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240718-191741.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update tox requirement from ~=4.11 to ~=4.16" +time: 2024-07-18T19:17:41.00000Z +custom: + Author: dependabot[bot] + PR: 870 diff --git a/.changes/unreleased/Dependencies-20240719-195946.yaml b/.changes/unreleased/Dependencies-20240719-195946.yaml new file mode 100644 index 000000000..45a061646 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240719-195946.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update twine requirement from ~=4.0 to ~=5.1" +time: 2024-07-19T19:59:46.00000Z +custom: + Author: dependabot[bot] + PR: 876 diff --git a/.changes/unreleased/Features-20231030-101055.yaml b/.changes/unreleased/Features-20231030-101055.yaml deleted file mode 100644 index 8648762e2..000000000 --- a/.changes/unreleased/Features-20231030-101055.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: allow user to set debug level for redshift-connector via env var -time: 2023-10-30T10:10:55.976191-07:00 -custom: - Author: colin-rogers-dbt - Issue: "650" diff --git a/.changes/unreleased/Features-20240419-145208.yaml b/.changes/unreleased/Features-20240419-145208.yaml new file mode 100644 index 000000000..3a7a3468b --- /dev/null +++ b/.changes/unreleased/Features-20240419-145208.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Support IAM user auth via direct parameters, in addition to the existing profile + method +time: 2024-04-19T14:52:08.086607-04:00 +custom: + Author: mikealfare + Issue: "760" diff --git a/.changes/unreleased/Features-20240425-011440.yaml b/.changes/unreleased/Features-20240425-011440.yaml new file mode 100644 index 000000000..a8197dd6f --- /dev/null +++ b/.changes/unreleased/Features-20240425-011440.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for IAM Role auth +time: 2024-04-25T01:14:40.601575-04:00 +custom: + Author: mikealfare,abbywh + Issue: "623" diff --git a/.changes/unreleased/Features-20240430-185708.yaml b/.changes/unreleased/Features-20240430-185708.yaml new file mode 100644 index 000000000..59407c196 --- /dev/null +++ b/.changes/unreleased/Features-20240430-185708.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add tests for cross-database `cast` macro +time: 2024-04-30T18:57:08.04492-06:00 +custom: + Author: dbeatty10 + Issue: "804" diff --git a/.changes/unreleased/Features-20240501-151859.yaml b/.changes/unreleased/Features-20240501-151859.yaml new file mode 100644 index 000000000..e4791638c --- /dev/null +++ b/.changes/unreleased/Features-20240501-151859.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Cross-database `date` macro +time: 2024-05-01T15:18:59.292022-06:00 +custom: + Author: dbeatty10 + Issue: 808 diff --git a/.changes/unreleased/Fixes-20231025-203732.yaml b/.changes/unreleased/Fixes-20231025-203732.yaml deleted file mode 100644 index 9e6bf1af7..000000000 --- a/.changes/unreleased/Fixes-20231025-203732.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix parsing of database results for materialized view auto refresh -time: 2023-10-25T20:37:32.191259-04:00 -custom: - Author: mikealfare - Issue: "643" diff --git a/.changes/unreleased/Fixes-20231026-164623.yaml b/.changes/unreleased/Fixes-20231026-164623.yaml deleted file mode 100644 index f06eff381..000000000 --- a/.changes/unreleased/Fixes-20231026-164623.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix describe_materialized_view for Redshift Serverless -time: 2023-10-26T16:46:23.253837-06:00 -custom: - Author: reptillicus - Issue: "641" diff --git a/.changes/unreleased/Fixes-20231030-234315.yaml b/.changes/unreleased/Fixes-20231030-234315.yaml deleted file mode 100644 index 823e252dd..000000000 --- a/.changes/unreleased/Fixes-20231030-234315.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Catalog queries assign the appropriate type for materialized views -time: 2023-10-30T23:43:15.281772-04:00 -custom: - Author: mikealfare - Issue: "652" diff --git a/.changes/unreleased/Fixes-20231103-181357.yaml b/.changes/unreleased/Fixes-20231103-181357.yaml deleted file mode 100644 index 2995560d2..000000000 --- a/.changes/unreleased/Fixes-20231103-181357.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix handling of `backup` parameter during config change monitoring -time: 2023-11-03T18:13:57.401994-04:00 -custom: - Author: mikealfare - Issue: "621" diff --git a/.changes/unreleased/Fixes-20240206-132326.yaml b/.changes/unreleased/Fixes-20240206-132326.yaml deleted file mode 100644 index 45b3a74c8..000000000 --- a/.changes/unreleased/Fixes-20240206-132326.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes - external tables -time: 2024-02-06T13:23:26.061133-05:00 -custom: - Author: mikealfare - Issue: "710" diff --git a/.changes/unreleased/Fixes-20240423-131503.yaml b/.changes/unreleased/Fixes-20240423-131503.yaml new file mode 100644 index 000000000..8b5b82b38 --- /dev/null +++ b/.changes/unreleased/Fixes-20240423-131503.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Stop adding aliases to subqueries when calling with `--empty` +time: 2024-04-23T13:15:03.118968-07:00 +custom: + Author: colin-rogers-dbt + Issue: "782" diff --git a/.changes/unreleased/Fixes-20240531-113620.yaml b/.changes/unreleased/Fixes-20240531-113620.yaml new file mode 100644 index 000000000..a020ca045 --- /dev/null +++ b/.changes/unreleased/Fixes-20240531-113620.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support IAM Role authentication for Redshift Serverless +time: 2024-05-31T11:36:20.397521-07:00 +custom: + Author: fleid + Issue: "835" diff --git a/.changes/unreleased/Fixes-20240612-131752.yaml b/.changes/unreleased/Fixes-20240612-131752.yaml new file mode 100644 index 000000000..36ea4d922 --- /dev/null +++ b/.changes/unreleased/Fixes-20240612-131752.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: update pin range for redshift-connector to allow 2.1.0 +time: 2024-06-12T13:17:52.460407-07:00 +custom: + Author: colin-rogers-dbt VersusFacit + Issue: "844" diff --git a/.changes/unreleased/Fixes-20240625-170324.yaml b/.changes/unreleased/Fixes-20240625-170324.yaml new file mode 100644 index 000000000..316a92b95 --- /dev/null +++ b/.changes/unreleased/Fixes-20240625-170324.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: 'Handle unit test fixtures where typing goes wrong from first value in column + being Null. ' +time: 2024-06-25T17:03:24.73937-07:00 +custom: + Author: versusfacit + Issue: "821" diff --git a/.changes/unreleased/Under the Hood-20231119-132157.yaml b/.changes/unreleased/Under the Hood-20231119-132157.yaml deleted file mode 100644 index 760c08ccf..000000000 --- a/.changes/unreleased/Under the Hood-20231119-132157.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add tests for --empty flag -time: 2023-11-19T13:21:57.588501-05:00 -custom: - Author: michelleark - Issue: "667" diff --git a/.changes/unreleased/Under the Hood-20240102-152425.yaml b/.changes/unreleased/Under the Hood-20240102-152425.yaml deleted file mode 100644 index 23a3eeb46..000000000 --- a/.changes/unreleased/Under the Hood-20240102-152425.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update base adapter references as part of decoupling migration -time: 2024-01-02T15:24:25.890421-08:00 -custom: - Author: colin-rogers-dbt VersusFacit - Issue: "698" diff --git a/.changes/unreleased/Under the Hood-20240331-103115.yaml b/.changes/unreleased/Under the Hood-20240331-103115.yaml new file mode 100644 index 000000000..ffb5a6489 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240331-103115.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Lazy load agate +time: 2024-03-31T10:31:15.65006-04:00 +custom: + Author: dwreeves + Issue: "745" diff --git a/.changes/unreleased/Under the Hood-20240719-133151.yaml b/.changes/unreleased/Under the Hood-20240719-133151.yaml new file mode 100644 index 000000000..b67ac6cea --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240719-133151.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Remove `freezegun` as a testing dependency; this package is no longer used +time: 2024-07-19T13:31:51.503575-04:00 +custom: + Author: mikealfare + Issue: "1136" diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b08ffcd53..000000000 --- a/.flake8 +++ /dev/null @@ -1,16 +0,0 @@ -[flake8] -select = - E - W - F -ignore = - # makes Flake8 work like black - W503, - W504, - # makes Flake8 work like black - E203, - E741, - E501, -exclude = test -per-file-ignores = - */__init__.py: F401 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6283d123..02ed72d45 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # This codeowners file is used to ensure all PRs require reviews from the adapters team -* @dbt-labs/core-adapters +* @dbt-labs/adapters diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f34492..746dcae22 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,29 @@ version: 2 updates: - # python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index 7db445d9e..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,87 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; - const supportedAdapters = ["redshift"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-latest", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh new file mode 100755 index 000000000..022df6a8a --- /dev/null +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +set -e + + +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file +else + sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml index 00a098df8..f49cf517c 100644 --- a/.github/workflows/docs-issues.yml +++ b/.github/workflows/docs-issues.yml @@ -1,19 +1,18 @@ # **what?** -# Open an issue in docs.getdbt.com when a PR is labeled `user docs` +# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed # **why?** # To reduce barriers for keeping docs up to date # **when?** -# When a PR is labeled `user docs` and is merged. Runs on pull_request_target to run off the workflow already merged, -# not the workflow that existed on the PR branch. This allows old PRs to get comments. +# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. -name: Open issues in docs.getdbt.com repo when a PR is labeled -run-name: "Open an issue in docs.getdbt.com for PR #${{ github.event.pull_request.number }}" +name: Open issues in docs.getdbt.com repo when an issue is labeled +run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" on: - pull_request_target: + issues: types: [labeled, closed] defaults: @@ -21,23 +20,22 @@ defaults: shell: bash permissions: - issues: write # opens new issues - pull-requests: write # comments on PRs - + issues: write # comments on issues jobs: open_issues: - # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`. Otherwise it runs the + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we # decide if it should run or not. if: | - (github.event.pull_request.merged == true) && - ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) || + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || (github.event.action == 'labeled' && github.event.label.name == 'user docs')) uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main with: issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} PR #${{ github.event.pull_request.number }}" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." secrets: inherit diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9d1fe0807..4d6bbd8d9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,8 @@ name: Adapter Integration Tests +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" + on: # pushes to release branches push: @@ -34,10 +36,31 @@ on: # manual trigger workflow_dispatch: inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" + name: + description: "Name to associate with run (example: 'dbt-adapters-242')" + required: false + type: string + default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string required: false + default: "main" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" type: string + required: false + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" + type: string + required: false + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -53,85 +76,34 @@ defaults: shell: bash jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: + test: + name: redshift / python ${{ matrix.python-version }} / ${{ matrix.os }} + # run if not a PR from a forked repository or has a label to mark as safe to test if: >- github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) runs-on: ${{ matrix.os }} - needs: test-metadata - strategy: fail-fast: false max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + matrix: + os: [ubuntu-22.04, macos-12, windows-2022] + python-version: ["3.8"] + include: + - os: ubuntu-22.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" env: - TOXENV: integration-${{ matrix.adapter }} + TOXENV: integration-redshift PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true @@ -141,17 +113,24 @@ jobs: DD_SERVICE: ${{ github.event.repository.name }} steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 + - name: Check out the repository (push) + if: github.event_name == 'push' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 with: persist-credentials: false + ref: ${{ inputs.adapter_branch }} - # explicity checkout the branch for the PR, + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} @@ -161,6 +140,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: github.event_name == 'workflow_dispatch' + run: | + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + - name: Install python dependencies run: | python -m pip install --user --upgrade pip @@ -168,24 +156,50 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' + - name: Create AWS IAM profiles run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - name: Run tox (redshift) - if: matrix.adapter == 'redshift' env: REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace + run: tox -- -m "not flaky" --ddtrace - uses: actions/upload-artifact@v3 if: always() @@ -202,11 +216,104 @@ jobs: - uses: actions/upload-artifact@v3 if: always() with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv + name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv path: integration_results.csv + test-flaky: + name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky + + # run this after the norm integration tests to avoid collisions + needs: test + + # run if not a PR from a forked repository or has a label to mark as safe to test + if: >- + github.event_name != 'pull_request_target' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-22.04 + + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_INSTRUMENTATION_TELEMETRY_ENABLED: false + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + steps: + - name: Check out the repository (push) + if: github.event_name == 'push' + uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ inputs.adapter_branch }} + + # explicitly checkout the branch for the PR, + # this is necessary for the `pull_request_target` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: github.event_name == 'workflow_dispatch' + run: | + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + + + - name: Run tox (redshift) + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m flaky --ddtrace + require-label-comment: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: test @@ -229,7 +336,7 @@ jobs: check_for_duplicate_msg: true post-failure: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: test if: ${{ failure() }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a76df7e9e..5acb4f7b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -58,8 +58,6 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip --version pre-commit --version - mypy --version - dbt --version - name: pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -72,7 +70,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -80,7 +78,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -122,7 +120,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false @@ -175,8 +173,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] + os: [ubuntu-22.04, macos-12, windows-2022] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Set up Python ${{ matrix.python-version }} @@ -204,7 +202,7 @@ jobs: - name: Check wheel distributions run: | - dbt --version + python -c "import dbt.adapters.redshift" - name: Install source distributions run: | @@ -212,4 +210,4 @@ jobs: - name: Check source distributions run: | - dbt --version + python -c "import dbt.adapters.redshift" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 46db5b749..b38929417 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -20,6 +20,7 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write # this is the permission that allows Docker release defaults: run: @@ -39,7 +40,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} @@ -57,7 +58,7 @@ jobs: - name: "Audit Version And Parse Into Parts" id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ steps.version-number-sources.outputs.current_version }} @@ -79,7 +80,7 @@ jobs: echo "number=$number" >> $GITHUB_OUTPUT - name: "Audit Nightly Release Version And Parse Into Parts" - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ steps.nightly-release-version.outputs.number }} diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml new file mode 100644 index 000000000..30afe9111 --- /dev/null +++ b/.github/workflows/release-internal.yml @@ -0,0 +1,207 @@ +# What? +# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. +# +# How? +# After checking out and testing the provided ref, the image is built and uploaded. +# +# When? +# Manual trigger. +name: "Release to Cloud" +run-name: "Release to Cloud off of ${{ inputs.ref }}" + +on: + workflow_dispatch: + inputs: + ref: + description: "The ref (sha or branch name) to use" + type: string + default: "main" + required: true + package_test_command: + description: "Package test command" + type: string + default: "python -c \"import dbt.adapters.redshift\"" + required: true + +defaults: + run: + shell: "bash" + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Internal Archive Release]" + TEMP_PROFILE_NAME: "temp_aws_profile" + +jobs: + job-setup: + name: Job Setup + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Print Variables" + run: | + echo The release ref: ${{ inputs.ref }} + echo Package test command: ${{ inputs.package_test_command }} + + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + unit-tests: + name: 'Unit Tests' + runs-on: ubuntu-latest + needs: job-setup + env: + TOXENV: unit + + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + python -m tox --version + + - name: "Run Tests" + run: tox + + integration-tests: + name: 'Integration Tests' + runs-on: ubuntu-latest + needs: unit-tests + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n4" + DBT_INVOCATION_ENV: github-actions + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: "Create AWS IAM profiles" + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: "Run tox" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m "not flaky" + + integration-tests-flaky: + name: 'Integration Tests (flaky)' + runs-on: ubuntu-latest + needs: integration-tests + + env: + TOXENV: integration-redshift + PYTEST_ADDOPTS: "-v --color=yes -n1" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout provided ref, default to branch main" + uses: actions/checkout@v4 + with: + ref: "${{ inputs.ref }}" + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: "Run tox" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + run: tox -- -m flaky + + call-release-workflow: + needs: [unit-tests, integration-tests, integration-tests-flaky] + name: "Create cloud release" + uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" + + with: + package_test_command: "${{ inputs.package_test_command }}" + dbms_name: "redshift" + ref: "${{ inputs.ref }}" + # Always skip tests since we run them above + skip_tests: true + + secrets: "inherit" diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml new file mode 100644 index 000000000..5c5ff0cd7 --- /dev/null +++ b/.github/workflows/release-prep.yml @@ -0,0 +1,702 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# sha: The commit to attach to this release +# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# target_branch: The branch that we will release from +# env_setup_script_path: Path to the environment setup script +# test_run: Test run (The temp branch will be used for release) +# nightly_release: Identifier that this is nightly release +# +# Outputs: +# final_sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For test runs: prep-release/test-run/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +# + +name: Version Bump and Changelog Generation + +on: + workflow_call: + inputs: + sha: + required: true + type: string + version_number: + required: true + type: string + target_branch: + required: true + type: string + env_setup_script_path: + required: false + type: string + default: "" + test_run: + required: false + default: true + type: boolean + nightly_release: + type: boolean + default: false + required: false + outputs: + final_sha: + description: The new commit that includes the changelog and version bump. + value: ${{ jobs.determine-release-sha.outputs.final_sha }} + changelog_path: + description: The path to the changelog for this version + value: ${{ jobs.audit-changelog.outputs.changelog_path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Release Preparation]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + + steps: + - name: "[DEBUG] Print Variables" + run: | + # WORKFLOW INPUTS + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo The branch that we will release from: ${{ inputs.target_branch }} + echo Path to the environment setup script: ${{ inputs.env_setup_script_path }} + echo Test run: ${{ inputs.test_run }} + echo Nightly release: ${{ inputs.nightly_release }} + # ENVIRONMENT VARIABLES + echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + runs-on: ubuntu-latest + + outputs: + changelog_path: ${{ steps.set_path.outputs.changelog_path }} + exists: ${{ steps.set_existence.outputs.exists }} + base_version: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + - name: "Set Changelog Path" + id: set_path + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + # Send notification + echo "changelog_path=$path" >> $GITHUB_OUTPUT + title="Changelog path" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" + + - name: "Set Changelog Existence For Subsequent Jobs" + id: set_existence + run: | + does_exist=false + if test -f ${{ steps.set_path.outputs.changelog_path }} + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[Notification] Set Changelog Existence For Subsequent Jobs" + run: | + title="Changelog exists" + if [[ ${{ steps.set_existence.outputs.exists }} == true ]] + then + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" + else + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} + echo exists: ${{ steps.set_existence.outputs.exists }} + echo base_version: ${{ steps.semver.outputs.base-version }} + echo prerelease: ${{ steps.semver.outputs.pre-release }} + echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + audit-version-in-code: + runs-on: ubuntu-latest + + outputs: + up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Check Current Version In Code" + id: version-check + run: | + is_updated=false + if grep -Fxq "current_version = ${{ inputs.version_number }}" .bumpversion.cfg + then + is_updated=true + fi + echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT + + - name: "[Notification] Check Current Version In Code" + run: | + title="Version check" + if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] + then + message="The version in the codebase is equal to the provided version" + else + message="The version in the codebase differs from the provided version" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + skip-generate-changelog: + runs-on: ubuntu-latest + needs: [audit-changelog] + if: needs.audit-changelog.outputs.exists == 'true' + + steps: + - name: "Changelog Exists, Skip Generating New Changelog" + run: | + # Send notification + title="Skip changelog generation" + message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + runs-on: ubuntu-latest + needs: [audit-version-in-code] + if: needs.audit-version-in-code.outputs.up_to_date == 'true' + + steps: + - name: "Version Already Bumped" + run: | + # Send notification + title="Skip version bump" + message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + create-temp-branch: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code] + if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' + + outputs: + branch_name: ${{ steps.variables.outputs.branch_name }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Generate Branch Name" + id: variables + run: | + name="prep-release/" + if [[ ${{ inputs.nightly_release }} == true ]] + then + name+="nightly-release/" + elif [[ ${{ inputs.test_run }} == true ]] + then + name+="test-run/" + fi + name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" + echo "branch_name=$name" >> $GITHUB_OUTPUT + + - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" + run: | + git checkout -b ${{ steps.variables.outputs.branch_name }} + git push -u origin ${{ steps.variables.outputs.branch_name }} + + - name: "[Notification] Temp branch created" + run: | + # Send notification + title="Temp branch generated" + message="The ${{ steps.variables.outputs.branch_name }} branch created" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo branch_name ${{ steps.variables.outputs.branch_name }} + + generate-changelog-bump-version: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code, create-temp-branch] + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Add Homebrew To PATH" + run: | + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install Homebrew Packages" + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set json File Name" + id: json_file + run: | + echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get Core Team Membership" + run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core Team Membership for Changie Contributors exclusion" + id: set_team_membership + run: | + team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) + echo $team_list + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV + + - name: "Delete the json File" + run: | + rm ${{ steps.json_file.outputs.name }} + + - name: "Generate Release Changelog" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + git status + + - name: "Check Changelog Created Successfully" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + title="Changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + python3 -m venv env + source env/bin/activate + python -m pip install --upgrade pip + + - name: "Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + # note: bumpversion is no longer supported, it actually points to bump2version now + run: | + source env/bin/activate + if [ -f "editable-requirements.txt" ] + then + python -m pip install -r dev-requirements.txt -r editable-requirements.txt + else + python -m pip install -r dev-requirements.txt + fi + env/bin/bumpversion --allow-dirty --new-version ${{ inputs.version_number }} major + git status + + - name: "[Notification] Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + title="Version bump" + message="Version successfully bumped in codebase to ${{ inputs.version_number }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + # this step will fail on whitespace errors but also correct them + - name: "Remove Trailing Whitespace Via Pre-commit" + continue-on-error: true + run: | + pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + # this step will fail on newline errors but also correct them + - name: "Removing Extra Newlines Via Pre-commit" + continue-on-error: true + run: | + pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + - name: "Commit & Push Changes" + run: | + #Data for commit + user="Github Build Bot" + email="buildbot@fishtownanalytics.com" + commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" + #Commit changes to branch + git config user.name "$user" + git config user.email "$email" + git pull + git add . + git commit -m "$commit_message" + git push + + run-unit-tests: + runs-on: ubuntu-latest + needs: [create-temp-branch, generate-changelog-bump-version] + + env: + TOXENV: unit + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + python -m tox --version + + - name: "Run Tox" + run: tox + + run-integration-tests: + runs-on: ubuntu-22.04 + needs: [create-temp-branch, generate-changelog-bump-version] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m "not flaky" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + run-integration-tests-flaky: + runs-on: ubuntu-22.04 + needs: [run-integration-tests, create-temp-branch] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m flaky + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + merge-changes-into-target-branch: + runs-on: ubuntu-latest + needs: [run-unit-tests, run-integration-tests-flaky, create-temp-branch, audit-version-in-code, audit-changelog] + if: | + !failure() && !cancelled() && + inputs.test_run == false && + ( + needs.audit-changelog.outputs.exists == 'false' || + needs.audit-version-in-code.outputs.up_to_date == 'false' + ) + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} + echo inputs.test_run: ${{ inputs.test_run }} + echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} + echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Checkout Repo ${{ github.repository }}" + uses: actions/checkout@v4 + + - name: "Merge Changes Into ${{ inputs.target_branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} + target_branch: ${{ inputs.target_branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[Notification] Changes Merged into ${{ inputs.target_branch }}" + run: | + title="Changelog and Version Bump Branch Merge" + message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into ${{ inputs.target_branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + determine-release-sha: + runs-on: ubuntu-latest + needs: + [ + create-temp-branch, + merge-changes-into-target-branch, + audit-changelog, + audit-version-in-code, + ] + # always run this job, regardless of if the dependant jobs were skipped + if: ${{ !failure() && !cancelled() }} + + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} + echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} + echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Resolve Branch To Checkout" + id: resolve_branch + run: | + branch="" + if [[ ${{ inputs.test_run == true }} ]] + then + branch=${{ needs.create-temp-branch.outputs.branch_name }} + else + branch=${{ inputs.target_branch }} + fi + echo "target_branch=$branch" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Branch To Checkout" + run: | + title="Branch pick" + message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.resolve_branch.outputs.target_branch }} + + - name: "[Debug] Log Branch" + run: git status + + - name: "Resolve Commit SHA For Release" + id: resolve_commit_sha + run: | + commit_sha="" + if [[ ${{ needs.audit-changelog.outputs.exists }} == false ]] || [[ ${{ needs.audit-version-in-code.outputs.up_to_date }} == false ]] + then + commit_sha=$(git rev-parse HEAD) + else + commit_sha=${{ inputs.sha }} + fi + echo "release_sha=$commit_sha" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Commit SHA For Release" + run: | + title="Release commit pick" + message="The ${{ steps.resolve_commit_sha.outputs.release_sha }} commit will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" + if: ${{ inputs.test_run == false && needs.create-temp-branch.outputs.branch_name != '' }} + run: | + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "[Debug] Print Outputs" + run: | + echo release_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 88942e251..e01b961f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: @@ -48,7 +49,7 @@ on: package_test_command: description: "Package test command" type: string - default: "dbt --version" + default: "python -c \"import dbt.adapters.redshift\"" required: true test_run: description: "Test run (Publish release as draft)" @@ -60,6 +61,11 @@ on: type: boolean default: false required: false + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false workflow_call: inputs: sha: @@ -92,7 +98,7 @@ on: package_test_command: description: "Package test command" type: string - default: "dbt --version" + default: "python -c \"import dbt.adapters.redshift\"" required: true test_run: description: "Test run (Publish release as draft)" @@ -131,9 +137,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - + uses: ./.github/workflows/release-prep.yml with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} @@ -141,17 +145,13 @@ jobs: env_setup_script_path: ${{ inputs.env_setup_script_path }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -160,11 +160,9 @@ jobs: build-test-package: name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -174,19 +172,15 @@ jobs: package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} github-release: name: GitHub Release - if: ${{ !failure() && !cancelled() }} - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -195,34 +189,41 @@ jobs: pypi-release: name: PyPI Release - + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [github-release] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + docker-release: + name: "Docker Release" + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} + needs: [bump-version-generate-changelog, build-test-package, github-release] + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - bump-version-generate-changelog, - build-test-package, github-release, pypi-release, + docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d80b955c..a46d0f050 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,56 @@ -# For more on configuring pre-commit hooks (see https://pre-commit.com/) - -# Force all unspecified python hooks to run python 3.8 default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - id: black - alias: black-check - stages: [manual] - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - "--check" - - "--diff" -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - id: flake8 - alias: flake8-check - stages: [manual] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 - hooks: - - id: mypy - # N.B.: Mypy is... a bit fragile. - # - # By using `language: system` we run this hook in the local - # environment instead of a pre-commit isolated one. This is needed - # to ensure mypy correctly parses the project. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + - --target-version=py312 + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + additional_dependencies: [flaky] - # It may cause trouble in that it adds environmental variables out - # of our control to the mix. Unfortunately, there's nothing we can - # do about per pre-commit's author. - # See https://github.com/pre-commit/pre-commit/issues/730 for details. - args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters/.* - language: system - - id: mypy - alias: mypy-check - stages: [manual] - args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters - language: system +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + args: + - --show-error-codes + - --pretty + - --ignore-missing-imports + - --explicit-package-bases + files: ^dbt/adapters + additional_dependencies: + - types-pytz + - types-requests diff --git a/CHANGELOG.md b/CHANGELOG.md index 04a8e7db1..218a3de96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ## Previous Releases For information on prior major and minor releases, see their changelogs: +- [1.8](https://github.com/dbt-labs/dbt-redshift/blob/1.8.latest/CHANGELOG.md) +- [1.7](https://github.com/dbt-labs/dbt-redshift/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md) diff --git a/Makefile b/Makefile index efd23b806..f32c3ba8f 100644 --- a/Makefile +++ b/Makefile @@ -65,3 +65,13 @@ help: ## Show this help message. @echo @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + + +.PHONY: docker-dev +docker-dev: + docker build -f docker/dev.Dockerfile -t dbt-redshift-dev . + docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker/Dockerfile -t dbt-redshift . diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index f15b401d1..6698ed64c 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0a1" +version = "1.9.0a1" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index b0fc0825d..d3fbcafea 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -1,14 +1,12 @@ import re from multiprocessing import Lock from contextlib import contextmanager -from typing import Tuple, Union, Optional, List +from typing import Any, Callable, Dict, Tuple, Union, Optional, List, TYPE_CHECKING from dataclasses import dataclass, field -import agate import sqlparse import redshift_connector from dbt.adapters.exceptions import FailedToConnectError -from dbt_common.clients import agate_helper from redshift_connector.utils.oids import get_datatype_name from dbt.adapters.sql import SQLConnectionManager @@ -19,6 +17,11 @@ from dbt_common.helper_types import Port from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError +if TYPE_CHECKING: + # Indirectly imported via agate_helper, which is lazy loaded further downfile. + # Used by mypy for earlier type hints. + import agate + class SSLConfigError(CompilationError): def __init__(self, exc: ValidationError): @@ -37,6 +40,7 @@ def get_message(self) -> str: class RedshiftConnectionMethod(StrEnum): DATABASE = "database" IAM = "iam" + IAM_ROLE = "iam_role" class UserSSLMode(StrEnum): @@ -50,7 +54,7 @@ class UserSSLMode(StrEnum): @classmethod def default(cls) -> "UserSSLMode": # default for `psycopg2`, which aligns with dbt-redshift 1.4 and provides backwards compatibility - return cls.prefer + return cls("prefer") class RedshiftSSLMode(StrEnum): @@ -60,11 +64,11 @@ class RedshiftSSLMode(StrEnum): SSL_MODE_TRANSLATION = { UserSSLMode.disable: None, - UserSSLMode.allow: RedshiftSSLMode.verify_ca, - UserSSLMode.prefer: RedshiftSSLMode.verify_ca, - UserSSLMode.require: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_ca: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_full: RedshiftSSLMode.verify_full, + UserSSLMode.allow: RedshiftSSLMode("verify-ca"), + UserSSLMode.prefer: RedshiftSSLMode("verify-ca"), + UserSSLMode.require: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_ca: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_full: RedshiftSSLMode("verify-full"), } @@ -102,9 +106,9 @@ def parse(cls, user_sslmode: UserSSLMode) -> "RedshiftSSLConfig": @dataclass class RedshiftCredentials(Credentials): host: str - user: str port: Port method: str = RedshiftConnectionMethod.DATABASE # type: ignore + user: Optional[str] = None password: Optional[str] = None # type: ignore cluster_id: Optional[str] = field( default=None, @@ -116,11 +120,13 @@ class RedshiftCredentials(Credentials): ra3_node: Optional[bool] = False connect_timeout: Optional[int] = None role: Optional[str] = None - sslmode: Optional[UserSSLMode] = field(default_factory=UserSSLMode.default) + sslmode: UserSSLMode = field(default_factory=UserSSLMode.default) retries: int = 1 region: Optional[str] = None # opt-in by default per team deliberation on https://peps.python.org/pep-0249/#autocommit autocommit: Optional[bool] = True + access_key_id: Optional[str] = None + secret_access_key: Optional[str] = None _ALIASES = {"dbname": "database", "pass": "password"} @@ -142,7 +148,6 @@ def _connection_keys(self): "region", "sslmode", "region", - "iam_profile", "autocreate", "db_groups", "ra3_node", @@ -150,6 +155,7 @@ def _connection_keys(self): "role", "retries", "autocommit", + "access_key_id", ) @property @@ -160,100 +166,153 @@ def unique_field(self) -> str: class RedshiftConnectMethodFactory: credentials: RedshiftCredentials - def __init__(self, credentials): + def __init__(self, credentials) -> None: self.credentials = credentials - def get_connect_method(self): - method = self.credentials.method + def get_connect_method(self) -> Callable[[], redshift_connector.Connection]: + + # Support missing 'method' for backwards compatibility + method = self.credentials.method or RedshiftConnectionMethod.DATABASE + if method == RedshiftConnectionMethod.DATABASE: + kwargs = self._database_kwargs + elif method == RedshiftConnectionMethod.IAM: + kwargs = self._iam_user_kwargs + elif method == RedshiftConnectionMethod.IAM_ROLE: + kwargs = self._iam_role_kwargs + else: + raise FailedToConnectError(f"Invalid 'method' in profile: '{method}'") + + def connect() -> redshift_connector.Connection: + c = redshift_connector.connect(**kwargs) + if self.credentials.autocommit: + c.autocommit = True + if self.credentials.role: + c.cursor().execute(f"set role {self.credentials.role}") + return c + + return connect + + @property + def _database_kwargs(self) -> Dict[str, Any]: + logger.debug("Connecting to redshift with 'database' credentials method") + kwargs = self._base_kwargs + + if self.credentials.user and self.credentials.password: + kwargs.update( + user=self.credentials.user, + password=self.credentials.password, + ) + else: + raise FailedToConnectError( + "'user' and 'password' fields are required for 'database' credentials method" + ) + + return kwargs + + @property + def _iam_user_kwargs(self) -> Dict[str, Any]: + logger.debug("Connecting to redshift with 'iam' credentials method") + kwargs = self._iam_kwargs + + if self.credentials.access_key_id and self.credentials.secret_access_key: + kwargs.update( + access_key_id=self.credentials.access_key_id, + secret_access_key=self.credentials.secret_access_key, + ) + elif self.credentials.access_key_id or self.credentials.secret_access_key: + raise FailedToConnectError( + "'access_key_id' and 'secret_access_key' are both needed if providing explicit credentials" + ) + else: + kwargs.update(profile=self.credentials.iam_profile) + + if user := self.credentials.user: + kwargs.update(db_user=user) + else: + raise FailedToConnectError("'user' field is required for 'iam' credentials method") + + return kwargs + + @property + def _iam_role_kwargs(self) -> Dict[str, Optional[Any]]: + logger.debug("Connecting to redshift with 'iam_role' credentials method") + kwargs = self._iam_kwargs + + # It's a role, we're ignoring the user + kwargs.update(db_user=None) + + # Serverless shouldn't get group_federation, Provisoned clusters should + if "serverless" in self.credentials.host: + kwargs.update(group_federation=False) + else: + kwargs.update(group_federation=True) + + if iam_profile := self.credentials.iam_profile: + kwargs.update(profile=iam_profile) + + return kwargs + + @property + def _iam_kwargs(self) -> Dict[str, Any]: + kwargs = self._base_kwargs + kwargs.update( + iam=True, + user="", + password="", + ) + + if "serverless" in self.credentials.host: + kwargs.update(cluster_identifier=None) + elif cluster_id := self.credentials.cluster_id: + kwargs.update(cluster_identifier=cluster_id) + else: + raise FailedToConnectError( + "Failed to use IAM method:" + " 'cluster_id' must be provided for provisioned cluster" + " 'host' must be provided for serverless endpoint" + ) + + return kwargs + + @property + def _base_kwargs(self) -> Dict[str, Any]: kwargs = { "host": self.credentials.host, - "database": self.credentials.database, "port": int(self.credentials.port) if self.credentials.port else int(5439), + "database": self.credentials.database, + "region": self.credentials.region, "auto_create": self.credentials.autocreate, "db_groups": self.credentials.db_groups, - "region": self.credentials.region, "timeout": self.credentials.connect_timeout, } - redshift_ssl_config = RedshiftSSLConfig.parse(self.credentials.sslmode) kwargs.update(redshift_ssl_config.to_dict()) - - # Support missing 'method' for backwards compatibility - if method == RedshiftConnectionMethod.DATABASE or method is None: - # this requirement is really annoying to encode into json schema, - # so validate it here - if self.credentials.password is None: - raise FailedToConnectError( - "'password' field is required for 'database' credentials" - ) - - def connect(): - logger.debug("Connecting to redshift with username/password based auth...") - c = redshift_connector.connect( - user=self.credentials.user, - password=self.credentials.password, - **kwargs, - ) - if self.credentials.autocommit: - c.autocommit = True - if self.credentials.role: - c.cursor().execute("set role {}".format(self.credentials.role)) - return c - - elif method == RedshiftConnectionMethod.IAM: - if not self.credentials.cluster_id and "serverless" not in self.credentials.host: - raise FailedToConnectError( - "Failed to use IAM method. 'cluster_id' must be provided for provisioned cluster. " - "'host' must be provided for serverless endpoint." - ) - - def connect(): - logger.debug("Connecting to redshift with IAM based auth...") - c = redshift_connector.connect( - iam=True, - db_user=self.credentials.user, - password="", - user="", - cluster_identifier=self.credentials.cluster_id, - profile=self.credentials.iam_profile, - **kwargs, - ) - if self.credentials.autocommit: - c.autocommit = True - if self.credentials.role: - c.cursor().execute("set role {}".format(self.credentials.role)) - return c - - else: - raise FailedToConnectError("Invalid 'method' in profile: '{}'".format(method)) - - return connect + return kwargs class RedshiftConnectionManager(SQLConnectionManager): TYPE = "redshift" - def _get_backend_pid(self): - sql = "select pg_backend_pid()" - _, cursor = self.add_query(sql) - - res = cursor.fetchone() - return res[0] - def cancel(self, connection: Connection): + pid = connection.backend_pid # type: ignore + sql = f"select pg_terminate_backend({pid})" + logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") + logger.debug(sql) + try: - pid = self._get_backend_pid() + self.add_query(sql) except redshift_connector.InterfaceError as e: if "is closed" in str(e): logger.debug(f"Connection {connection.name} was already closed") return raise - sql = f"select pg_terminate_backend({pid})" - cursor = connection.handle.cursor() - logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") - logger.debug(sql) - cursor.execute(sql) + @classmethod + def _get_backend_pid(cls, connection): + with connection.handle.cursor() as c: + sql = "select pg_backend_pid()" + res = c.execute(sql).fetchone() + return res[0] @classmethod def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse: @@ -325,7 +384,7 @@ def exponential_backoff(attempt: int): redshift_connector.DataError, ] - return cls.retry_connection( + open_connection = cls.retry_connection( connection, connect=connect_method_factory.get_connect_method(), logger=logger, @@ -333,6 +392,8 @@ def exponential_backoff(attempt: int): retry_timeout=exponential_backoff, retryable_exceptions=retryable_exceptions, ) + open_connection.backend_pid = cls._get_backend_pid(open_connection) # type: ignore + return open_connection def execute( self, @@ -340,13 +401,15 @@ def execute( auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None, - ) -> Tuple[AdapterResponse, agate.Table]: + ) -> Tuple[AdapterResponse, "agate.Table"]: sql = self._add_query_comment(sql) _, cursor = self.add_query(sql, auto_begin) response = self.get_response(cursor) if fetch: table = self.get_result_from_cursor(cursor, limit) else: + from dbt_common.clients import agate_helper + table = agate_helper.empty_table() return response, table diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index b41308db0..d498685ed 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,11 +1,12 @@ import os from dataclasses import dataclass from dbt_common.contracts.constraints import ConstraintType -from typing import Optional, Set, Any, Dict, Type +from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING from collections import namedtuple from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available +from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support from dbt.adapters.sql import SQLAdapter from dbt.adapters.contracts.connection import AdapterResponse from dbt.adapters.events.logging import AdapterLogger @@ -27,6 +28,9 @@ GET_RELATIONS_MACRO_NAME = "redshift__get_relations" +if TYPE_CHECKING: + import agate + @dataclass class RedshiftConfig(AdapterConfig): @@ -53,6 +57,14 @@ class RedshiftAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, } + _capabilities = CapabilityDict( + { + Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), + } + ) + @classmethod def date_function(cls): return "getdate()" @@ -76,7 +88,7 @@ def drop_relation(self, relation): return super().drop_relation(relation) @classmethod - def convert_text_type(cls, agate_table, col_idx): + def convert_text_type(cls, agate_table: "agate.Table", col_idx): column = agate_table.columns[col_idx] # `lens` must be a list, so this can't be a generator expression, # because max() raises ane exception if its argument has no members. @@ -85,7 +97,7 @@ def convert_text_type(cls, agate_table, col_idx): return "varchar({})".format(max_len) @classmethod - def convert_time_type(cls, agate_table, col_idx): + def convert_time_type(cls, agate_table: "agate.Table", col_idx): return "varchar(24)" @available diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index 7255288e2..eaf60f54c 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional +from typing import FrozenSet, Optional from dbt.adapters.base.relation import BaseRelation from dbt.adapters.relation_configs import ( @@ -27,19 +27,24 @@ class RedshiftRelation(BaseRelation): include_policy = RedshiftIncludePolicy # type: ignore quote_policy = RedshiftQuotePolicy # type: ignore + require_alias: bool = False relation_configs = { RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, } - renameable_relations = frozenset( - { - RelationType.View, - RelationType.Table, - } + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) ) - replaceable_relations = frozenset( - { - RelationType.View, - } + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + } + ) ) def __post_init__(self): diff --git a/dbt/adapters/redshift/relation_configs/base.py b/dbt/adapters/redshift/relation_configs/base.py index c4faab664..6f1409659 100644 --- a/dbt/adapters/redshift/relation_configs/base.py +++ b/dbt/adapters/redshift/relation_configs/base.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from typing import Optional, Dict +from typing import Optional, Dict, TYPE_CHECKING -import agate from dbt.adapters.base.relation import Policy from dbt.adapters.contracts.relation import ComponentName, RelationConfig from dbt.adapters.relation_configs import ( @@ -15,6 +14,10 @@ RedshiftQuotePolicy, ) +if TYPE_CHECKING: + # Imported downfile for specific row gathering function. + import agate + @dataclass(frozen=True, eq=True, unsafe_hash=True) class RedshiftRelationConfigBase(RelationConfigBase): @@ -63,8 +66,10 @@ def _render_part(cls, component: ComponentName, value: Optional[str]) -> Optiona return None @classmethod - def _get_first_row(cls, results: agate.Table) -> agate.Row: + def _get_first_row(cls, results: "agate.Table") -> "agate.Row": try: return results.rows[0] except IndexError: + import agate + return agate.Row(values=set()) diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/relation_configs/dist.py index c41eda578..2bcdb9566 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/relation_configs/dist.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional, Set, Dict +from typing import Optional, Set, Dict, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationConfigChange, RelationConfigChangeAction, @@ -15,6 +14,9 @@ from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +if TYPE_CHECKING: + import agate + class RedshiftDistStyle(StrEnum): auto = "auto" @@ -24,7 +26,7 @@ class RedshiftDistStyle(StrEnum): @classmethod def default(cls) -> "RedshiftDistStyle": - return cls.auto + return cls("auto") @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -103,12 +105,12 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> dict: config = {"diststyle": diststyle} else: - config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} + config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} # type: ignore return config @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> Dict: + def parse_relation_results(cls, relation_results_entry: "agate.Row") -> Dict: """ Translate agate objects from the database into a standard dictionary. diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index 05f4b170d..48c04b554 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -1,7 +1,6 @@ from dataclasses import dataclass, field -from typing import Optional, Set, Dict, Any +from typing import Optional, Set, Dict, Any, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationResults, RelationConfigChange, @@ -25,6 +24,9 @@ ) from dbt.adapters.redshift.utility import evaluate_bool +if TYPE_CHECKING: + import agate + @dataclass(frozen=True, eq=True, unsafe_hash=True) class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigValidationMixin): @@ -57,7 +59,7 @@ class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigV database_name: str query: str backup: bool = field(default=True, compare=False, hash=False) - dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) + dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle("even")) sort: RedshiftSortConfig = RedshiftSortConfig() autorefresh: bool = False @@ -173,10 +175,10 @@ def parse_relation_results(cls, relation_results: RelationResults) -> Dict: Returns: a standard dictionary describing this `RedshiftMaterializedViewConfig` instance """ - materialized_view: agate.Row = cls._get_first_row( + materialized_view: "agate.Row" = cls._get_first_row( relation_results.get("materialized_view") ) - query: agate.Row = cls._get_first_row(relation_results.get("query")) + query: "agate.Row" = cls._get_first_row(relation_results.get("query")) config_dict = { "mv_name": materialized_view.get("table"), diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/relation_configs/sort.py index e44784c2f..be5b3627d 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/relation_configs/sort.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from dbt.adapters.contracts.relation import RelationConfig -from typing import Optional, FrozenSet, Set, Dict, Any +from typing import Optional, FrozenSet, Set, Dict, Any, TYPE_CHECKING -import agate from dbt.adapters.relation_configs import ( RelationConfigChange, RelationConfigChangeAction, @@ -15,6 +14,9 @@ from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase +if TYPE_CHECKING: + import agate + class RedshiftSortStyle(StrEnum): auto = "auto" @@ -23,11 +25,11 @@ class RedshiftSortStyle(StrEnum): @classmethod def default(cls) -> "RedshiftSortStyle": - return cls.auto + return cls("auto") @classmethod def default_with_columns(cls) -> "RedshiftSortStyle": - return cls.compound + return cls("compound") @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -136,7 +138,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any return config_dict @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_relation_results(cls, relation_results_entry: "agate.Row") -> dict: """ Translate agate objects from the database into a standard dictionary. diff --git a/dbt/include/redshift/macros/adapters/unit_testing.sql b/dbt/include/redshift/macros/adapters/unit_testing.sql new file mode 100644 index 000000000..5463f4e2b --- /dev/null +++ b/dbt/include/redshift/macros/adapters/unit_testing.sql @@ -0,0 +1,11 @@ +{%- macro redshift__validate_fixture_rows(rows, row_number) -%} + {%- if rows is not none and rows|length > 0 -%} + {%- set row = rows[0] -%} + {%- for key, value in row.items() -%} + {%- if value is none -%} + {%- set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") -%} + {{ exceptions.raise_compiler_error("Unit test fixture " ~ fixture_name ~ " in " ~ model.name ~ " does not have any row free of null values, which may cause type mismatch errors during unit test execution.") }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/catalog.sql b/dbt/include/redshift/macros/catalog.sql deleted file mode 100644 index 69dc71713..000000000 --- a/dbt/include/redshift/macros/catalog.sql +++ /dev/null @@ -1,258 +0,0 @@ - -{% macro redshift__get_base_catalog(information_schema, schemas) -%} - {%- call statement('base_catalog', fetch_result=True) -%} - {% set database = information_schema.database %} - {{ adapter.verify_database(database) }} - - with late_binding as ( - select - '{{ database }}'::varchar as table_database, - table_schema, - table_name, - 'LATE BINDING VIEW'::varchar as table_type, - null::text as table_comment, - - column_name, - column_index, - column_type, - null::text as column_comment - from pg_get_late_binding_view_cols() - cols(table_schema name, table_name name, column_name name, - column_type varchar, - column_index int) - order by "column_index" - ), - - materialized_views as ( - select - table_schema as nspname, - table_name as relname - from information_schema.views - where ( - {%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and table_catalog = '{{ database }}' - and view_definition ilike '%create materialized view%' - ), - - early_binding as ( - select - '{{ database }}'::varchar as table_database, - sch.nspname as table_schema, - tbl.relname as table_name, - case - when tbl.relkind = 'v' and materialized_views.relname is not null then 'MATERIALIZED VIEW' - when tbl.relkind = 'v' then 'VIEW' - else 'BASE TABLE' - end as table_type, - tbl_desc.description as table_comment, - col.attname as column_name, - col.attnum as column_index, - pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, - col_desc.description as column_comment - - from pg_catalog.pg_namespace sch - join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid - join pg_catalog.pg_attribute col on col.attrelid = tbl.oid - left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) - left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) - left outer join materialized_views on (materialized_views.nspname = sch.nspname and materialized_views.relname = tbl.relname) - where ( - {%- for schema in schemas -%} - upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and tbl.relkind in ('r', 'v', 'f', 'p') - and col.attnum > 0 - and not col.attisdropped - ), - - table_owners as ( - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - tablename as table_name, - tableowner as table_owner - - from pg_tables - - union all - - select - '{{ database }}'::varchar as table_database, - schemaname as table_schema, - viewname as table_name, - viewowner as table_owner - - from pg_views - - ), - - unioned as ( - - select * - from early_binding - - union all - - select * - from late_binding - - ) - - select *, - table_database || '.' || table_schema || '.' || table_name as table_id - - from unioned - join table_owners using (table_database, table_schema, table_name) - - where ( - {%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - order by "column_index" - {%- endcall -%} - - {{ return(load_result('base_catalog').table) }} -{%- endmacro %} - -{% macro redshift__get_extended_catalog(schemas) %} - {%- call statement('extended_catalog', fetch_result=True) -%} - - select - "database" || '.' || "schema" || '.' || "table" as table_id, - - 'Encoded'::text as "stats:encoded:label", - encoded as "stats:encoded:value", - 'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description", - true as "stats:encoded:include", - - 'Dist Style' as "stats:diststyle:label", - diststyle as "stats:diststyle:value", - 'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description", - true as "stats:diststyle:include", - - 'Sort Key 1' as "stats:sortkey1:label", - -- handle 0xFF byte in response for interleaved sort styles - case - when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text - else sortkey1 - end as "stats:sortkey1:value", - 'First column in the sort key.'::text as "stats:sortkey1:description", - (sortkey1 is not null) as "stats:sortkey1:include", - - 'Max Varchar' as "stats:max_varchar:label", - max_varchar as "stats:max_varchar:value", - 'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description", - true as "stats:max_varchar:include", - - -- exclude this, as the data is strangely returned with null-byte characters - 'Sort Key 1 Encoding' as "stats:sortkey1_enc:label", - sortkey1_enc as "stats:sortkey1_enc:value", - 'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description", - false as "stats:sortkey1_enc:include", - - '# Sort Keys' as "stats:sortkey_num:label", - sortkey_num as "stats:sortkey_num:value", - 'Number of columns defined as sort keys.' as "stats:sortkey_num:description", - (sortkey_num > 0) as "stats:sortkey_num:include", - - 'Approximate Size' as "stats:size:label", - size * 1000000 as "stats:size:value", - 'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description", - true as "stats:size:include", - - 'Disk Utilization' as "stats:pct_used:label", - pct_used / 100.0 as "stats:pct_used:value", - 'Percent of available space that is used by the table.'::text as "stats:pct_used:description", - true as "stats:pct_used:include", - - 'Unsorted %' as "stats:unsorted:label", - unsorted / 100.0 as "stats:unsorted:value", - 'Percent of unsorted rows in the table.'::text as "stats:unsorted:description", - (unsorted is not null) as "stats:unsorted:include", - - 'Stats Off' as "stats:stats_off:label", - stats_off as "stats:stats_off:value", - 'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description", - true as "stats:stats_off:include", - - 'Approximate Row Count' as "stats:rows:label", - tbl_rows as "stats:rows:value", - 'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description", - true as "stats:rows:include", - - 'Sort Key Skew' as "stats:skew_sortkey1:label", - skew_sortkey1 as "stats:skew_sortkey1:value", - 'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description", - (skew_sortkey1 is not null) as "stats:skew_sortkey1:include", - - 'Skew Rows' as "stats:skew_rows:label", - skew_rows as "stats:skew_rows:value", - 'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description", - (skew_rows is not null) as "stats:skew_rows:include" - - from svv_table_info - where ( - {%- for schema in schemas -%} - upper(schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - - {%- endcall -%} - - {{ return(load_result('extended_catalog').table) }} - -{% endmacro %} - -{% macro redshift__can_select_from(table_name) %} - - {%- call statement('has_table_privilege', fetch_result=True) -%} - - select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select - - {%- endcall -%} - - {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %} - {{ return(can_select) }} - -{% endmacro %} - -{% macro redshift__no_svv_table_info_warning() %} - - {% set msg %} - - Warning: The database user "{{ target.user }}" has insufficient permissions to - query the "svv_table_info" table. Please grant SELECT permissions on this table - to the "{{ target.user }}" user to fetch extended table details from Redshift. - - {% endset %} - - {{ log(msg, info=True) }} - -{% endmacro %} - - -{% macro redshift__get_catalog(information_schema, schemas) %} - - {#-- Compute a left-outer join in memory. Some Redshift queries are - -- leader-only, and cannot be joined to other compute-based queries #} - - {% set catalog = redshift__get_base_catalog(information_schema, schemas) %} - - {% set select_extended = redshift__can_select_from('svv_table_info') %} - {% if select_extended %} - {% set extended_catalog = redshift__get_extended_catalog(schemas) %} - {% set catalog = catalog.join(extended_catalog, 'table_id') %} - {% else %} - {{ redshift__no_svv_table_info_warning() }} - {% endif %} - - {{ return(catalog.exclude(['table_id'])) }} - -{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/by_relation.sql b/dbt/include/redshift/macros/catalog/by_relation.sql new file mode 100644 index 000000000..d0d79c65a --- /dev/null +++ b/dbt/include/redshift/macros/catalog/by_relation.sql @@ -0,0 +1,82 @@ +{% macro redshift__get_catalog_relations(information_schema, relations) -%} + + {% set database = information_schema.database %} + {{ adapter.verify_database(database) }} + + {#-- Compute a left-outer join in memory. Some Redshift queries are + -- leader-only, and cannot be joined to other compute-based queries #} + + {% set catalog = _redshift__get_base_catalog_by_relation(database, relations) %} + + {% set select_extended = redshift__can_select_from('svv_table_info') %} + {% if select_extended %} + {% set extended_catalog = _redshift__get_extended_catalog_by_relation(relations) %} + {% set catalog = catalog.join(extended_catalog, ['table_schema', 'table_name']) %} + {% else %} + {{ redshift__no_svv_table_info_warning() }} + {% endif %} + + {{ return(catalog) }} + +{% endmacro %} + + +{% macro _redshift__get_base_catalog_by_relation(database, relations) -%} + {%- call statement('base_catalog', fetch_result=True) -%} + with + late_binding as ({{ _redshift__get_late_binding_by_relation_sql(relations) }}), + early_binding as ({{ _redshift__get_early_binding_by_relation_sql(database, relations) }}), + unioned as (select * from early_binding union all select * from late_binding), + table_owners as ({{ redshift__get_table_owners_sql() }}) + select '{{ database }}' as table_database, * + from unioned + join table_owners using (table_schema, table_name) + order by "column_index" + {%- endcall -%} + {{ return(load_result('base_catalog').table) }} +{%- endmacro %} + + +{% macro _redshift__get_late_binding_by_relation_sql(relations) %} + {{ redshift__get_late_binding_sql() }} + where ( + {%- for relation in relations -%} + ( + upper(table_schema) = upper('{{ relation.schema }}') + and upper(table_name) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_early_binding_by_relation_sql(database, relations) %} + {{ redshift__get_early_binding_sql(database) }} + and ( + {%- for relation in relations -%} + ( + upper(sch.nspname) = upper('{{ relation.schema }}') + and upper(tbl.relname) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_extended_catalog_by_relation(relations) %} + {%- call statement('extended_catalog', fetch_result=True) -%} + {{ redshift__get_extended_catalog_sql() }} + where ( + {%- for relation in relations -%} + ( + upper("schema") = upper('{{ relation.schema }}') + and upper("table") = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {%- endcall -%} + {{ return(load_result('extended_catalog').table) }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/by_schema.sql b/dbt/include/redshift/macros/catalog/by_schema.sql new file mode 100644 index 000000000..99325f765 --- /dev/null +++ b/dbt/include/redshift/macros/catalog/by_schema.sql @@ -0,0 +1,70 @@ +{% macro redshift__get_catalog(information_schema, schemas) %} + + {% set database = information_schema.database %} + {{ adapter.verify_database(database) }} + + {#-- Compute a left-outer join in memory. Some Redshift queries are + -- leader-only, and cannot be joined to other compute-based queries #} + + {% set catalog = _redshift__get_base_catalog_by_schema(database, schemas) %} + + {% set select_extended = redshift__can_select_from('svv_table_info') %} + {% if select_extended %} + {% set extended_catalog = _redshift__get_extended_catalog_by_schema(schemas) %} + {% set catalog = catalog.join(extended_catalog, ['table_schema', 'table_name']) %} + {% else %} + {{ redshift__no_svv_table_info_warning() }} + {% endif %} + + {{ return(catalog) }} + +{% endmacro %} + + +{% macro _redshift__get_base_catalog_by_schema(database, schemas) -%} + {%- call statement('base_catalog', fetch_result=True) -%} + with + late_binding as ({{ _redshift__get_late_binding_by_schema_sql(schemas) }}), + early_binding as ({{ _redshift__get_early_binding_by_schema_sql(database, schemas) }}), + unioned as (select * from early_binding union all select * from late_binding), + table_owners as ({{ redshift__get_table_owners_sql() }}) + select '{{ database }}' as table_database, * + from unioned + join table_owners using (table_schema, table_name) + order by "column_index" + {%- endcall -%} + {{ return(load_result('base_catalog').table) }} +{%- endmacro %} + + +{% macro _redshift__get_late_binding_by_schema_sql(schemas) %} + {{ redshift__get_late_binding_sql() }} + where ( + {%- for schema in schemas -%} + upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_early_binding_by_schema_sql(database, schemas) %} + {{ redshift__get_early_binding_sql(database) }} + and ( + {%- for schema in schemas -%} + upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{% endmacro %} + + +{% macro _redshift__get_extended_catalog_by_schema(schemas) %} + {%- call statement('extended_catalog', fetch_result=True) -%} + {{ redshift__get_extended_catalog_sql() }} + where ( + {%- for schema in schemas -%} + upper("schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + {%- endcall -%} + {{ return(load_result('extended_catalog').table) }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog/catalog.sql b/dbt/include/redshift/macros/catalog/catalog.sql new file mode 100644 index 000000000..694a9441b --- /dev/null +++ b/dbt/include/redshift/macros/catalog/catalog.sql @@ -0,0 +1,176 @@ +{% macro redshift__get_late_binding_sql() %} + select + table_schema, + table_name, + 'LATE BINDING VIEW'::varchar as table_type, + null::text as table_comment, + column_name, + column_index, + column_type, + null::text as column_comment + from pg_get_late_binding_view_cols() + cols( + table_schema name, + table_name name, + column_name name, + column_type varchar, + column_index int + ) +{% endmacro %} + + +{% macro redshift__get_early_binding_sql(database) %} + select + sch.nspname as table_schema, + tbl.relname as table_name, + case + when tbl.relkind = 'v' and mat_views.table_name is not null then 'MATERIALIZED VIEW' + when tbl.relkind = 'v' then 'VIEW' + else 'BASE TABLE' + end as table_type, + tbl_desc.description as table_comment, + col.attname as column_name, + col.attnum as column_index, + pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, + col_desc.description as column_comment + from pg_catalog.pg_namespace sch + join pg_catalog.pg_class tbl + on tbl.relnamespace = sch.oid + join pg_catalog.pg_attribute col + on col.attrelid = tbl.oid + left outer join pg_catalog.pg_description tbl_desc + on tbl_desc.objoid = tbl.oid + and tbl_desc.objsubid = 0 + left outer join pg_catalog.pg_description col_desc + on col_desc.objoid = tbl.oid + and col_desc.objsubid = col.attnum + left outer join information_schema.views mat_views + on mat_views.table_schema = sch.nspname + and mat_views.table_name = tbl.relname + and mat_views.view_definition ilike '%create materialized view%' + and mat_views.table_catalog = '{{ database }}' + where tbl.relkind in ('r', 'v', 'f', 'p') + and col.attnum > 0 + and not col.attisdropped +{% endmacro %} + + +{% macro redshift__get_table_owners_sql() %} + select + schemaname as table_schema, + tablename as table_name, + tableowner as table_owner + from pg_tables + union all + select + schemaname as table_schema, + viewname as table_name, + viewowner as table_owner + from pg_views +{% endmacro %} + + +{% macro redshift__get_extended_catalog_sql() %} + select + "schema" as table_schema, + "table" as table_name, + + 'Encoded'::text as "stats:encoded:label", + encoded as "stats:encoded:value", + 'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description", + true as "stats:encoded:include", + + 'Dist Style' as "stats:diststyle:label", + diststyle as "stats:diststyle:value", + 'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description", + true as "stats:diststyle:include", + + 'Sort Key 1' as "stats:sortkey1:label", + -- handle 0xFF byte in response for interleaved sort styles + case + when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text + else sortkey1 + end as "stats:sortkey1:value", + 'First column in the sort key.'::text as "stats:sortkey1:description", + (sortkey1 is not null) as "stats:sortkey1:include", + + 'Max Varchar' as "stats:max_varchar:label", + max_varchar as "stats:max_varchar:value", + 'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description", + true as "stats:max_varchar:include", + + -- exclude this, as the data is strangely returned with null-byte characters + 'Sort Key 1 Encoding' as "stats:sortkey1_enc:label", + sortkey1_enc as "stats:sortkey1_enc:value", + 'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description", + false as "stats:sortkey1_enc:include", + + '# Sort Keys' as "stats:sortkey_num:label", + sortkey_num as "stats:sortkey_num:value", + 'Number of columns defined as sort keys.' as "stats:sortkey_num:description", + (sortkey_num > 0) as "stats:sortkey_num:include", + + 'Approximate Size' as "stats:size:label", + size * 1000000 as "stats:size:value", + 'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description", + true as "stats:size:include", + + 'Disk Utilization' as "stats:pct_used:label", + pct_used / 100.0 as "stats:pct_used:value", + 'Percent of available space that is used by the table.'::text as "stats:pct_used:description", + true as "stats:pct_used:include", + + 'Unsorted %' as "stats:unsorted:label", + unsorted / 100.0 as "stats:unsorted:value", + 'Percent of unsorted rows in the table.'::text as "stats:unsorted:description", + (unsorted is not null) as "stats:unsorted:include", + + 'Stats Off' as "stats:stats_off:label", + stats_off as "stats:stats_off:value", + 'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description", + true as "stats:stats_off:include", + + 'Approximate Row Count' as "stats:rows:label", + tbl_rows as "stats:rows:value", + 'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description", + true as "stats:rows:include", + + 'Sort Key Skew' as "stats:skew_sortkey1:label", + skew_sortkey1 as "stats:skew_sortkey1:value", + 'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description", + (skew_sortkey1 is not null) as "stats:skew_sortkey1:include", + + 'Skew Rows' as "stats:skew_rows:label", + skew_rows as "stats:skew_rows:value", + 'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description", + (skew_rows is not null) as "stats:skew_rows:include" + + from svv_table_info +{% endmacro %} + + +{% macro redshift__can_select_from(table_name) %} + + {%- call statement('has_table_privilege', fetch_result=True) -%} + select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select + {%- endcall -%} + + {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %} + {{ return(can_select) }} + +{% endmacro %} + + +{% macro redshift__no_svv_table_info_warning() %} + + {% set msg %} + + Warning: The database user "{{ target.user }}" has insufficient permissions to + query the "svv_table_info" table. Please grant SELECT permissions on this table + to the "{{ target.user }}" user to fetch extended table details from Redshift. + + {% endset %} + + {{ log(msg, info=True) }} + +{% endmacro %} diff --git a/dbt/include/redshift/macros/metadata/relation_last_modified.sql b/dbt/include/redshift/macros/metadata/relation_last_modified.sql new file mode 100644 index 000000000..f21299c72 --- /dev/null +++ b/dbt/include/redshift/macros/metadata/relation_last_modified.sql @@ -0,0 +1,29 @@ +{% macro redshift__get_relation_last_modified(information_schema, relations) -%} + + {%- call statement('last_modified', fetch_result=True) -%} + select + ns.nspname as "schema", + c.relname as identifier, + max(qd.start_time) as last_modified, + {{ current_timestamp() }} as snapshotted_at + from pg_class c + join pg_namespace ns + on ns.oid = c.relnamespace + join sys_query_detail qd + on qd.table_id = c.oid + where qd.step_name = 'insert' + and ( + {%- for relation in relations -%} + ( + upper(ns.nspname) = upper('{{ relation.schema }}') + and upper(c.relname) = upper('{{ relation.identifier }}') + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + group by 1, 2, 4 + {%- endcall -%} + + {{ return(load_result('last_modified')) }} + +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/create.sql b/dbt/include/redshift/macros/relations/materialized_view/create.sql index 06fe2b6b5..1b81992e4 100644 --- a/dbt/include/redshift/macros/relations/materialized_view/create.sql +++ b/dbt/include/redshift/macros/relations/materialized_view/create.sql @@ -10,6 +10,6 @@ auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} as ( {{ materialized_view.query }} - ); + ) {% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/rename.sql b/dbt/include/redshift/macros/relations/view/rename.sql index 0c6cdcdfa..a96b04451 100644 --- a/dbt/include/redshift/macros/relations/view/rename.sql +++ b/dbt/include/redshift/macros/relations/view/rename.sql @@ -1,3 +1,3 @@ {% macro redshift__get_rename_view_sql(relation, new_name) %} - alter view {{ relation }} rename to {{ new_name }} + alter table {{ relation }} rename to {{ new_name }} {% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/replace.sql b/dbt/include/redshift/macros/relations/view/replace.sql index 25a9d8b38..7ae89ab45 100644 --- a/dbt/include/redshift/macros/relations/view/replace.sql +++ b/dbt/include/redshift/macros/relations/view/replace.sql @@ -13,6 +13,6 @@ {{ get_assert_columns_equivalent(sql) }} {%- endif %} as ( {{ sql }} - ) {{ bind_qualifier }}; + ) {{ bind_qualifier }} {%- endmacro %} diff --git a/dbt/include/redshift/profile_template.yml b/dbt/include/redshift/profile_template.yml index 41f33e87e..d78356923 100644 --- a/dbt/include/redshift/profile_template.yml +++ b/dbt/include/redshift/profile_template.yml @@ -15,6 +15,8 @@ prompts: hide_input: true iam: _fixed_method: iam + iam_role: + _fixed_method: iam_role dbname: hint: 'default database that dbt will build objects in' schema: diff --git a/dev-requirements.txt b/dev-requirements.txt index 0c8c9b1a3..52c26c936 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,31 +1,25 @@ # install latest changes in dbt-core + dbt-postgres -# TODO: how to switch from HEAD to x.y.latest branches after minor releases? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git@main +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-common.git +git+https://github.com/dbt-labs/dbt-postgres.git -# if version 1.x or greater -> pin to major version -# if version 0.x -> pin to minor -black~=23.12 -bumpversion~=0.6.0 -click~=8.1 -ddtrace~=2.3 -flake8~=6.1 -flaky~=3.7 -freezegun~=1.3 +# dev ipdb~=0.13.13 -mypy==1.7.1 # patch updates have historically introduced breaking changes -pip-tools~=7.3 -pre-commit~=3.5 -pre-commit-hooks~=4.5 +pre-commit~=3.7.0;python_version >="3.9" +pre-commit~=3.5.0;python_version <"3.9" + +# test +ddtrace==2.3.0 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 -pytz~=2023.3 -tox~=4.11 -types-pytz~=2023.3 -types-requests~=2.31 -twine~=4.0 +tox~=4.16 + +# build +bumpversion~=0.6.0 +twine~=5.1 wheel~=0.42 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..5914e6396 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,37 @@ +# this image gets published to GHCR for production use +ARG py_version=3.11.2 + +FROM python:$py_version-slim-bullseye as base + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-redshift + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..5be9e56ef --- /dev/null +++ b/docker/README.md @@ -0,0 +1,58 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-redshift` + +In order to build a new image, run the following docker command. +```shell +docker build --tag --target dbt-redshift +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the image will be populated with the latest version of `dbt-redshift` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +```shell +docker build --tag \ + --target dbt-redshift \ + --build-arg commit_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +```shell +cd dbt-core/docker +docker build --tag my-dbt --target dbt-redshift . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +```shell +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-redshift \ + --build-arg commit_ref=v1.0.0b1 \ + . +``` + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +```shell +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 000000000..aac016320 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,50 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + + +FROM base as dbt-redshift-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index b6e603581..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -namespace_packages = True diff --git a/pytest.ini b/pytest.ini index b3d74bc14..8c290dc14 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,3 +7,5 @@ env_files = testpaths = tests/unit tests/functional +markers = + flaky: marks tests as flaky so they run one at a time (de-select with '-m "not flaky"') diff --git a/setup.py b/setup.py index 4925bb04b..712da3870 100644 --- a/setup.py +++ b/setup.py @@ -48,13 +48,16 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common<1.0", - "dbt-adapters~=0.1.0a1", - f"dbt-postgres~={_plugin_version()}", + "dbt-common>=0.1.0a1,<2.0", + "dbt-adapters>=0.1.0a1,<2.0", + "dbt-postgres>=1.8,<1.10", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<=2.0.918, >=2.0.913, !=2.0.914", + "redshift-connector<2.1.1,>=2.0.913,!=2.0.914", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0b3", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "sqlparse>=0.5.0,<0.6.0", "agate", ], zip_safe=False, @@ -68,6 +71,7 @@ def _plugin_version() -> str: "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], python_requires=">=3.8", ) diff --git a/test.env.example b/test.env.example index 4de05edab..6816b4ec2 100644 --- a/test.env.example +++ b/test.env.example @@ -1,18 +1,26 @@ # Note: Make sure you have a Redshift account that is set up so these fields are easy to complete. - -### Test Environment field definitions # These will all be gathered from account information or created by you. -# Endpoint for Redshift connection + +# Database Authentication Method REDSHIFT_TEST_HOST= -# Username on your account -REDSHIFT_TEST_USER= -# Password for Redshift account -REDSHIFT_TEST_PASS= -# Local port to connect on REDSHIFT_TEST_PORT= -# Name of Redshift database in your account to test against REDSHIFT_TEST_DBNAME= -# Users for testing +REDSHIFT_TEST_USER= +REDSHIFT_TEST_PASS= +REDSHIFT_TEST_REGION= + +# IAM Methods +REDSHIFT_TEST_CLUSTER_ID= + +# IAM User Authentication Method +REDSHIFT_TEST_IAM_USER_PROFILE= +REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID= +REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY= + +# IAM Role Authentication Method +REDSHIFT_TEST_IAM_ROLE_PROFILE= + +# Database users for testing DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 diff --git a/tests/conftest.py b/tests/conftest.py index 96f0d43e4..712bf047a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,12 @@ -import pytest -import os - -# Import the functional fixtures as a plugin -# Note: fixtures with session scope need to be local - pytest_plugins = ["dbt.tests.fixtures.project"] -# The profile dictionary, used to write out profiles.yml -@pytest.fixture(scope="class") -def dbt_profile_target(): - return { - "type": "redshift", - "threads": 1, - "retries": 6, - "host": os.getenv("REDSHIFT_TEST_HOST"), - "port": int(os.getenv("REDSHIFT_TEST_PORT")), - "user": os.getenv("REDSHIFT_TEST_USER"), - "pass": os.getenv("REDSHIFT_TEST_PASS"), - "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), - } +def pytest_sessionfinish(session, exitstatus): + """ + Configures pytest to treat a scenario with no tests as passing + + pytest returns a code 5 when it collects no tests in an effort to warn when tests are expected but not collected + We don't want this when running tox because some combinations of markers and test segments return nothing + """ + if exitstatus == 5: + session.exitstatus = 0 diff --git a/tests/functional/adapter/catalog_tests/test_get_catalog.py b/tests/functional/adapter/catalog_tests/test_get_catalog.py new file mode 100644 index 000000000..e0b512896 --- /dev/null +++ b/tests/functional/adapter/catalog_tests/test_get_catalog.py @@ -0,0 +1,144 @@ +from dbt.adapters.contracts.relation import RelationType +from dbt.tests.util import get_connection +import pytest + + +class TestGetCatalog: + @pytest.fixture(scope="class") + def my_schema(self, project, adapter): + schema = adapter.Relation.create( + database=project.database, + schema=project.test_schema, + identifier="", + ) + yield schema + + @pytest.fixture(scope="class") + def my_seed(self, adapter, my_schema): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_seed", + type=RelationType.Table, + ) + with get_connection(adapter): + sql = f""" + create table {relation.database}.{relation.schema}.{relation.identifier} ( + id integer, + value integer, + record_valid_date timestamp + ); + insert into {relation.database}.{relation.schema}.{relation.identifier} + (id, value, record_valid_date) values + (1,100,'2023-01-01 00:00:00'), + (2,200,'2023-01-02 00:00:00'), + (3,300,'2023-01-02 00:00:00') + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_table(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_table", + type=RelationType.Table, + ) + with get_connection(adapter): + sql = f""" + create table {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_view(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_view", + type=RelationType.View, + ) + with get_connection(adapter): + sql = f""" + create view {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_materialized_view(self, adapter, my_schema, my_seed): + relation = adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="my_materialized_view", + type=RelationType.MaterializedView, + ) + with get_connection(adapter): + sql = f""" + create materialized view {relation.database}.{relation.schema}.{relation.identifier} as + select * from {my_seed.database}.{my_seed.schema}.{my_seed.identifier} + ; + """ + adapter.execute(sql) + yield relation + + @pytest.fixture(scope="class") + def my_information_schema(self, adapter, my_schema): + yield adapter.Relation.create( + database=my_schema.database, + schema=my_schema.schema, + identifier="INFORMATION_SCHEMA", + ).information_schema() + + @pytest.mark.flaky + def test_get_one_catalog_by_relations( + self, + adapter, + my_schema, + my_seed, + my_table, + my_view, + my_materialized_view, + my_information_schema, + ): + my_schemas = frozenset({(my_schema.database, my_schema.schema)}) + my_relations = [my_seed, my_table, my_view, my_materialized_view] + with get_connection(adapter): + catalog = adapter._get_one_catalog_by_relations( + information_schema=my_information_schema, + relations=my_relations, + used_schemas=my_schemas, + ) + # my_seed, my_table, my_view, my_materialized_view each have 3 cols = 12 cols + # my_materialized_view creates an underlying table with 2 additional = 5 cols + # note the underlying table is missing as it's not in `my_relations` + assert len(catalog) == 12 + + @pytest.mark.flaky + def test_get_one_catalog_by_schemas( + self, + adapter, + my_schema, + my_seed, + my_table, + my_view, + my_materialized_view, + my_information_schema, + ): + my_schemas = frozenset({(my_schema.database, my_schema.schema)}) + with get_connection(adapter): + catalog = adapter._get_one_catalog( + information_schema=my_information_schema, + schemas={my_schema.schema}, + used_schemas=my_schemas, + ) + # my_seed, my_table, my_view, my_materialized_view each have 3 cols = 12 cols + # my_materialized_view creates an underlying table with 2 additional = 5 cols + assert len(catalog) == 17 diff --git a/tests/functional/adapter/catalog_tests/test_relation_types.py b/tests/functional/adapter/catalog_tests/test_relation_types.py index 9b9156dec..657bf215b 100644 --- a/tests/functional/adapter/catalog_tests/test_relation_types.py +++ b/tests/functional/adapter/catalog_tests/test_relation_types.py @@ -24,6 +24,7 @@ def docs(self, project): run_dbt(["run"]) yield run_dbt(["docs", "generate"]) + @pytest.mark.flaky @pytest.mark.parametrize( "node_name,relation_type", [ diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index 808a7733c..83cb399ca 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -1,4 +1,8 @@ -from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowSqlHeader, + BaseShowLimit, + BaseShowDoesNotHandleDoubleLimit, +) class TestRedshiftShowLimit(BaseShowLimit): @@ -7,3 +11,7 @@ class TestRedshiftShowLimit(BaseShowLimit): class TestRedshiftShowSqlHeader(BaseShowSqlHeader): pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 3368a90f9..27f36f1df 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -1,5 +1,9 @@ -from dbt.tests.adapter.empty.test_empty import BaseTestEmpty +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef class TestRedshiftEmpty(BaseTestEmpty): pass + + +class TestRedshiftEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + pass diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index 63bcede61..f201196b4 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -3,7 +3,7 @@ import pytest from dbt.adapters.base.relation import BaseRelation -from dbt.contracts.graph.model_config import OnConfigurationChangeOption +from dbt.adapters.contracts.relation import OnConfigurationChangeOption from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic from dbt.tests.adapter.materialized_view.changes import ( @@ -75,6 +75,14 @@ def test_materialized_view_create_idempotent(self, project, my_materialized_view ) assert self.query_relation_type(project, my_materialized_view) == "materialized_view" + @pytest.mark.flaky + def test_table_replaces_materialized_view(self, project, my_materialized_view): + super().test_table_replaces_materialized_view(project, my_materialized_view) + + @pytest.mark.flaky + def test_view_replaces_materialized_view(self, project, my_materialized_view): + super().test_view_replaces_materialized_view(project, my_materialized_view) + class RedshiftMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) @@ -200,6 +208,7 @@ def test_change_is_not_applied_via_alter(self, project, my_materialized_view): assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs, False) assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) + @pytest.mark.flaky def test_change_is_not_applied_via_replace(self, project, my_materialized_view): self.check_start_state(project, my_materialized_view) @@ -224,7 +233,10 @@ class TestRedshiftMaterializedViewChangesFail( RedshiftMaterializedViewChanges, MaterializedViewChangesFailMixin ): # Note: using retries doesn't work when we expect `dbt_run` to fail - pass + + @pytest.mark.flaky + def test_change_is_not_applied_via_replace(self, project, my_materialized_view): + super().test_change_is_not_applied_via_replace(project, my_materialized_view) NO_BACKUP_MATERIALIZED_VIEW = """ @@ -259,6 +271,7 @@ def dbt_run_results(self, project): def test_running_mv_with_backup_false_succeeds(self, dbt_run_results): assert dbt_run_results[0].node.config_call_dict["backup"] is False + @pytest.mark.flaky def test_running_mv_with_backup_false_is_idempotent(self, project, dbt_run_results): """ Addresses: https://github.com/dbt-labs/dbt-redshift/issues/621 diff --git a/tests/functional/adapter/sources_freshness_tests/files.py b/tests/functional/adapter/sources_freshness_tests/files.py new file mode 100644 index 000000000..f2dd6fbbe --- /dev/null +++ b/tests/functional/adapter/sources_freshness_tests/files.py @@ -0,0 +1,38 @@ +SCHEMA_YML = """version: 2 +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_source_no_last_modified + - name: test_source_last_modified + loaded_at_field: last_modified +""" + +SEED_TEST_SOURCE_NO_LAST_MODIFIED_CSV = """ +id,name +1,Martin +2,Jeter +3,Ruth +4,Gehrig +5,DiMaggio +6,Torre +7,Mantle +8,Berra +9,Maris +""".strip() + +SEED_TEST_SOURCE_LAST_MODIFIED_CSV = """ +id,name,last_modified +1,Martin,2023-01-01 00:00:00 +2,Jeter,2023-02-01 00:00:00 +3,Ruth,2023-03-01 00:00:00 +4,Gehrig,2023-04-01 00:00:00 +5,DiMaggio,2023-05-01 00:00:00 +6,Torre,2023-06-01 00:00:00 +7,Mantle,2023-07-01 00:00:00 +8,Berra,2023-08-01 00:00:00 +9,Maris,2023-09-01 00:00:00 +""".strip() diff --git a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py new file mode 100644 index 000000000..c31e9ac61 --- /dev/null +++ b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py @@ -0,0 +1,150 @@ +import os +import pytest +from unittest import mock + +from dbt.adapters.redshift.impl import RedshiftAdapter +from dbt.adapters.capability import Capability, CapabilityDict +from dbt.cli.main import dbtRunner +from dbt.tests.util import run_dbt + +from tests.functional.adapter.sources_freshness_tests import files + + +class SetupGetLastRelationModified: + @pytest.fixture(scope="class", autouse=True) + def set_env_vars(self, project): + os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema + yield + del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + + +class TestGetLastRelationModified(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def seeds(self): + return { + "test_source_no_last_modified.csv": files.SEED_TEST_SOURCE_NO_LAST_MODIFIED_CSV, + "test_source_last_modified.csv": files.SEED_TEST_SOURCE_LAST_MODIFIED_CSV, + } + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": files.SCHEMA_YML} + + @pytest.mark.parametrize( + "source,status,expect_pass", + [ + ("test_source.test_source_no_last_modified", "pass", True), + ("test_source.test_source_last_modified", "error", False), # stale + ], + ) + def test_get_last_relation_modified(self, project, source, status, expect_pass): + run_dbt(["seed"]) + + results = run_dbt( + ["source", "freshness", "--select", f"source:{source}"], expect_pass=expect_pass + ) + assert len(results) == 1 + result = results[0] + assert result.status == status + + +freshness_metadata_schema_batch_yml = """ +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_table + - name: test_table2 + - name: test_table_with_loaded_at_field + loaded_at_field: my_loaded_at_field +""" + + +class TestGetLastRelationModifiedBatch(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def custom_schema(self, project, set_env_vars): + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + ) + project.adapter.drop_schema(relation) + project.adapter.create_schema(relation) + + yield relation.schema + + with project.adapter.connection_named("__test"): + project.adapter.drop_schema(relation) + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": freshness_metadata_schema_batch_yml} + + def get_freshness_result_for_table(self, table_name, results): + for result in results: + if result.node.name == table_name: + return result + return None + + def test_get_last_relation_modified_batch(self, project, custom_schema): + project.run_sql( + f"create table {custom_schema}.test_table as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table2 as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table_with_loaded_at_field as (select 1 as id, timestamp '2009-09-15 10:59:43' as my_loaded_at_field);" + ) + + runner = dbtRunner() + freshness_results_batch = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results_batch) == 3 + test_table_batch_result = self.get_freshness_result_for_table( + "test_table", freshness_results_batch + ) + test_table2_batch_result = self.get_freshness_result_for_table( + "test_table2", freshness_results_batch + ) + test_table_with_loaded_at_field_batch_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results_batch + ) + + # Remove TableLastModifiedMetadataBatch and run freshness on same input without batch strategy + capabilities_no_batch = CapabilityDict( + { + capability: support + for capability, support in RedshiftAdapter.capabilities().items() + if capability != Capability.TableLastModifiedMetadataBatch + } + ) + with mock.patch.object( + RedshiftAdapter, "capabilities", return_value=capabilities_no_batch + ): + freshness_results = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results) == 3 + test_table_result = self.get_freshness_result_for_table("test_table", freshness_results) + test_table2_result = self.get_freshness_result_for_table("test_table2", freshness_results) + test_table_with_loaded_at_field_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results + ) + + # assert results between batch vs non-batch freshness strategy are equivalent + assert test_table_result.status == test_table_batch_result.status + assert test_table_result.max_loaded_at == test_table_batch_result.max_loaded_at + + assert test_table2_result.status == test_table2_batch_result.status + assert test_table2_result.max_loaded_at == test_table2_batch_result.max_loaded_at + + assert ( + test_table_with_loaded_at_field_batch_result.status + == test_table_with_loaded_at_field_result.status + ) + assert ( + test_table_with_loaded_at_field_batch_result.max_loaded_at + == test_table_with_loaded_at_field_result.max_loaded_at + ) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 64ab24e42..cd04bb1ba 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -38,7 +38,9 @@ # TODO: update these with test cases or remove them if not needed class TestSimpleMaterializationsRedshift(BaseSimpleMaterializations): - pass + @pytest.mark.flaky + def test_base(self, project): + super().test_base(project) class TestSingularTestsRedshift(BaseSingularTests): @@ -54,11 +56,15 @@ class TestEmptyRedshift(BaseEmpty): class TestEphemeralRedshift(BaseEphemeral): - pass + @pytest.mark.flaky + def test_ephemeral(self, project): + super().test_ephemeral(project) class TestIncrementalRedshift(BaseIncremental): - pass + @pytest.mark.flaky + def test_incremental(self, project): + super().test_incremental(project) class TestGenericTestsRedshift(BaseGenericTests): diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py index 61b8bd5a6..6eeaf881c 100644 --- a/tests/functional/adapter/test_persist_docs.py +++ b/tests/functional/adapter/test_persist_docs.py @@ -12,15 +12,21 @@ class TestPersistDocs(BasePersistDocs): - pass + @pytest.mark.flaky + def test_has_comments_pglike(self, project): + super().test_has_comments_pglike(project) class TestPersistDocsColumnMissing(BasePersistDocsColumnMissing): - pass + @pytest.mark.flaky + def test_missing_column(self, project): + super().test_missing_column(project) class TestPersistDocsCommentOnQuotedColumn(BasePersistDocsCommentOnQuotedColumn): - pass + @pytest.mark.flaky + def test_quoted_column_comments(self, run_has_comments): + super().test_quoted_column_comments(run_has_comments) class TestPersistDocsLateBinding(BasePersistDocsBase): @@ -40,6 +46,7 @@ def project_config_update(self): } } + @pytest.mark.flaky def test_comment_on_late_binding_view(self, project): run_dbt() run_dbt(["docs", "generate"]) diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py index db6a440d7..75c87ee38 100644 --- a/tests/functional/adapter/test_query_comment.py +++ b/tests/functional/adapter/test_query_comment.py @@ -6,6 +6,7 @@ BaseNullQueryComments, BaseEmptyQueryComments, ) +import pytest class TestQueryCommentsRedshift(BaseQueryComments): @@ -17,7 +18,12 @@ class TestMacroQueryCommentsRedshift(BaseMacroQueryComments): class TestMacroArgsQueryCommentsRedshift(BaseMacroArgsQueryComments): - pass + @pytest.mark.skip( + "This test is incorrectly comparing the version of `dbt-core`" + "to the version of `dbt-postgres`, which is not always the same." + ) + def test_matches_comment(self, project, get_package_version): + pass class TestMacroInvalidQueryCommentsRedshift(BaseMacroInvalidQueryComments): diff --git a/tests/functional/adapter/unit_testing/fixtures.py b/tests/functional/adapter/unit_testing/fixtures.py new file mode 100644 index 000000000..36212dff3 --- /dev/null +++ b/tests/functional/adapter/unit_testing/fixtures.py @@ -0,0 +1,73 @@ +model_none_value_base = """ +{{ config(materialized="table") }} + +select 1 as id, 'a' as col1 +""" + +model_none_value_model = """ +{{config(materialized="table")}} + +select * from {{ ref('none_value_base') }} +""" + + +test_none_column_value_doesnt_throw_error_csv = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + format: csv + rows: | + id,col1 + ,d + ,e + 6,f + + expect: + format: csv + rows: | + id,col1 + ,d + ,e + 6,f +""" + +test_none_column_value_doesnt_throw_error_dct = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": "f"} + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": "f"} +""" + +test_none_column_value_will_throw_error = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": } + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": } +""" diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..27ed54cb6 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,106 @@ +import pytest + +from dbt.artifacts.schemas.results import RunStatus +from dbt.tests.fixtures.project import write_project_files +from dbt.tests.util import run_dbt + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +from tests.functional.adapter.unit_testing.fixtures import ( + model_none_value_base, + model_none_value_model, + test_none_column_value_doesnt_throw_error_csv, + test_none_column_value_doesnt_throw_error_dct, + test_none_column_value_will_throw_error, +) + +from dbt_common.exceptions import CompilationError + + +class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["1.0", "1.0"], + ["'1'", "1"], + ["'1'::numeric", "1"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + [ + """JSON_PARSE('{"bar": "baz", "balance": 7.77, "active": false}')""", + """'{"bar": "baz", "balance": 7.77, "active": false}'""", + ], + # TODO: array types + # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + # ["ARRAY[1.0,2.0,3.0]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY[1::numeric,2::numeric,3::numeric]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + # ["ARRAY[true,true,false]", """'{true, true, false}'"""], + # ["ARRAY[DATE '2020-01-02']", """'{"2020-01-02"}'"""], + # ["ARRAY[TIMESTAMP '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + # ["ARRAY[TIMESTAMPTZ '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + ] + + +class RedshiftUnitTestingNone: + def test_nones_handled_dict(self, project): + run_dbt(["build"]) + + +class TestRedshiftUnitTestCsvNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_csv, + } + + +class TestRedshiftUnitTestDictNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_dct, + } + + +class TestRedshiftUnitTestingTooManyNonesFails: + @pytest.fixture(scope="class") + def models(self): + return { + "__properties.yml": test_none_column_value_will_throw_error, + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + } + + def test_invalid_input(self, project): + """This is a user-facing exception, so we can't pytest.raise(CompilationError)""" + + def _find_first_error(items): + return next((item for item in items if item.status == RunStatus.Error), None) + + run_result = run_dbt(["build"], expect_pass=False) + first_item = _find_first_error(run_result) + + assert first_item is not None + assert ( + "does not have any row free of null values, which may cause type mismatch errors during unit test execution" + in str(first_item.message) + ) + + +class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestRedshiftUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 61a706f4d..3a085712a 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -3,9 +3,11 @@ from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_concat import BaseConcat from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive +from dbt.tests.adapter.utils.test_date import BaseDate from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine @@ -48,6 +50,10 @@ class TestBoolOr(BaseBoolOr): pass +class TestCast(BaseCast): + pass + + class TestCastBoolToText(BaseCastBoolToText): pass @@ -61,6 +67,10 @@ class TestCurrentTimestamp(BaseCurrentTimestampNaive): pass +class TestDate(BaseDate): + pass + + class TestDateAdd(BaseDateAdd): pass diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 000000000..73329936d --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,19 @@ +import os + +import pytest + + +# The profile dictionary, used to write out profiles.yml +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "redshift", + "host": os.getenv("REDSHIFT_TEST_HOST"), + "port": int(os.getenv("REDSHIFT_TEST_PORT")), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "pass": os.getenv("REDSHIFT_TEST_PASS"), + "region": os.getenv("REDSHIFT_TEST_REGION"), + "threads": 1, + "retries": 6, + } diff --git a/tests/functional/test_auth_method.py b/tests/functional/test_auth_method.py new file mode 100644 index 000000000..b2273e02c --- /dev/null +++ b/tests/functional/test_auth_method.py @@ -0,0 +1,103 @@ +import os + +import pytest + +from dbt.adapters.redshift.connections import RedshiftConnectionMethod +from dbt.tests.util import run_dbt + + +MY_SEED = """ +id,name +1,apple +2,banana +3,cherry +""".strip() + + +MY_VIEW = """ +select * from {{ ref("my_seed") }} +""" + + +class AuthMethod: + + @pytest.fixture(scope="class") + def seeds(self): + yield {"my_seed.csv": MY_SEED} + + @pytest.fixture(scope="class") + def models(self): + yield {"my_view.sql": MY_VIEW} + + def test_connection(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 1 + + +class TestDatabaseMethod(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.DATABASE.value, + "host": os.getenv("REDSHIFT_TEST_HOST"), + "port": int(os.getenv("REDSHIFT_TEST_PORT")), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "pass": os.getenv("REDSHIFT_TEST_PASS"), + "threads": 1, + "retries": 6, + } + + +class TestIAMUserMethodProfile(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "iam_profile": os.getenv("REDSHIFT_TEST_IAM_USER_PROFILE"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } + + +class TestIAMUserMethodExplicit(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "access_key_id": os.getenv("REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID"), + "secret_access_key": os.getenv("REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY"), + "region": os.getenv("REDSHIFT_TEST_REGION"), + "user": os.getenv("REDSHIFT_TEST_USER"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } + + +class TestIAMRoleAuthProfile(AuthMethod): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "redshift", + "method": RedshiftConnectionMethod.IAM_ROLE.value, + "cluster_id": os.getenv("REDSHIFT_TEST_CLUSTER_ID"), + "dbname": os.getenv("REDSHIFT_TEST_DBNAME"), + "iam_profile": os.getenv("REDSHIFT_TEST_IAM_ROLE_PROFILE"), + "threads": 1, + "retries": 6, + "host": "", # host is a required field in dbt-core + "port": 0, # port is a required field in dbt-core + } diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index 8547480d1..6e4143b9c 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -1,5 +1,5 @@ -from unittest import mock from contextlib import contextmanager +from unittest import mock from dbt.adapters.base import BaseAdapter diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_auth_method.py similarity index 54% rename from tests/unit/test_redshift_adapter.py rename to tests/unit/test_auth_method.py index 671e47032..55b1aad74 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_auth_method.py @@ -1,34 +1,22 @@ -import unittest - from multiprocessing import get_context -from unittest import mock - -from dbt_common.exceptions import DbtRuntimeError -from unittest.mock import Mock, call +from unittest import TestCase, mock +from unittest.mock import MagicMock -import agate -import dbt +from dbt.adapters.exceptions import FailedToConnectError import redshift_connector from dbt.adapters.redshift import ( - RedshiftAdapter, Plugin as RedshiftPlugin, + RedshiftAdapter, ) -from dbt_common.clients import agate_helper -from dbt.adapters.exceptions import FailedToConnectError from dbt.adapters.redshift.connections import RedshiftConnectMethodFactory, RedshiftSSLConfig -from .utils import ( - config_from_parts_or_dicts, - mock_connection, - TestAdapterConversions, - inject_adapter, -) +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter DEFAULT_SSL_CONFIG = RedshiftSSLConfig().to_dict() -class TestRedshiftAdapter(unittest.TestCase): +class AuthMethod(TestCase): def setUp(self): profile_cfg = { "outputs": { @@ -67,8 +55,78 @@ def adapter(self): inject_adapter(self._adapter, RedshiftPlugin) return self._adapter - @mock.patch("redshift_connector.connect", Mock()) - def test_implicit_database_conn(self): + +class TestInvalidMethod(AuthMethod): + def test_invalid_auth_method(self): + # we have to set method this way, otherwise it won't validate + self.config.credentials.method = "badmethod" + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() + self.assertTrue("badmethod" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_missing_region_failure(self): + # Failure test with no region + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233_no_region", + region=None, + ) + + with self.assertRaises(FailedToConnectError): + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233_no_region", + database="redshift", + cluster_identifier=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_invalid_region_failure(self): + # Invalid region test + self.config.credentials = self.config.credentials.replace( + method="iam", + iam_profile="test", + host="doesnotexist.1233_no_region.us-not-a-region-1", + region=None, + ) + + with self.assertRaises(FailedToConnectError): + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, + host="doesnotexist.1233_no_region", + database="redshift", + cluster_identifier=None, + auto_create=False, + db_groups=[], + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, + **DEFAULT_SSL_CONFIG, + ) + + +class TestDatabaseMethod(AuthMethod): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( @@ -84,8 +142,8 @@ def test_implicit_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_region_with_database_conn(self): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_explicit_auth_method(self): self.config.method = "database" connection = self.adapter.acquire_connection("dummy") @@ -103,8 +161,73 @@ def test_explicit_region_with_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_iam_conn_without_profile(self): + def test_database_verification_is_case_insensitive(self): + # Override adapter settings from setUp() + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "Redshift", + "user": "root", + "host": "thishostshouldnotexist", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self.adapter.cleanup_connections() + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + self.adapter.verify_database("redshift") + + +class TestIAMUserMethod(AuthMethod): + + def test_iam_optionals(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist", + "port": 5439, + "schema": "public", + "method": "iam", + "cluster_id": "my_redshift", + "db_groups": ["my_dbgroup"], + "autocreate": True, + } + }, + "target": "test", + } + + config_from_parts_or_dicts(self.config, profile_cfg) + + def test_no_cluster_id(self): + self.config.credentials = self.config.credentials.replace(method="iam") + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() + + self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): self.config.credentials = self.config.credentials.replace( method="iam", cluster_id="my_redshift", @@ -129,54 +252,68 @@ def test_explicit_iam_conn_without_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_conn_timeout_30(self): - self.config.credentials = self.config.credentials.replace(connect_timeout=30) + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile(self): + self.config.credentials = self.config.credentials.replace( + method="iam", + cluster_id="my_redshift", + iam_profile="test", + host="thishostshouldnotexist.test.us-east-1", + ) connection = self.adapter.acquire_connection("dummy") connection.handle + redshift_connector.connect.assert_called_once_with( + iam=True, host="thishostshouldnotexist.test.us-east-1", database="redshift", - user="root", - password="password", - port=5439, + cluster_identifier="my_redshift", + region=None, auto_create=False, db_groups=[], - region=None, - timeout=30, + db_user="root", + password="", + user="", + profile="test", + timeout=None, + port=5439, **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_iam_conn_with_profile(self): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_explicit(self): self.config.credentials = self.config.credentials.replace( method="iam", cluster_id="my_redshift", - iam_profile="test", host="thishostshouldnotexist.test.us-east-1", + access_key_id="my_access_key_id", + secret_access_key="my_secret_access_key", ) connection = self.adapter.acquire_connection("dummy") connection.handle - redshift_connector.connect.assert_called_once_with( iam=True, host="thishostshouldnotexist.test.us-east-1", + access_key_id="my_access_key_id", + secret_access_key="my_secret_access_key", database="redshift", - cluster_identifier="my_redshift", - region=None, - auto_create=False, - db_groups=[], db_user="root", password="", user="", - profile="test", + cluster_identifier="my_redshift", + region=None, timeout=None, + auto_create=False, + db_groups=[], port=5439, **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_iam_serverless_with_profile(self): + +class TestIAMUserMethodServerless(AuthMethod): + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_default_region(self): self.config.credentials = self.config.credentials.replace( method="iam", iam_profile="test", @@ -201,8 +338,8 @@ def test_explicit_iam_serverless_with_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_region(self): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_explicit_region(self): # Successful test self.config.credentials = self.config.credentials.replace( method="iam", @@ -229,167 +366,193 @@ def test_explicit_region(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_region_failure(self): - # Failure test with no region + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_invalid_serverless(self): self.config.credentials = self.config.credentials.replace( method="iam", iam_profile="test", - host="doesnotexist.1233_no_region", - region=None, + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", ) - - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): + with self.assertRaises(FailedToConnectError) as context: connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( iam=True, - host="doesnotexist.1233_no_region", + host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", database="redshift", cluster_identifier=None, + region=None, auto_create=False, db_groups=[], db_user="root", password="", user="", profile="test", - timeout=None, port=5439, + timeout=None, **DEFAULT_SSL_CONFIG, ) + self.assertTrue("'host' must be provided" in context.exception.msg) - @mock.patch("redshift_connector.connect", Mock()) - def test_explicit_invalid_region(self): - # Invalid region test - self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", - host="doesnotexist.1233_no_region.us-not-a-region-1", - region=None, - ) - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): - connection = self.adapter.acquire_connection("dummy") - connection.handle - redshift_connector.connect.assert_called_once_with( - iam=True, - host="doesnotexist.1233_no_region", - database="redshift", - cluster_identifier=None, - auto_create=False, - db_groups=[], - db_user="root", - password="", - user="", - profile="test", - timeout=None, - port=5439, - **DEFAULT_SSL_CONFIG, - ) +class TestIAMRoleMethod(AuthMethod): + + def test_no_cluster_id(self): + self.config.credentials = self.config.credentials.replace(method="iam_role") + with self.assertRaises(FailedToConnectError) as context: + connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) + connect_method_factory.get_connect_method() - @mock.patch("redshift_connector.connect", Mock()) - def test_sslmode_disable(self): - self.config.credentials.sslmode = "disable" + self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_default(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + cluster_id="my_redshift", + ) connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( + iam=True, host="thishostshouldnotexist.test.us-east-1", database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], + cluster_identifier="my_redshift", + db_user=None, + password="", + user="", region=None, timeout=None, - ssl=False, - sslmode=None, + auto_create=False, + db_groups=[], + port=5439, + group_federation=True, + **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_sslmode_allow(self): - self.config.credentials.sslmode = "allow" + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + cluster_id="my_redshift", + iam_profile="test", + ) connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( + iam=True, host="thishostshouldnotexist.test.us-east-1", database="redshift", - user="root", - password="password", - port=5439, - auto_create=False, - db_groups=[], + cluster_identifier="my_redshift", + db_user=None, + password="", + user="", region=None, timeout=None, - ssl=True, - sslmode="verify-ca", + auto_create=False, + db_groups=[], + profile="test", + port=5439, + group_federation=True, + **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_sslmode_verify_full(self): - self.config.credentials.sslmode = "verify-full" + +class TestIAMRoleMethodServerless(AuthMethod): + # Should behave like IAM Role provisioned, with the exception of not having group_federation set + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_default_region(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + ) connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", + iam=True, + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", database="redshift", - user="root", - password="password", - port=5439, + cluster_identifier=None, + region=None, auto_create=False, db_groups=[], - region=None, + db_user=None, + password="", + user="", + profile="iam_profile_test", timeout=None, - ssl=True, - sslmode="verify-full", + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_sslmode_verify_ca(self): - self.config.credentials.sslmode = "verify-ca" + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_ignore_cluster(self): + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", + cluster_id="my_redshift", + ) connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", + iam=True, + host="doesnotexist.1233.us-east-2.redshift-serverless.amazonaws.com", database="redshift", - user="root", - password="password", - port=5439, + cluster_identifier=None, + region=None, auto_create=False, db_groups=[], - region=None, + db_user=None, + password="", + user="", + profile="iam_profile_test", timeout=None, - ssl=True, - sslmode="verify-ca", + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_sslmode_prefer(self): - self.config.credentials.sslmode = "prefer" + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_explicit_region(self): + # Successful test + self.config.credentials = self.config.credentials.replace( + method="iam_role", + iam_profile="iam_profile_test", + host="doesnotexist.1233.redshift-serverless.amazonaws.com", + region="us-east-2", + ) connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( - host="thishostshouldnotexist.test.us-east-1", + iam=True, + host="doesnotexist.1233.redshift-serverless.amazonaws.com", database="redshift", - user="root", - password="password", - port=5439, + cluster_identifier=None, + region="us-east-2", auto_create=False, db_groups=[], - region=None, + db_user=None, + password="", + user="", + profile="iam_profile_test", timeout=None, - ssl=True, - sslmode="verify-ca", + port=5439, + group_federation=False, + **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) - def test_serverless_iam_failure(self): + @mock.patch("redshift_connector.connect", MagicMock()) + def test_profile_invalid_serverless(self): self.config.credentials = self.config.credentials.replace( - method="iam", - iam_profile="test", + method="iam_role", + iam_profile="iam_profile_test", host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", ) - with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError) as context: + with self.assertRaises(FailedToConnectError) as context: connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( @@ -400,259 +563,13 @@ def test_serverless_iam_failure(self): region=None, auto_create=False, db_groups=[], - db_user="root", + db_user=None, password="", user="", - profile="test", + profile="iam_profile_test", port=5439, timeout=None, + group_federation=False, **DEFAULT_SSL_CONFIG, ) self.assertTrue("'host' must be provided" in context.exception.msg) - - def test_iam_conn_optionals(self): - profile_cfg = { - "outputs": { - "test": { - "type": "redshift", - "dbname": "redshift", - "user": "root", - "host": "thishostshouldnotexist", - "port": 5439, - "schema": "public", - "method": "iam", - "cluster_id": "my_redshift", - "db_groups": ["my_dbgroup"], - "autocreate": True, - } - }, - "target": "test", - } - - config_from_parts_or_dicts(self.config, profile_cfg) - - def test_invalid_auth_method(self): - # we have to set method this way, otherwise it won't validate - self.config.credentials.method = "badmethod" - with self.assertRaises(FailedToConnectError) as context: - connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) - connect_method_factory.get_connect_method() - self.assertTrue("badmethod" in context.exception.msg) - - def test_invalid_iam_no_cluster_id(self): - self.config.credentials = self.config.credentials.replace(method="iam") - with self.assertRaises(FailedToConnectError) as context: - connect_method_factory = RedshiftConnectMethodFactory(self.config.credentials) - connect_method_factory.get_connect_method() - - self.assertTrue("'cluster_id' must be provided" in context.exception.msg) - - def test_cancel_open_connections_empty(self): - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_master(self): - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections[key] = mock_connection("master") - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) - - def test_cancel_open_connections_single(self): - master = mock_connection("master") - model = mock_connection("model") - - key = self.adapter.connections.get_thread_identifier() - self.adapter.connections.thread_connections.update( - { - key: master, - 1: model, - } - ) - with mock.patch.object(self.adapter.connections, "add_query") as add_query: - query_result = mock.MagicMock() - cursor = mock.Mock() - cursor.fetchone.return_value = (42,) - add_query.side_effect = [(None, cursor), (None, query_result)] - - self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) - add_query.assert_has_calls( - [ - call("select pg_backend_pid()"), - ] - ) - - master.handle.get_backend_pid.assert_not_called() - - def test_dbname_verification_is_case_insensitive(self): - # Override adapter settings from setUp() - profile_cfg = { - "outputs": { - "test": { - "type": "redshift", - "dbname": "Redshift", - "user": "root", - "host": "thishostshouldnotexist", - "pass": "password", - "port": 5439, - "schema": "public", - } - }, - "target": "test", - } - - project_cfg = { - "name": "X", - "version": "0.1", - "profile": "test", - "project-root": "/tmp/dbt/does-not-exist", - "quoting": { - "identifier": False, - "schema": True, - }, - "config-version": 2, - } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self.adapter.cleanup_connections() - self._adapter = RedshiftAdapter(self.config, get_context("spawn")) - self.adapter.verify_database("redshift") - - def test_execute_with_fetch(self): - cursor = mock.Mock() - table = agate_helper.empty_table() - with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: - mock_add_query.return_value = ( - None, - cursor, - ) # when mock_add_query is called, it will always return None, cursor - with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: - mock_get_response.return_value = None - with mock.patch.object( - self.adapter.connections, "get_result_from_cursor" - ) as mock_get_result_from_cursor: - mock_get_result_from_cursor.return_value = table - self.adapter.connections.execute(sql="select * from test", fetch=True) - mock_add_query.assert_called_once_with("select * from test", False) - mock_get_result_from_cursor.assert_called_once_with(cursor, None) - mock_get_response.assert_called_once_with(cursor) - - def test_execute_without_fetch(self): - cursor = mock.Mock() - with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: - mock_add_query.return_value = ( - None, - cursor, - ) # when mock_add_query is called, it will always return None, cursor - with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: - mock_get_response.return_value = None - with mock.patch.object( - self.adapter.connections, "get_result_from_cursor" - ) as mock_get_result_from_cursor: - self.adapter.connections.execute(sql="select * from test2", fetch=False) - mock_add_query.assert_called_once_with("select * from test2", False) - mock_get_result_from_cursor.assert_not_called() - mock_get_response.assert_called_once_with(cursor) - - def test_add_query_with_no_cursor(self): - with mock.patch.object( - self.adapter.connections, "get_thread_connection" - ) as mock_get_thread_connection: - mock_get_thread_connection.return_value = None - with self.assertRaisesRegex(DbtRuntimeError, "Tried to run invalid SQL: on "): - self.adapter.connections.add_query(sql="") - mock_get_thread_connection.assert_called_once() - - def test_add_query_success(self): - cursor = mock.Mock() - with mock.patch.object( - dbt.adapters.redshift.connections.SQLConnectionManager, "add_query" - ) as mock_add_query: - mock_add_query.return_value = None, cursor - self.adapter.connections.add_query("select * from test3") - mock_add_query.assert_called_once_with( - "select * from test3", True, bindings=None, abridge_sql_log=False - ) - - @mock.patch.object( - dbt.adapters.redshift.connections.SQLConnectionManager, "get_thread_connection" - ) - def mock_cursor(self, mock_get_thread_conn): - conn = mock.MagicMock - mock_get_thread_conn.return_value = conn - mock_handle = mock.MagicMock - conn.return_value = mock_handle - mock_cursor = mock.MagicMock - mock_handle.return_value = mock_cursor - return mock_cursor - - -class TestRedshiftAdapterConversions(TestAdapterConversions): - def test_convert_text_type(self): - rows = [ - ["", "a1", "stringval1"], - ["", "a2", "stringvalasdfasdfasdfa"], - ["", "a3", "stringval3"], - ] - agate_table = self._make_table_of(rows, agate.Text) - expected = ["varchar(64)", "varchar(2)", "varchar(22)"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_text_type(agate_table, col_idx) == expect - - def test_convert_number_type(self): - rows = [ - ["", "23.98", "-1"], - ["", "12.78", "-2"], - ["", "79.41", "-3"], - ] - agate_table = self._make_table_of(rows, agate.Number) - expected = ["integer", "float8", "integer"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_number_type(agate_table, col_idx) == expect - - def test_convert_boolean_type(self): - rows = [ - ["", "false", "true"], - ["", "false", "false"], - ["", "false", "true"], - ] - agate_table = self._make_table_of(rows, agate.Boolean) - expected = ["boolean", "boolean", "boolean"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_boolean_type(agate_table, col_idx) == expect - - def test_convert_datetime_type(self): - rows = [ - ["", "20190101T01:01:01Z", "2019-01-01 01:01:01"], - ["", "20190102T01:01:01Z", "2019-01-01 01:01:01"], - ["", "20190103T01:01:01Z", "2019-01-01 01:01:01"], - ] - agate_table = self._make_table_of( - rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime] - ) - expected = [ - "timestamp without time zone", - "timestamp without time zone", - "timestamp without time zone", - ] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_datetime_type(agate_table, col_idx) == expect - - def test_convert_date_type(self): - rows = [ - ["", "2019-01-01", "2019-01-04"], - ["", "2019-01-02", "2019-01-04"], - ["", "2019-01-03", "2019-01-04"], - ] - agate_table = self._make_table_of(rows, agate.Date) - expected = ["date", "date", "date"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_date_type(agate_table, col_idx) == expect - - def test_convert_time_type(self): - # dbt's default type testers actually don't have a TimeDelta at all. - rows = [ - ["", "120s", "10s"], - ["", "3m", "11s"], - ["", "1h", "12s"], - ] - agate_table = self._make_table_of(rows, agate.TimeDelta) - expected = ["varchar(24)", "varchar(24)", "varchar(24)"] - for col_idx, expect in enumerate(expected): - assert RedshiftAdapter.convert_time_type(agate_table, col_idx) == expect diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py new file mode 100644 index 000000000..17a676286 --- /dev/null +++ b/tests/unit/test_connection.py @@ -0,0 +1,130 @@ +from multiprocessing import get_context +from unittest import TestCase, mock +from unittest.mock import MagicMock, call + +import redshift_connector + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from tests.unit.utils import ( + config_from_parts_or_dicts, + inject_adapter, + mock_connection, +) + + +class TestConnection(TestCase): + + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + def test_cancel_open_connections_empty(self): + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_master(self): + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections[key] = mock_connection("master") + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) + + def test_cancel_open_connections_single(self): + master = mock_connection("master") + model = mock_connection("model") + + key = self.adapter.connections.get_thread_identifier() + self.adapter.connections.thread_connections.update( + { + key: master, + 1: model, + } + ) + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + query_result = mock.MagicMock() + cursor = mock.Mock() + cursor.fetchone.return_value = (42,) + add_query.side_effect = [(None, cursor), (None, query_result)] + + self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({model.backend_pid})"), + ] + ) + + master.handle.backend_pid.assert_not_called() + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_has_backend_pid(self): + backend_pid = 42 + + cursor = mock.MagicMock() + execute = cursor().__enter__().execute + execute().fetchone.return_value = (backend_pid,) + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + assert connection.backend_pid == backend_pid + + execute.assert_has_calls( + [ + call("select pg_backend_pid()"), + ] + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_backend_pid_used_in_pg_terminate_backend(self): + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + backend_pid = 42 + query_result = (backend_pid,) + + cursor = mock.MagicMock() + cursor().__enter__().execute().fetchone.return_value = query_result + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + + self.adapter.connections.cancel(connection) + + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({backend_pid})"), + ] + ) diff --git a/tests/unit/test_conversion.py b/tests/unit/test_conversion.py new file mode 100644 index 000000000..a375c4f6a --- /dev/null +++ b/tests/unit/test_conversion.py @@ -0,0 +1,80 @@ +import agate +from dbt_common.clients import agate_helper + +from dbt.adapters.redshift import RedshiftAdapter +from tests.unit.utils import TestAdapterConversions + + +class TestConversion(TestAdapterConversions): + def test_convert_text_type(self): + rows = [ + ["", "a1", "stringval1"], + ["", "a2", "stringvalasdfasdfasdfa"], + ["", "a3", "stringval3"], + ] + agate_table = self._make_table_of(rows, agate.Text) + expected = ["varchar(64)", "varchar(2)", "varchar(22)"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_text_type(agate_table, col_idx) == expect + + def test_convert_number_type(self): + rows = [ + ["", "23.98", "-1"], + ["", "12.78", "-2"], + ["", "79.41", "-3"], + ] + agate_table = self._make_table_of(rows, agate.Number) + expected = ["integer", "float8", "integer"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_number_type(agate_table, col_idx) == expect + + def test_convert_boolean_type(self): + rows = [ + ["", "false", "true"], + ["", "false", "false"], + ["", "false", "true"], + ] + agate_table = self._make_table_of(rows, agate.Boolean) + expected = ["boolean", "boolean", "boolean"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_boolean_type(agate_table, col_idx) == expect + + def test_convert_datetime_type(self): + rows = [ + ["", "20190101T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190102T01:01:01Z", "2019-01-01 01:01:01"], + ["", "20190103T01:01:01Z", "2019-01-01 01:01:01"], + ] + agate_table = self._make_table_of( + rows, [agate.DateTime, agate_helper.ISODateTime, agate.DateTime] + ) + expected = [ + "timestamp without time zone", + "timestamp without time zone", + "timestamp without time zone", + ] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_datetime_type(agate_table, col_idx) == expect + + def test_convert_date_type(self): + rows = [ + ["", "2019-01-01", "2019-01-04"], + ["", "2019-01-02", "2019-01-04"], + ["", "2019-01-03", "2019-01-04"], + ] + agate_table = self._make_table_of(rows, agate.Date) + expected = ["date", "date", "date"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_date_type(agate_table, col_idx) == expect + + def test_convert_time_type(self): + # dbt's default type testers actually don't have a TimeDelta at all. + rows = [ + ["", "120s", "10s"], + ["", "3m", "11s"], + ["", "1h", "12s"], + ] + agate_table = self._make_table_of(rows, agate.TimeDelta) + expected = ["varchar(24)", "varchar(24)", "varchar(24)"] + for col_idx, expect in enumerate(expected): + assert RedshiftAdapter.convert_time_type(agate_table, col_idx) == expect diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/test_materialized_view.py similarity index 79% rename from tests/unit/relation_configs/test_materialized_view.py rename to tests/unit/test_materialized_view.py index 5e454fe5e..8e4f6ca3e 100644 --- a/tests/unit/relation_configs/test_materialized_view.py +++ b/tests/unit/test_materialized_view.py @@ -14,8 +14,8 @@ def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) config_dict = config.parse_relation_config(model_node) assert isinstance(config_dict["autorefresh"], bool) @@ -33,8 +33,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(TypeError): config.parse_relation_config(model_node) @@ -48,8 +48,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(ValueError): config.parse_relation_config(model_node) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py new file mode 100644 index 000000000..ff8076215 --- /dev/null +++ b/tests/unit/test_query.py @@ -0,0 +1,116 @@ +from multiprocessing import get_context +from unittest import TestCase, mock + +from dbt.adapters.sql.connections import SQLConnectionManager +from dbt_common.clients import agate_helper +from dbt_common.exceptions import DbtRuntimeError + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter + + +class TestQuery(TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + @mock.patch.object(SQLConnectionManager, "get_thread_connection") + def mock_cursor(self, mock_get_thread_conn): + conn = mock.MagicMock + mock_get_thread_conn.return_value = conn + mock_handle = mock.MagicMock + conn.return_value = mock_handle + mock_cursor = mock.MagicMock + mock_handle.return_value = mock_cursor + return mock_cursor + + def test_execute_with_fetch(self): + cursor = mock.Mock() + table = agate_helper.empty_table() + with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: + mock_add_query.return_value = ( + None, + cursor, + ) # when mock_add_query is called, it will always return None, cursor + with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: + mock_get_response.return_value = None + with mock.patch.object( + self.adapter.connections, "get_result_from_cursor" + ) as mock_get_result_from_cursor: + mock_get_result_from_cursor.return_value = table + self.adapter.connections.execute(sql="select * from test", fetch=True) + mock_add_query.assert_called_once_with("select * from test", False) + mock_get_result_from_cursor.assert_called_once_with(cursor, None) + mock_get_response.assert_called_once_with(cursor) + + def test_execute_without_fetch(self): + cursor = mock.Mock() + with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: + mock_add_query.return_value = ( + None, + cursor, + ) # when mock_add_query is called, it will always return None, cursor + with mock.patch.object(self.adapter.connections, "get_response") as mock_get_response: + mock_get_response.return_value = None + with mock.patch.object( + self.adapter.connections, "get_result_from_cursor" + ) as mock_get_result_from_cursor: + self.adapter.connections.execute(sql="select * from test2", fetch=False) + mock_add_query.assert_called_once_with("select * from test2", False) + mock_get_result_from_cursor.assert_not_called() + mock_get_response.assert_called_once_with(cursor) + + def test_add_query_success(self): + cursor = mock.Mock() + with mock.patch.object(SQLConnectionManager, "add_query") as mock_add_query: + mock_add_query.return_value = None, cursor + self.adapter.connections.add_query("select * from test3") + mock_add_query.assert_called_once_with( + "select * from test3", True, bindings=None, abridge_sql_log=False + ) + + def test_add_query_with_no_cursor(self): + with mock.patch.object( + self.adapter.connections, "get_thread_connection" + ) as mock_get_thread_connection: + mock_get_thread_connection.return_value = None + with self.assertRaisesRegex(DbtRuntimeError, "Tried to run invalid SQL: on "): + self.adapter.connections.add_query(sql="") + mock_get_thread_connection.assert_called_once() diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py new file mode 100644 index 000000000..4817ab100 --- /dev/null +++ b/tests/unit/test_relation.py @@ -0,0 +1,17 @@ +from dbt.adapters.redshift.relation import RedshiftRelation +from dbt.adapters.contracts.relation import RelationType + + +def test_renameable_relation(): + relation = RedshiftRelation.create( + database="my_db", + schema="my_schema", + identifier="my_table", + type=RelationType.Table, + ) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + } + ) diff --git a/tests/unit/test_ssl_mode.py b/tests/unit/test_ssl_mode.py new file mode 100644 index 000000000..832e0718b --- /dev/null +++ b/tests/unit/test_ssl_mode.py @@ -0,0 +1,168 @@ +from multiprocessing import get_context +from unittest import TestCase, mock +from unittest.mock import MagicMock + +import redshift_connector + +from dbt.adapters.redshift import ( + Plugin as RedshiftPlugin, + RedshiftAdapter, +) +from dbt.adapters.redshift.connections import RedshiftSSLConfig +from tests.unit.utils import config_from_parts_or_dicts, inject_adapter + + +DEFAULT_SSL_CONFIG = RedshiftSSLConfig().to_dict() + + +class TestSSLMode(TestCase): + def setUp(self): + profile_cfg = { + "outputs": { + "test": { + "type": "redshift", + "dbname": "redshift", + "user": "root", + "host": "thishostshouldnotexist.test.us-east-1", + "pass": "password", + "port": 5439, + "schema": "public", + } + }, + "target": "test", + } + + project_cfg = { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": True, + }, + "config-version": 2, + } + + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self._adapter = None + + @property + def adapter(self): + if self._adapter is None: + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) + inject_adapter(self._adapter, RedshiftPlugin) + return self._adapter + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_disable(self): + self.config.credentials.sslmode = "disable" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=False, + sslmode=None, + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_allow(self): + self.config.credentials.sslmode = "allow" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_verify_full(self): + self.config.credentials.sslmode = "verify-full" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-full", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_verify_ca(self): + self.config.credentials.sslmode = "verify-ca" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_prefer(self): + self.config.credentials.sslmode = "prefer" + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=None, + ssl=True, + sslmode="verify-ca", + ) + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_timeout(self): + self.config.credentials = self.config.credentials.replace(connect_timeout=30) + connection = self.adapter.acquire_connection("dummy") + connection.handle + redshift_connector.connect.assert_called_once_with( + host="thishostshouldnotexist.test.us-east-1", + database="redshift", + user="root", + password="password", + port=5439, + auto_create=False, + db_groups=[], + region=None, + timeout=30, + **DEFAULT_SSL_CONFIG, + ) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 3fc1d7ec6..ee580eb9a 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -2,15 +2,15 @@ Note that all imports should be inside the functions to avoid import/mocking issues. """ + import string import os -from unittest import mock -from unittest import TestCase +from unittest import TestCase, mock import agate -import pytest -from dbt_common.dataclass_schema import ValidationError from dbt.config.project import PartialProject +from dbt_common.dataclass_schema import ValidationError +import pytest def normalize(path): diff --git a/tox.ini b/tox.ini index c490fed9a..fafa867a3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311 +envlist = py38,py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py}] +[testenv:{unit,py38,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py}-{redshift}] +[testenv:{integration,py38,py39,py310,py311,py312,py}-{redshift}] description = adapter plugin integration testing skip_install = true passenv = @@ -21,6 +21,7 @@ passenv = REDSHIFT_TEST_* PYTEST_ADDOPTS DD_CIVISIBILITY_AGENTLESS_ENABLED + DD_INSTRUMENTATION_TELEMETRY_ENABLED DD_API_KEY DD_SITE DD_ENV