diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cffe9f374..f54032556 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.0a1 +current_version = 1.8.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/unreleased/Dependencies-20230711-192047.yaml b/.changes/unreleased/Dependencies-20230711-192047.yaml deleted file mode 100644 index fa7c1df80..000000000 --- a/.changes/unreleased/Dependencies-20230711-192047.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.3 to ~=23.7" -time: 2023-07-11T19:20:47.00000Z -custom: - Author: dependabot[bot] - PR: 529 diff --git a/.changes/unreleased/Dependencies-20230717-195349.yaml b/.changes/unreleased/Dependencies-20230717-195349.yaml deleted file mode 100644 index 542970ab8..000000000 --- a/.changes/unreleased/Dependencies-20230717-195349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pip-tools requirement from ~=6.14 to ~=7.0" -time: 2023-07-17T19:53:49.00000Z -custom: - Author: dependabot[bot] - PR: 541 diff --git a/.changes/unreleased/Dependencies-20230720-191748.yaml b/.changes/unreleased/Dependencies-20230720-191748.yaml deleted file mode 100644 index eb79676d4..000000000 --- a/.changes/unreleased/Dependencies-20230720-191748.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update ddtrace requirement from ~=1.16 to ~=1.17" -time: 2023-07-20T19:17:48.00000Z -custom: - Author: dependabot[bot] - PR: 550 diff --git a/.changes/unreleased/Dependencies-20230724-190537.yaml b/.changes/unreleased/Dependencies-20230724-190537.yaml deleted file mode 100644 index c61cb9df3..000000000 --- a/.changes/unreleased/Dependencies-20230724-190537.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.40 to ~=0.41" -time: 2023-07-24T19:05:37.00000Z -custom: - Author: dependabot[bot] - PR: 551 diff --git a/.changes/unreleased/Dependencies-20230731-194135.yaml b/.changes/unreleased/Dependencies-20230731-194135.yaml deleted file mode 100644 index c9990b89a..000000000 --- a/.changes/unreleased/Dependencies-20230731-194135.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update flake8 requirement from ~=6.0 to ~=6.1" -time: 2023-07-31T19:41:35.00000Z -custom: - Author: dependabot[bot] - PR: 562 diff --git a/.changes/unreleased/Dependencies-20230803-131633.yaml b/.changes/unreleased/Dependencies-20230803-131633.yaml deleted file mode 100644 index d6755cd13..000000000 --- a/.changes/unreleased/Dependencies-20230803-131633.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pip-tools requirement from ~=7.0 to ~=7.2" -time: 2023-08-03T13:16:33.00000Z -custom: - Author: dependabot[bot] - PR: 567 diff --git a/.changes/unreleased/Dependencies-20230912-120620.yaml b/.changes/unreleased/Dependencies-20230912-120620.yaml new file mode 100644 index 000000000..be139237d --- /dev/null +++ b/.changes/unreleased/Dependencies-20230912-120620.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Update redshift-connector requirement from ~=2.0.913 to ~=2.0.914 +time: 2023-09-12T12:06:20.401643-07:00 +custom: + Author: soksamnanglim + PR: "601" diff --git a/.changes/unreleased/Dependencies-20231002-164037.yaml b/.changes/unreleased/Dependencies-20231002-164037.yaml new file mode 100644 index 000000000..54f51a402 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231002-164037.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.19 to ~=1.20" +time: 2023-10-02T16:40:37.00000Z +custom: + Author: dependabot[bot] + PR: 622 diff --git a/.changes/unreleased/Dependencies-20231009-192801.yaml b/.changes/unreleased/Dependencies-20231009-192801.yaml new file mode 100644 index 000000000..2a53a074f --- /dev/null +++ b/.changes/unreleased/Dependencies-20231009-192801.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pre-commit-hooks requirement from ~=4.4 to ~=4.5" +time: 2023-10-09T19:28:01.00000Z +custom: + Author: dependabot[bot] + PR: 627 diff --git a/.changes/unreleased/Dependencies-20231010-195348.yaml b/.changes/unreleased/Dependencies-20231010-195348.yaml new file mode 100644 index 000000000..4e8af02c1 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231010-195348.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.5.1 to 1.6.0" +time: 2023-10-10T19:53:48.00000Z +custom: + Author: dependabot[bot] + PR: 629 diff --git a/.changes/unreleased/Dependencies-20231013-190517.yaml b/.changes/unreleased/Dependencies-20231013-190517.yaml new file mode 100644 index 000000000..8db17538a --- /dev/null +++ b/.changes/unreleased/Dependencies-20231013-190517.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pre-commit requirement from ~=3.4 to ~=3.5" +time: 2023-10-13T19:05:17.00000Z +custom: + Author: dependabot[bot] + PR: 634 diff --git a/.changes/unreleased/Dependencies-20231017-191545.yaml b/.changes/unreleased/Dependencies-20231017-191545.yaml new file mode 100644 index 000000000..407615e32 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231017-191545.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update black requirement from ~=23.9 to ~=23.10" +time: 2023-10-17T19:15:45.00000Z +custom: + Author: dependabot[bot] + PR: 636 diff --git a/.changes/unreleased/Dependencies-20231027-173152.yaml b/.changes/unreleased/Dependencies-20231027-173152.yaml new file mode 100644 index 000000000..8ed464977 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231027-173152.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.6.0 to 1.6.1" +time: 2023-10-27T17:31:52.00000Z +custom: + Author: dependabot[bot] + PR: 648 diff --git a/.changes/unreleased/Dependencies-20231030-193514.yaml b/.changes/unreleased/Dependencies-20231030-193514.yaml new file mode 100644 index 000000000..e2771051b --- /dev/null +++ b/.changes/unreleased/Dependencies-20231030-193514.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=1.20 to ~=2.1" +time: 2023-10-30T19:35:14.00000Z +custom: + Author: dependabot[bot] + PR: 651 diff --git a/.changes/unreleased/Dependencies-20231108-190800.yaml b/.changes/unreleased/Dependencies-20231108-190800.yaml new file mode 100644 index 000000000..e8ce0c5b4 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231108-190800.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update black requirement from ~=23.10 to ~=23.11" +time: 2023-11-08T19:08:00.00000Z +custom: + Author: dependabot[bot] + PR: 660 diff --git a/.changes/unreleased/Dependencies-20231110-192349.yaml b/.changes/unreleased/Dependencies-20231110-192349.yaml new file mode 100644 index 000000000..90b285065 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231110-192349.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.6.1 to 1.7.0" +time: 2023-11-10T19:23:49.00000Z +custom: + Author: dependabot[bot] + PR: 662 diff --git a/.changes/unreleased/Dependencies-20231113-195504.yaml b/.changes/unreleased/Dependencies-20231113-195504.yaml new file mode 100644 index 000000000..39b81e3e8 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231113-195504.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pytest-xdist requirement from ~=3.3 to ~=3.4" +time: 2023-11-13T19:55:04.00000Z +custom: + Author: dependabot[bot] + PR: 664 diff --git a/.changes/unreleased/Dependencies-20231116-194405.yaml b/.changes/unreleased/Dependencies-20231116-194405.yaml new file mode 100644 index 000000000..608ade10b --- /dev/null +++ b/.changes/unreleased/Dependencies-20231116-194405.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=2.1 to ~=2.2" +time: 2023-11-16T19:44:05.00000Z +custom: + Author: dependabot[bot] + PR: 665 diff --git a/.changes/unreleased/Dependencies-20231127-201640.yaml b/.changes/unreleased/Dependencies-20231127-201640.yaml new file mode 100644 index 000000000..bbcdb6ff9 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231127-201640.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update ddtrace requirement from ~=2.2 to ~=2.3" +time: 2023-11-27T20:16:40.00000Z +custom: + Author: dependabot[bot] + PR: 669 diff --git a/.changes/unreleased/Dependencies-20231127-201942.yaml b/.changes/unreleased/Dependencies-20231127-201942.yaml new file mode 100644 index 000000000..ac6c15d53 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231127-201942.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update wheel requirement from ~=0.41 to ~=0.42" +time: 2023-11-27T20:19:42.00000Z +custom: + Author: dependabot[bot] + PR: 670 diff --git a/.changes/unreleased/Dependencies-20231128-194822.yaml b/.changes/unreleased/Dependencies-20231128-194822.yaml new file mode 100644 index 000000000..820ba25d1 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231128-194822.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pytest-xdist requirement from ~=3.4 to ~=3.5" +time: 2023-11-28T19:48:22.00000Z +custom: + Author: dependabot[bot] + PR: 672 diff --git a/.changes/unreleased/Dependencies-20231129-195044.yaml b/.changes/unreleased/Dependencies-20231129-195044.yaml new file mode 100644 index 000000000..235a89777 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231129-195044.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.7.0 to 1.7.1" +time: 2023-11-29T19:50:44.00000Z +custom: + Author: dependabot[bot] + PR: 676 diff --git a/.changes/unreleased/Dependencies-20231130-044332.yaml b/.changes/unreleased/Dependencies-20231130-044332.yaml new file mode 100644 index 000000000..0fbeb9a18 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231130-044332.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Remove direct boto3 dependency +time: 2023-11-30T04:43:32.872452+11:00 +custom: + Author: hexDoor + PR: "674" diff --git a/.changes/unreleased/Dependencies-20231204-193730.yaml b/.changes/unreleased/Dependencies-20231204-193730.yaml new file mode 100644 index 000000000..b1d803288 --- /dev/null +++ b/.changes/unreleased/Dependencies-20231204-193730.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update freezegun requirement from ~=1.2 to ~=1.3" +time: 2023-12-04T19:37:30.00000Z +custom: + Author: dependabot[bot] + PR: 681 diff --git a/.changes/unreleased/Dependencies-20231212-195417.yaml b/.changes/unreleased/Dependencies-20231212-195417.yaml new file mode 100644 index 000000000..2117f6ead --- /dev/null +++ b/.changes/unreleased/Dependencies-20231212-195417.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update black requirement from ~=23.11 to ~=23.12" +time: 2023-12-12T19:54:17.00000Z +custom: + Author: dependabot[bot] + PR: 688 diff --git a/.changes/unreleased/Dependencies-20240118-095025.yaml b/.changes/unreleased/Dependencies-20240118-095025.yaml new file mode 100644 index 000000000..ba61481ba --- /dev/null +++ b/.changes/unreleased/Dependencies-20240118-095025.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: upgrade redshift driver to 2.0.918 +time: 2024-01-18T09:50:25.871857-08:00 +custom: + Author: colin-rogers-dbt + PR: "700" diff --git a/.changes/unreleased/Dependencies-20240124-111727.yaml b/.changes/unreleased/Dependencies-20240124-111727.yaml new file mode 100644 index 000000000..f5874984a --- /dev/null +++ b/.changes/unreleased/Dependencies-20240124-111727.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Migrate to dbt-common and dbt-adapters +time: 2024-01-24T11:17:27.580348-08:00 +custom: + Author: colin-rogers-dbt + PR: "706" diff --git a/.changes/unreleased/Features-20230803-111146.yaml b/.changes/unreleased/Features-20230803-111146.yaml deleted file mode 100644 index 098e8888e..000000000 --- a/.changes/unreleased/Features-20230803-111146.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support DISTSTYLE for seeds -time: 2023-08-03T11:11:46.571996-06:00 -custom: - Author: WillAyd - Issue: "255" diff --git a/.changes/unreleased/Features-20231030-101055.yaml b/.changes/unreleased/Features-20231030-101055.yaml new file mode 100644 index 000000000..8648762e2 --- /dev/null +++ b/.changes/unreleased/Features-20231030-101055.yaml @@ -0,0 +1,6 @@ +kind: Features +body: allow user to set debug level for redshift-connector via env var +time: 2023-10-30T10:10:55.976191-07:00 +custom: + Author: colin-rogers-dbt + Issue: "650" diff --git a/.changes/unreleased/Fixes-20230802-103350.yaml b/.changes/unreleased/Fixes-20230802-103350.yaml deleted file mode 100644 index 5c5a1fe79..000000000 --- a/.changes/unreleased/Fixes-20230802-103350.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Insert values of `merge_exclude_columns` when not matched -time: 2023-08-02T10:33:50.107228-06:00 -custom: - Author: dbeatty10 - Issue: "563" diff --git a/.changes/unreleased/Fixes-20231025-203732.yaml b/.changes/unreleased/Fixes-20231025-203732.yaml new file mode 100644 index 000000000..9e6bf1af7 --- /dev/null +++ b/.changes/unreleased/Fixes-20231025-203732.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix parsing of database results for materialized view auto refresh +time: 2023-10-25T20:37:32.191259-04:00 +custom: + Author: mikealfare + Issue: "643" diff --git a/.changes/unreleased/Fixes-20231026-164623.yaml b/.changes/unreleased/Fixes-20231026-164623.yaml new file mode 100644 index 000000000..f06eff381 --- /dev/null +++ b/.changes/unreleased/Fixes-20231026-164623.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix describe_materialized_view for Redshift Serverless +time: 2023-10-26T16:46:23.253837-06:00 +custom: + Author: reptillicus + Issue: "641" diff --git a/.changes/unreleased/Fixes-20231030-234315.yaml b/.changes/unreleased/Fixes-20231030-234315.yaml new file mode 100644 index 000000000..823e252dd --- /dev/null +++ b/.changes/unreleased/Fixes-20231030-234315.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Catalog queries assign the appropriate type for materialized views +time: 2023-10-30T23:43:15.281772-04:00 +custom: + Author: mikealfare + Issue: "652" diff --git a/.changes/unreleased/Fixes-20231103-181357.yaml b/.changes/unreleased/Fixes-20231103-181357.yaml new file mode 100644 index 000000000..2995560d2 --- /dev/null +++ b/.changes/unreleased/Fixes-20231103-181357.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix handling of `backup` parameter during config change monitoring +time: 2023-11-03T18:13:57.401994-04:00 +custom: + Author: mikealfare + Issue: "621" diff --git a/.changes/unreleased/Fixes-20240206-132326.yaml b/.changes/unreleased/Fixes-20240206-132326.yaml new file mode 100644 index 000000000..45b3a74c8 --- /dev/null +++ b/.changes/unreleased/Fixes-20240206-132326.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Initialize sqlparse.Lexer to resolve issue with `dbt docs generate` that includes + external tables +time: 2024-02-06T13:23:26.061133-05:00 +custom: + Author: mikealfare + Issue: "710" diff --git a/.changes/unreleased/Under the Hood-20230724-164439.yaml b/.changes/unreleased/Under the Hood-20230724-164439.yaml deleted file mode 100644 index 59f929e69..000000000 --- a/.changes/unreleased/Under the Hood-20230724-164439.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update stale workflow to use the centralized version -time: 2023-07-24T16:44:39.683995-04:00 -custom: - Author: mikealfare - Issue: "552" diff --git a/.changes/unreleased/Under the Hood-20231119-132157.yaml b/.changes/unreleased/Under the Hood-20231119-132157.yaml new file mode 100644 index 000000000..760c08ccf --- /dev/null +++ b/.changes/unreleased/Under the Hood-20231119-132157.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add tests for --empty flag +time: 2023-11-19T13:21:57.588501-05:00 +custom: + Author: michelleark + Issue: "667" diff --git a/.changes/unreleased/Under the Hood-20240102-152425.yaml b/.changes/unreleased/Under the Hood-20240102-152425.yaml new file mode 100644 index 000000000..23a3eeb46 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240102-152425.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update base adapter references as part of decoupling migration +time: 2024-01-02T15:24:25.890421-08:00 +custom: + Author: colin-rogers-dbt VersusFacit + Issue: "698" diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml new file mode 100644 index 000000000..00a098df8 --- /dev/null +++ b/.github/workflows/docs-issues.yml @@ -0,0 +1,43 @@ +# **what?** +# Open an issue in docs.getdbt.com when a PR is labeled `user docs` + +# **why?** +# To reduce barriers for keeping docs up to date + +# **when?** +# When a PR is labeled `user docs` and is merged. Runs on pull_request_target to run off the workflow already merged, +# not the workflow that existed on the PR branch. This allows old PRs to get comments. + + +name: Open issues in docs.getdbt.com repo when a PR is labeled +run-name: "Open an issue in docs.getdbt.com for PR #${{ github.event.pull_request.number }}" + +on: + pull_request_target: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # opens new issues + pull-requests: write # comments on PRs + + +jobs: + open_issues: + # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`. Otherwise it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.pull_request.merged == true) && + ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs')) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} PR #${{ github.event.pull_request.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit diff --git a/.github/workflows/jira-creation.yml b/.github/workflows/jira-creation.yml deleted file mode 100644 index 2611a8bdd..000000000 --- a/.github/workflows/jira-creation.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# Mirrors issues into Jira. Includes the information: title, -# GitHub Issue ID and URL - -# **why?** -# Jira is our tool for tracking and we need to see these issues in there - -# **when?** -# On issue creation or when an issue is labeled `Jira` - -name: Jira Issue Creation - -on: - issues: - types: [opened, labeled] - -permissions: - issues: write - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-creation.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/jira-label.yml b/.github/workflows/jira-label.yml deleted file mode 100644 index 1637cbe38..000000000 --- a/.github/workflows/jira-label.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# Calls mirroring Jira label Action. Includes adding a new label -# to an existing issue or removing a label as well - -# **why?** -# Jira is our tool for tracking and we need to see these labels in there - -# **when?** -# On labels being added or removed from issues - -name: Jira Label Mirroring - -on: - issues: - types: [labeled, unlabeled] - -permissions: - issues: read - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-label.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/jira-transition.yml b/.github/workflows/jira-transition.yml deleted file mode 100644 index 99158a15f..000000000 --- a/.github/workflows/jira-transition.yml +++ /dev/null @@ -1,29 +0,0 @@ -# **what?** -# Transition a Jira issue to a new state -# Only supports these GitHub Issue transitions: -# closed, deleted, reopened - -# **why?** -# Jira needs to be kept up-to-date - -# **when?** -# On issue closing, deletion, reopened - -name: Jira Issue Transition - -on: - issues: - types: [closed, deleted, reopened] - -# no special access is needed -permissions: read-all - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-transition.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/repository-cleanup.yml b/.github/workflows/repository-cleanup.yml new file mode 100644 index 000000000..c1d780281 --- /dev/null +++ b/.github/workflows/repository-cleanup.yml @@ -0,0 +1,30 @@ +# **what?** +# Cleanup branches left over from automation and testing. Also cleanup +# draft releases from release testing. + +# **why?** +# The automations are leaving behind branches and releases that clutter +# the repository. Sometimes we need them to debug processes so we don't +# want them immediately deleted. Running on Saturday to avoid running +# at the same time as an actual release to prevent breaking a release +# mid-release. + +# **when?** +# Mainly on a schedule of 12:00 Saturday. +# Manual trigger can also run on demand + +name: Repository Cleanup + +on: + schedule: + - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above + + workflow_dispatch: # for manual triggering + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 874bd74c8..f15b401d1 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.7.0a1" +version = "1.8.0a1" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 500de430f..b0fc0825d 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -1,22 +1,23 @@ import re from multiprocessing import Lock from contextlib import contextmanager -from typing import NewType, Tuple, Union, Optional, List +from typing import Tuple, Union, Optional, List from dataclasses import dataclass, field import agate import sqlparse import redshift_connector +from dbt.adapters.exceptions import FailedToConnectError +from dbt_common.clients import agate_helper from redshift_connector.utils.oids import get_datatype_name from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import AdapterResponse, Connection, Credentials -from dbt.contracts.util import Replaceable -from dbt.dataclass_schema import FieldEncoder, dbtClassMixin, StrEnum, ValidationError -from dbt.events import AdapterLogger -from dbt.exceptions import DbtRuntimeError, CompilationError -import dbt.flags -from dbt.helper_types import Port +from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials +from dbt.adapters.events.logging import AdapterLogger +from dbt_common.contracts.util import Replaceable +from dbt_common.dataclass_schema import dbtClassMixin, StrEnum, ValidationError +from dbt_common.helper_types import Port +from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError class SSLConfigError(CompilationError): @@ -33,21 +34,6 @@ def get_message(self) -> str: logger = AdapterLogger("Redshift") -drop_lock: Lock = dbt.flags.MP_CONTEXT.Lock() # type: ignore - - -IAMDuration = NewType("IAMDuration", int) - - -class IAMDurationEncoder(FieldEncoder): - @property - def json_schema(self): - return {"type": "integer", "minimum": 0, "maximum": 65535} - - -dbtClassMixin.register_field_encoders({IAMDuration: IAMDurationEncoder()}) - - class RedshiftConnectionMethod(StrEnum): DATABASE = "database" IAM = "iam" @@ -197,7 +183,7 @@ def get_connect_method(self): # this requirement is really annoying to encode into json schema, # so validate it here if self.credentials.password is None: - raise dbt.exceptions.FailedToConnectError( + raise FailedToConnectError( "'password' field is required for 'database' credentials" ) @@ -216,7 +202,7 @@ def connect(): elif method == RedshiftConnectionMethod.IAM: if not self.credentials.cluster_id and "serverless" not in self.credentials.host: - raise dbt.exceptions.FailedToConnectError( + raise FailedToConnectError( "Failed to use IAM method. 'cluster_id' must be provided for provisioned cluster. " "'host' must be provided for serverless endpoint." ) @@ -239,9 +225,7 @@ def connect(): return c else: - raise dbt.exceptions.FailedToConnectError( - "Invalid 'method' in profile: '{}'".format(method) - ) + raise FailedToConnectError("Invalid 'method' in profile: '{}'".format(method)) return connect @@ -252,8 +236,9 @@ class RedshiftConnectionManager(SQLConnectionManager): def _get_backend_pid(self): sql = "select pg_backend_pid()" _, cursor = self.add_query(sql) + res = cursor.fetchone() - return res + return res[0] def cancel(self, connection: Connection): try: @@ -265,9 +250,10 @@ def cancel(self, connection: Connection): raise sql = f"select pg_terminate_backend({pid})" - _, cursor = self.add_query(sql) - res = cursor.fetchone() - logger.debug(f"Cancel query '{connection.name}': {res}") + cursor = connection.handle.cursor() + logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") + logger.debug(sql) + cursor.execute(sql) @classmethod def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse: @@ -288,16 +274,16 @@ def exception_handler(self, sql): err_msg = str(e).strip() logger.debug(f"Redshift error: {err_msg}") self.rollback_if_open() - raise dbt.exceptions.DbtDatabaseError(err_msg) from e + raise DbtDatabaseError(err_msg) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() # Raise DBT native exceptions as is. - if isinstance(e, dbt.exceptions.DbtRuntimeError): + if isinstance(e, DbtRuntimeError): raise - raise dbt.exceptions.DbtRuntimeError(str(e)) from e + raise DbtRuntimeError(str(e)) from e @contextmanager def fresh_transaction(self): @@ -307,6 +293,8 @@ def fresh_transaction(self): See drop_relation in RedshiftAdapter for more information. """ + drop_lock: Lock = self.lock + with drop_lock: connection = self.get_thread_connection() @@ -359,13 +347,14 @@ def execute( if fetch: table = self.get_result_from_cursor(cursor, limit) else: - table = dbt.clients.agate_helper.empty_table() + table = agate_helper.empty_table() return response, table def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False): connection = None cursor = None + self._initialize_sqlparse_lexer() queries = sqlparse.split(sql) for query in queries: @@ -397,3 +386,14 @@ def get_credentials(cls, credentials): @classmethod def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: return get_datatype_name(type_code) + + @staticmethod + def _initialize_sqlparse_lexer(): + """ + Resolves: https://github.com/dbt-labs/dbt-redshift/issues/710 + Implementation of this fix: https://github.com/dbt-labs/dbt-core/pull/8215 + """ + from sqlparse.lexer import Lexer # type: ignore + + if hasattr(Lexer, "get_default_instance"): + Lexer.get_default_instance() diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index ae9f18392..b41308db0 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,22 +1,29 @@ +import os from dataclasses import dataclass +from dbt_common.contracts.constraints import ConstraintType from typing import Optional, Set, Any, Dict, Type from collections import namedtuple from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.sql import SQLAdapter -from dbt.contracts.connection import AdapterResponse -from dbt.contracts.graph.nodes import ConstraintType -from dbt.events import AdapterLogger +from dbt.adapters.contracts.connection import AdapterResponse +from dbt.adapters.events.logging import AdapterLogger -import dbt.exceptions +import dbt_common.exceptions from dbt.adapters.redshift import RedshiftConnectionManager, RedshiftRelation - logger = AdapterLogger("Redshift") - +packages = ["redshift_connector", "redshift_connector.core"] +if os.getenv("DBT_REDSHIFT_CONNECTOR_DEBUG_LOGGING"): + level = "DEBUG" +else: + level = "ERROR" +for package in packages: + logger.debug(f"Setting {package} to {level}") + logger.set_adapter_dependency_log_level(package, level) GET_RELATIONS_MACRO_NAME = "redshift__get_relations" @@ -89,7 +96,7 @@ def verify_database(self, database): ra3_node = self.config.credentials.ra3_node if database.lower() != expected.lower() and not ra3_node: - raise dbt.exceptions.NotImplementedError( + raise dbt_common.exceptions.NotImplementedError( "Cross-db references allowed only in RA3.* node. ({} vs {})".format( database, expected ) @@ -102,9 +109,9 @@ def _get_catalog_schemas(self, manifest): schemas = super(SQLAdapter, self)._get_catalog_schemas(manifest) try: return schemas.flatten(allow_multiple_databases=self.config.credentials.ra3_node) - except dbt.exceptions.DbtRuntimeError as exc: + except dbt_common.exceptions.DbtRuntimeError as exc: msg = f"Cross-db references allowed only in {self.type()} RA3.* node. Got {exc.msg}" - raise dbt.exceptions.CompilationError(msg) + raise dbt_common.exceptions.CompilationError(msg) def valid_incremental_strategies(self): """The set of standard builtin strategies which this adapter supports out-of-the-box. diff --git a/dbt/adapters/redshift/relation.py b/dbt/adapters/redshift/relation.py index 0ef4fe276..7255288e2 100644 --- a/dbt/adapters/redshift/relation.py +++ b/dbt/adapters/redshift/relation.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from dbt.adapters.contracts.relation import RelationConfig from typing import Optional from dbt.adapters.base.relation import BaseRelation @@ -7,16 +8,13 @@ RelationConfigChangeAction, RelationResults, ) -from dbt.context.providers import RuntimeConfigObject -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import RelationType -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.base import RelationType +from dbt_common.exceptions import DbtRuntimeError from dbt.adapters.redshift.relation_configs import ( RedshiftMaterializedViewConfig, RedshiftMaterializedViewConfigChangeset, RedshiftAutoRefreshConfigChange, - RedshiftBackupConfigChange, RedshiftDistConfigChange, RedshiftSortConfigChange, RedshiftIncludePolicy, @@ -32,6 +30,17 @@ class RedshiftRelation(BaseRelation): relation_configs = { RelationType.MaterializedView.value: RedshiftMaterializedViewConfig, } + renameable_relations = frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + replaceable_relations = frozenset( + { + RelationType.View, + } + ) def __post_init__(self): # Check for length of Redshift table/view names. @@ -50,31 +59,28 @@ def relation_max_name_length(self): return MAX_CHARACTERS_IN_IDENTIFIER @classmethod - def from_runtime_config(cls, runtime_config: RuntimeConfigObject) -> RelationConfigBase: - model_node: ModelNode = runtime_config.model - relation_type: str = model_node.config.materialized + def from_config(cls, config: RelationConfig) -> RelationConfigBase: + relation_type: str = config.config.materialized # type: ignore if relation_config := cls.relation_configs.get(relation_type): - return relation_config.from_model_node(model_node) + return relation_config.from_relation_config(config) raise DbtRuntimeError( - f"from_runtime_config() is not supported for the provided relation type: {relation_type}" + f"from_config() is not supported for the provided relation type: {relation_type}" ) @classmethod def materialized_view_config_changeset( - cls, relation_results: RelationResults, runtime_config: RuntimeConfigObject + cls, relation_results: RelationResults, relation_config: RelationConfig ) -> Optional[RedshiftMaterializedViewConfigChangeset]: config_change_collection = RedshiftMaterializedViewConfigChangeset() existing_materialized_view = RedshiftMaterializedViewConfig.from_relation_results( relation_results ) - new_materialized_view = RedshiftMaterializedViewConfig.from_model_node( - runtime_config.model + new_materialized_view = RedshiftMaterializedViewConfig.from_relation_config( + relation_config ) - assert isinstance(existing_materialized_view, RedshiftMaterializedViewConfig) - assert isinstance(new_materialized_view, RedshiftMaterializedViewConfig) if new_materialized_view.autorefresh != existing_materialized_view.autorefresh: config_change_collection.autorefresh = RedshiftAutoRefreshConfigChange( @@ -82,12 +88,6 @@ def materialized_view_config_changeset( context=new_materialized_view.autorefresh, ) - if new_materialized_view.backup != existing_materialized_view.backup: - config_change_collection.backup = RedshiftBackupConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.backup, - ) - if new_materialized_view.dist != existing_materialized_view.dist: config_change_collection.dist = RedshiftDistConfigChange( action=RelationConfigChangeAction.alter, diff --git a/dbt/adapters/redshift/relation_configs/__init__.py b/dbt/adapters/redshift/relation_configs/__init__.py index 26e36c86c..0bc69ef4a 100644 --- a/dbt/adapters/redshift/relation_configs/__init__.py +++ b/dbt/adapters/redshift/relation_configs/__init__.py @@ -9,7 +9,6 @@ from dbt.adapters.redshift.relation_configs.materialized_view import ( RedshiftMaterializedViewConfig, RedshiftAutoRefreshConfigChange, - RedshiftBackupConfigChange, RedshiftMaterializedViewConfigChangeset, ) from dbt.adapters.redshift.relation_configs.policies import ( diff --git a/dbt/adapters/redshift/relation_configs/base.py b/dbt/adapters/redshift/relation_configs/base.py index ebbd46b1b..c4faab664 100644 --- a/dbt/adapters/redshift/relation_configs/base.py +++ b/dbt/adapters/redshift/relation_configs/base.py @@ -1,14 +1,14 @@ from dataclasses import dataclass -from typing import Optional +from typing import Optional, Dict import agate from dbt.adapters.base.relation import Policy +from dbt.adapters.contracts.relation import ComponentName, RelationConfig from dbt.adapters.relation_configs import ( RelationConfigBase, RelationResults, ) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName +from typing_extensions import Self from dbt.adapters.redshift.relation_configs.policies import ( RedshiftIncludePolicy, @@ -31,25 +31,25 @@ def quote_policy(cls) -> Policy: return RedshiftQuotePolicy() @classmethod - def from_model_node(cls, model_node: ModelNode) -> "RelationConfigBase": - relation_config = cls.parse_model_node(model_node) - relation = cls.from_dict(relation_config) - return relation + def from_relation_config(cls, relation_config: RelationConfig) -> Self: + relation_config_dict = cls.parse_relation_config(relation_config) + relation = cls.from_dict(relation_config_dict) + return relation # type: ignore @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict: raise NotImplementedError( - "`parse_model_node()` needs to be implemented on this RelationConfigBase instance" + "`parse_relation_config()` needs to be implemented on this RelationConfigBase instance" ) @classmethod - def from_relation_results(cls, relation_results: RelationResults) -> "RelationConfigBase": + def from_relation_results(cls, relation_results: RelationResults) -> Self: relation_config = cls.parse_relation_results(relation_results) relation = cls.from_dict(relation_config) - return relation + return relation # type: ignore @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> dict: + def parse_relation_results(cls, relation_results: RelationResults) -> Dict: raise NotImplementedError( "`parse_relation_results()` needs to be implemented on this RelationConfigBase instance" ) diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/relation_configs/dist.py index 668f3f65a..c41eda578 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/relation_configs/dist.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from typing import Optional, Set +from dbt.adapters.contracts.relation import RelationConfig +from typing import Optional, Set, Dict import agate from dbt.adapters.relation_configs import ( @@ -8,9 +9,9 @@ RelationConfigValidationMixin, RelationConfigValidationRule, ) -from dbt.contracts.graph.nodes import ModelNode -from dbt.dataclass_schema import StrEnum -from dbt.exceptions import DbtRuntimeError +from dbt_common.dataclass_schema import StrEnum +from dbt_common.exceptions import DbtRuntimeError +from typing_extensions import Self from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase @@ -65,21 +66,21 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftDistConfig": + def from_dict(cls, config_dict) -> Self: kwargs_dict = { "diststyle": config_dict.get("diststyle"), "distkey": config_dict.get("distkey"), } - dist: "RedshiftDistConfig" = super().from_dict(kwargs_dict) # type: ignore + dist: Self = super().from_dict(kwargs_dict) # type: ignore return dist @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: + def parse_relation_config(cls, relation_config: RelationConfig) -> dict: """ Translate ModelNode objects from the user-provided config into a standard dictionary. Args: - model_node: the description of the distkey and diststyle from the user in this format: + relation_config: the description of the distkey and diststyle from the user in this format: { "dist": any("auto", "even", "all") or "" @@ -87,7 +88,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist = model_node.config.extra.get("dist", "") + dist = relation_config.config.extra.get("dist", "") # type: ignore diststyle = dist.lower() @@ -107,7 +108,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: return config @classmethod - def parse_relation_results(cls, relation_results_entry: agate.Row) -> dict: + def parse_relation_results(cls, relation_results_entry: agate.Row) -> Dict: """ Translate agate objects from the database into a standard dictionary. diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index 5c76e949b..05f4b170d 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Optional, Set +from dataclasses import dataclass, field +from typing import Optional, Set, Dict, Any import agate from dbt.adapters.relation_configs import ( @@ -8,9 +8,9 @@ RelationConfigValidationMixin, RelationConfigValidationRule, ) -from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.contracts.relation import ComponentName, RelationConfig +from dbt_common.exceptions import DbtRuntimeError +from typing_extensions import Self from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase from dbt.adapters.redshift.relation_configs.dist import ( @@ -23,6 +23,7 @@ RedshiftSortConfig, RedshiftSortConfigChange, ) +from dbt.adapters.redshift.utility import evaluate_bool @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -55,7 +56,7 @@ class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigV schema_name: str database_name: str query: str - backup: bool = True + backup: bool = field(default=True, compare=False, hash=False) dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) sort: RedshiftSortConfig = RedshiftSortConfig() autorefresh: bool = False @@ -94,7 +95,7 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftMaterializedViewConfig": + def from_dict(cls, config_dict) -> Self: kwargs_dict = { "mv_name": cls._render_part(ComponentName.Identifier, config_dict.get("mv_name")), "schema_name": cls._render_part(ComponentName.Schema, config_dict.get("schema_name")), @@ -113,32 +114,39 @@ def from_dict(cls, config_dict) -> "RedshiftMaterializedViewConfig": if sort := config_dict.get("sort"): kwargs_dict.update({"sort": RedshiftSortConfig.from_dict(sort)}) - materialized_view: "RedshiftMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore + materialized_view: Self = super().from_dict(kwargs_dict) # type: ignore return materialized_view @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: - config_dict = { - "mv_name": model_node.identifier, - "schema_name": model_node.schema, - "database_name": model_node.database, - "backup": model_node.config.extra.get("backup"), - "autorefresh": model_node.config.extra.get("auto_refresh"), + def parse_relation_config(cls, config: RelationConfig) -> Dict[str, Any]: + config_dict: Dict[str, Any] = { + "mv_name": config.identifier, + "schema_name": config.schema, + "database_name": config.database, } - if query := model_node.compiled_code: + # backup/autorefresh can be bools or strings + backup_value = config.config.extra.get("backup") # type: ignore + if backup_value is not None: + config_dict["backup"] = evaluate_bool(backup_value) + + autorefresh_value = config.config.extra.get("auto_refresh") # type: ignore + if autorefresh_value is not None: + config_dict["autorefresh"] = evaluate_bool(autorefresh_value) + + if query := config.compiled_code: # type: ignore config_dict.update({"query": query.strip()}) - if model_node.config.get("dist"): - config_dict.update({"dist": RedshiftDistConfig.parse_model_node(model_node)}) + if config.config.get("dist"): # type: ignore + config_dict.update({"dist": RedshiftDistConfig.parse_relation_config(config)}) - if model_node.config.get("sort"): - config_dict.update({"sort": RedshiftSortConfig.parse_model_node(model_node)}) + if config.config.get("sort"): # type: ignore + config_dict.update({"sort": RedshiftSortConfig.parse_relation_config(config)}) return config_dict @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> dict: + def parse_relation_results(cls, relation_results: RelationResults) -> Dict: """ Translate agate objects from the database into a standard dictionary. @@ -174,10 +182,14 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict: "mv_name": materialized_view.get("table"), "schema_name": materialized_view.get("schema"), "database_name": materialized_view.get("database"), - "autorefresh": materialized_view.get("autorefresh"), "query": cls._parse_query(query.get("definition")), } + autorefresh_value = materialized_view.get("autorefresh") + if autorefresh_value is not None: + bool_filter = {"t": True, "f": False} + config_dict["autorefresh"] = bool_filter.get(autorefresh_value, autorefresh_value) + # the default for materialized views differs from the default for diststyle in general # only set it if we got a value if materialized_view.get("diststyle"): @@ -229,18 +241,8 @@ def requires_full_refresh(self) -> bool: return False -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class RedshiftBackupConfigChange(RelationConfigChange): - context: Optional[bool] = None - - @property - def requires_full_refresh(self) -> bool: - return True - - @dataclass class RedshiftMaterializedViewConfigChangeset: - backup: Optional[RedshiftBackupConfigChange] = None dist: Optional[RedshiftDistConfigChange] = None sort: Optional[RedshiftSortConfigChange] = None autorefresh: Optional[RedshiftAutoRefreshConfigChange] = None @@ -250,7 +252,6 @@ def requires_full_refresh(self) -> bool: return any( { self.autorefresh.requires_full_refresh if self.autorefresh else False, - self.backup.requires_full_refresh if self.backup else False, self.dist.requires_full_refresh if self.dist else False, self.sort.requires_full_refresh if self.sort else False, } @@ -260,7 +261,6 @@ def requires_full_refresh(self) -> bool: def has_changes(self) -> bool: return any( { - self.backup if self.backup else False, self.dist if self.dist else False, self.sort if self.sort else False, self.autorefresh if self.autorefresh else False, diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/relation_configs/sort.py index 58104b65f..e44784c2f 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/relation_configs/sort.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from typing import Optional, FrozenSet, Set +from dbt.adapters.contracts.relation import RelationConfig +from typing import Optional, FrozenSet, Set, Dict, Any import agate from dbt.adapters.relation_configs import ( @@ -8,9 +9,9 @@ RelationConfigValidationMixin, RelationConfigValidationRule, ) -from dbt.contracts.graph.nodes import ModelNode -from dbt.dataclass_schema import StrEnum -from dbt.exceptions import DbtRuntimeError +from dbt_common.dataclass_schema import StrEnum +from dbt_common.exceptions import DbtRuntimeError +from typing_extensions import Self from dbt.adapters.redshift.relation_configs.base import RedshiftRelationConfigBase @@ -97,21 +98,21 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]: } @classmethod - def from_dict(cls, config_dict) -> "RedshiftSortConfig": + def from_dict(cls, config_dict) -> Self: kwargs_dict = { "sortstyle": config_dict.get("sortstyle"), "sortkey": frozenset(column for column in config_dict.get("sortkey", {})), } - sort: "RedshiftSortConfig" = super().from_dict(kwargs_dict) # type: ignore - return sort + sort: Self = super().from_dict(kwargs_dict) # type: ignore + return sort # type: ignore @classmethod - def parse_model_node(cls, model_node: ModelNode) -> dict: + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: """ Translate ModelNode objects from the user-provided config into a standard dictionary. Args: - model_node: the description of the sortkey and sortstyle from the user in this format: + relation_config: the description of the sortkey and sortstyle from the user in this format: { "sort_key": "" or [""] or ["",...] @@ -122,10 +123,10 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: """ config_dict = {} - if sortstyle := model_node.config.extra.get("sort_type"): + if sortstyle := relation_config.config.extra.get("sort_type"): # type: ignore config_dict.update({"sortstyle": sortstyle.lower()}) - if sortkey := model_node.config.extra.get("sort"): + if sortkey := relation_config.config.extra.get("sort"): # type: ignore # we allow users to specify the `sort_key` as a string if it's a single column if isinstance(sortkey, str): sortkey = [sortkey] diff --git a/dbt/adapters/redshift/utility.py b/dbt/adapters/redshift/utility.py new file mode 100644 index 000000000..64f5e9cd8 --- /dev/null +++ b/dbt/adapters/redshift/utility.py @@ -0,0 +1,25 @@ +from typing import Union + + +def evaluate_bool_str(value: str) -> bool: + value = value.strip().lower() + if value == "true": + return True + elif value == "false": + return False + else: + raise ValueError(f"Invalid boolean string value: {value}") + + +def evaluate_bool(value: Union[str, bool]) -> bool: + if not value: + return False + if isinstance(value, bool): + return value + elif isinstance(value, str): + return evaluate_bool_str(value) + else: + raise TypeError( + f"Invalid type for boolean evaluation, " + f"expecting boolean or str, recieved: {type(value)}" + ) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 491241d7a..9eb6e1cee 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -316,12 +316,3 @@ {% endif %} {% endmacro %} - - -{% macro redshift__get_drop_relation_sql(relation) %} - {%- if relation.is_materialized_view -%} - {{ redshift__drop_materialized_view(relation) }} - {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endif -%} -{% endmacro %} diff --git a/dbt/include/redshift/macros/catalog.sql b/dbt/include/redshift/macros/catalog.sql index b34460035..69dc71713 100644 --- a/dbt/include/redshift/macros/catalog.sql +++ b/dbt/include/redshift/macros/catalog.sql @@ -23,13 +23,28 @@ order by "column_index" ), + materialized_views as ( + select + table_schema as nspname, + table_name as relname + from information_schema.views + where ( + {%- for schema in schemas -%} + upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + and table_catalog = '{{ database }}' + and view_definition ilike '%create materialized view%' + ), + early_binding as ( select '{{ database }}'::varchar as table_database, sch.nspname as table_schema, tbl.relname as table_name, - case tbl.relkind - when 'v' then 'VIEW' + case + when tbl.relkind = 'v' and materialized_views.relname is not null then 'MATERIALIZED VIEW' + when tbl.relkind = 'v' then 'VIEW' else 'BASE TABLE' end as table_type, tbl_desc.description as table_comment, @@ -43,6 +58,7 @@ join pg_catalog.pg_attribute col on col.attrelid = tbl.oid left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) + left outer join materialized_views on (materialized_views.nspname = sch.nspname and materialized_views.relname = tbl.relname) where ( {%- for schema in schemas -%} upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} diff --git a/dbt/include/redshift/macros/materializations/materialized_view.sql b/dbt/include/redshift/macros/materializations/materialized_view.sql index 736eaacca..9b1ef2d50 100644 --- a/dbt/include/redshift/macros/materializations/materialized_view.sql +++ b/dbt/include/redshift/macros/materializations/materialized_view.sql @@ -1,106 +1,5 @@ -{% macro redshift__get_alter_materialized_view_as_sql( - relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) %} - - -- apply a full refresh immediately if needed - {% if configuration_changes.requires_full_refresh %} - - {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }} - - -- otherwise apply individual changes as needed - {% else %} - - {%- set autorefresh = configuration_changes.autorefresh -%} - {%- if autorefresh -%}{{- log('Applying UPDATE AUTOREFRESH to: ' ~ relation) -}}{%- endif -%} - - alter materialized view {{ relation }} - auto refresh {% if autorefresh.context %}yes{% else %}no{% endif %} - - {%- endif -%} - -{% endmacro %} - - -{% macro redshift__get_create_materialized_view_as_sql(relation, sql) %} - - {%- set materialized_view = relation.from_runtime_config(config) -%} - - create materialized view {{ materialized_view.path }} - backup {% if materialized_view.backup %}yes{% else %}no{% endif %} - diststyle {{ materialized_view.dist.diststyle }} - {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} - {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} - auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} - as ( - {{ materialized_view.query }} - ); - -{% endmacro %} - - -{% macro redshift__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %} - {{ redshift__get_drop_relation_sql(existing_relation) }}; - {{ get_create_materialized_view_as_sql(relation, sql) }} -{% endmacro %} - - {% macro redshift__get_materialized_view_configuration_changes(existing_relation, new_config) %} {% set _existing_materialized_view = redshift__describe_materialized_view(existing_relation) %} - {% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config) %} + {% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config.model) %} {% do return(_configuration_changes) %} {% endmacro %} - - -{% macro redshift__refresh_materialized_view(relation) -%} - refresh materialized view {{ relation }} -{% endmacro %} - - -{% macro redshift__describe_materialized_view(relation) %} - {#- - These need to be separate queries because redshift will not let you run queries - against svv_table_info and pg_views in the same query. The same is true of svv_redshift_columns. - -#} - - {%- set _materialized_view_sql -%} - select - tb.database, - tb.schema, - tb.table, - tb.diststyle, - tb.sortkey1, - mv.autorefresh - from svv_table_info tb - left join stv_mv_info mv - on mv.db_name = tb.database - and mv.schema = tb.schema - and mv.name = tb.table - where tb.table ilike '{{ relation.identifier }}' - and tb.schema ilike '{{ relation.schema }}' - and tb.database ilike '{{ relation.database }}' - {%- endset %} - {% set _materialized_view = run_query(_materialized_view_sql) %} - - {%- set _query_sql -%} - select - vw.definition - from pg_views vw - where vw.viewname = '{{ relation.identifier }}' - and vw.schemaname = '{{ relation.schema }}' - and vw.definition ilike '%create materialized view%' - {%- endset %} - {% set _query = run_query(_query_sql) %} - - {% do return({'materialized_view': _materialized_view, 'query': _query}) %} - -{% endmacro %} - - -{% macro redshift__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} -{%- endmacro %} diff --git a/dbt/include/redshift/macros/materializations/table.sql b/dbt/include/redshift/macros/materializations/table.sql new file mode 100644 index 000000000..907c83874 --- /dev/null +++ b/dbt/include/redshift/macros/materializations/table.sql @@ -0,0 +1,69 @@ +{% materialization table, adapter='redshift' %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {% if existing_relation.can_be_renamed %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% else %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + {% endif %} + {% endif %} + + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/redshift/macros/materializations/view.sql b/dbt/include/redshift/macros/materializations/view.sql new file mode 100644 index 000000000..f353f913f --- /dev/null +++ b/dbt/include/redshift/macros/materializations/view.sql @@ -0,0 +1,77 @@ +{%- materialization view, adapter='redshift' -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {% if existing_relation.can_be_renamed %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% else %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/relations.sql index 28c6bc377..6d83c36b9 100644 --- a/dbt/include/redshift/macros/relations.sql +++ b/dbt/include/redshift/macros/relations.sql @@ -24,6 +24,7 @@ with from pg_depend left join pg_rewrite on pg_depend.objid = pg_rewrite.oid + where coalesce(pg_rewrite.ev_class, pg_depend.objid) != pg_depend.refobjid ) select distinct @@ -36,7 +37,6 @@ join relation ref on dependency.ref_relation_id = ref.relation_id join relation dep on dependency.dep_relation_id = dep.relation_id -where ref.relation_name != dep.relation_name {%- endcall -%} diff --git a/dbt/include/redshift/macros/relations/materialized_view/alter.sql b/dbt/include/redshift/macros/relations/materialized_view/alter.sql new file mode 100644 index 000000000..7f0379847 --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view/alter.sql @@ -0,0 +1,26 @@ +{% macro redshift__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + + -- apply a full refresh immediately if needed + {% if configuration_changes.requires_full_refresh %} + + {{ get_replace_sql(existing_relation, relation, sql) }} + + -- otherwise apply individual changes as needed + {% else %} + + {%- set autorefresh = configuration_changes.autorefresh -%} + {%- if autorefresh -%}{{- log('Applying UPDATE AUTOREFRESH to: ' ~ relation) -}}{%- endif -%} + + alter materialized view {{ relation }} + auto refresh {% if autorefresh.context %}yes{% else %}no{% endif %} + + {%- endif -%} + +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/create.sql b/dbt/include/redshift/macros/relations/materialized_view/create.sql new file mode 100644 index 000000000..06fe2b6b5 --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view/create.sql @@ -0,0 +1,15 @@ +{% macro redshift__get_create_materialized_view_as_sql(relation, sql) %} + + {%- set materialized_view = relation.from_config(config.model) -%} + + create materialized view {{ materialized_view.path }} + backup {% if materialized_view.backup %}yes{% else %}no{% endif %} + diststyle {{ materialized_view.dist.diststyle }} + {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} + {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} + auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} + as ( + {{ materialized_view.query }} + ); + +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/describe.sql b/dbt/include/redshift/macros/relations/materialized_view/describe.sql new file mode 100644 index 000000000..a56d0756e --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view/describe.sql @@ -0,0 +1,39 @@ +{% macro redshift__describe_materialized_view(relation) %} + {#- + These need to be separate queries because redshift will not let you run queries + against svv_table_info and pg_views in the same query. The same is true of svv_redshift_columns. + -#} + + {%- set _materialized_view_sql -%} + select + tb.database, + tb.schema, + tb.table, + tb.diststyle, + tb.sortkey1, + mv.autorefresh + from svv_table_info tb + -- svv_mv_info is queryable by Redshift Serverless, but stv_mv_info is not + left join svv_mv_info mv + on mv.database_name = tb.database + and mv.schema_name = tb.schema + and mv.name = tb.table + where tb.table ilike '{{ relation.identifier }}' + and tb.schema ilike '{{ relation.schema }}' + and tb.database ilike '{{ relation.database }}' + {%- endset %} + {% set _materialized_view = run_query(_materialized_view_sql) %} + + {%- set _query_sql -%} + select + vw.definition + from pg_views vw + where vw.viewname = '{{ relation.identifier }}' + and vw.schemaname = '{{ relation.schema }}' + and vw.definition ilike '%create materialized view%' + {%- endset %} + {% set _query = run_query(_query_sql) %} + + {% do return({'materialized_view': _materialized_view, 'query': _query}) %} + +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/drop.sql b/dbt/include/redshift/macros/relations/materialized_view/drop.sql new file mode 100644 index 000000000..76f92e0e9 --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view/drop.sql @@ -0,0 +1,3 @@ +{% macro redshift__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation }} +{%- endmacro %} diff --git a/dbt/include/redshift/macros/relations/materialized_view/refresh.sql b/dbt/include/redshift/macros/relations/materialized_view/refresh.sql new file mode 100644 index 000000000..c53ed2786 --- /dev/null +++ b/dbt/include/redshift/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,3 @@ +{% macro redshift__refresh_materialized_view(relation) -%} + refresh materialized view {{ relation }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/table/drop.sql b/dbt/include/redshift/macros/relations/table/drop.sql new file mode 100644 index 000000000..64ffc1f22 --- /dev/null +++ b/dbt/include/redshift/macros/relations/table/drop.sql @@ -0,0 +1,3 @@ +{%- macro redshift__drop_table(relation) -%} + drop table if exists {{ relation }} cascade +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/table/rename.sql b/dbt/include/redshift/macros/relations/table/rename.sql new file mode 100644 index 000000000..08fd5a172 --- /dev/null +++ b/dbt/include/redshift/macros/relations/table/rename.sql @@ -0,0 +1,3 @@ +{% macro redshift__get_rename_table_sql(relation, new_name) %} + alter table {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/drop.sql b/dbt/include/redshift/macros/relations/view/drop.sql new file mode 100644 index 000000000..cba066a53 --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/drop.sql @@ -0,0 +1,3 @@ +{%- macro redshift__drop_view(relation) -%} + drop view if exists {{ relation }} cascade +{%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/view/rename.sql b/dbt/include/redshift/macros/relations/view/rename.sql new file mode 100644 index 000000000..0c6cdcdfa --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/rename.sql @@ -0,0 +1,3 @@ +{% macro redshift__get_rename_view_sql(relation, new_name) %} + alter view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/relations/view/replace.sql b/dbt/include/redshift/macros/relations/view/replace.sql new file mode 100644 index 000000000..25a9d8b38 --- /dev/null +++ b/dbt/include/redshift/macros/relations/view/replace.sql @@ -0,0 +1,18 @@ +{% macro redshift__get_replace_view_sql(relation, sql) -%} + + {%- set binding = config.get('bind', default=True) -%} + + {% set bind_qualifier = '' if binding else 'with no schema binding' %} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create or replace view {{ relation }} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} as ( + {{ sql }} + ) {{ bind_qualifier }}; + +{%- endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 07364088f..0c8c9b1a3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,31 +1,31 @@ # install latest changes in dbt-core + dbt-postgres # TODO: how to switch from HEAD to x.y.latest branches after minor releases? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres +git+https://github.com/dbt-labs/dbt-postgres.git@main +git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=23.7 +black~=23.12 bumpversion~=0.6.0 click~=8.1 -ddtrace~=1.17 +ddtrace~=2.3 flake8~=6.1 flaky~=3.7 -freezegun~=1.2 +freezegun~=1.3 ipdb~=0.13.13 -mypy==1.4.1 # patch updates have historically introduced breaking changes -pip-tools~=7.2 -pre-commit~=3.3 -pre-commit-hooks~=4.4 +mypy==1.7.1 # patch updates have historically introduced breaking changes +pip-tools~=7.3 +pre-commit~=3.5 +pre-commit-hooks~=4.5 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 pytest-logbook~=1.2 -pytest-xdist~=3.3 +pytest-xdist~=3.5 pytz~=2023.3 -tox~=4.6 +tox~=4.11 types-pytz~=2023.3 types-requests~=2.31 twine~=4.0 -wheel~=0.41 +wheel~=0.42 diff --git a/setup.py b/setup.py index 7b4856351..4925bb04b 100644 --- a/setup.py +++ b/setup.py @@ -36,40 +36,6 @@ def _plugin_version() -> str: return attributes["version"] -def _core_patch(plugin_patch: str): - """ - Determines the compatible dbt-core patch given this plugin's patch - - Args: - plugin_patch: the version patch of this plugin - """ - pre_release_phase = "".join([i for i in plugin_patch if not i.isdigit()]) - if pre_release_phase: - if pre_release_phase not in ["a", "b", "rc"]: - raise ValueError(f"Invalid prerelease patch: {plugin_patch}") - return f"0{pre_release_phase}1" - return "0" - - -# require a compatible minor version (~=) and prerelease if this is a prerelease -def _core_version(plugin_version: str = _plugin_version()) -> str: - """ - Determine the compatible dbt-core version give this plugin's version. - - We assume that the plugin must agree with `dbt-core` down to the minor version. - - Args: - plugin_version: the version of this plugin, this is an argument in case we ever want to unit test this - """ - try: - # *_ may indicate a dev release which won't affect the core version needed - major, minor, plugin_patch, *_ = plugin_version.split(".", maxsplit=3) - except ValueError: - raise ValueError(f"Invalid version: {plugin_version}") - - return f"{major}.{minor}.{_core_patch(plugin_patch)}" - - setup( name="dbt-redshift", version=_plugin_version(), @@ -82,12 +48,12 @@ def _core_version(plugin_version: str = _plugin_version()) -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - f"dbt-core~={_core_version()}", - f"dbt-postgres~={_core_version()}", - "boto3~=1.26.157", + "dbt-common<1.0", + "dbt-adapters~=0.1.0a1", + f"dbt-postgres~={_plugin_version()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector==2.0.913", + "redshift-connector<=2.0.918, >=2.0.913, !=2.0.914", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core "agate", ], diff --git a/tests/functional/adapter/catalog_tests/files.py b/tests/functional/adapter/catalog_tests/files.py new file mode 100644 index 000000000..9c19522e7 --- /dev/null +++ b/tests/functional/adapter/catalog_tests/files.py @@ -0,0 +1,33 @@ +MY_SEED = """ +id,value,record_valid_date +1,100,2023-01-01 00:00:00 +2,200,2023-01-02 00:00:00 +3,300,2023-01-02 00:00:00 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * +from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * +from {{ ref('my_seed') }} +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', +) }} +select * +from {{ ref('my_seed') }} +""" diff --git a/tests/functional/adapter/catalog_tests/test_relation_types.py b/tests/functional/adapter/catalog_tests/test_relation_types.py new file mode 100644 index 000000000..9b9156dec --- /dev/null +++ b/tests/functional/adapter/catalog_tests/test_relation_types.py @@ -0,0 +1,44 @@ +from dbt.contracts.results import CatalogArtifact +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.catalog_tests import files + + +class TestCatalogRelationTypes: + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": files.MY_TABLE, + "my_view.sql": files.MY_VIEW, + "my_materialized_view.sql": files.MY_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def docs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield run_dbt(["docs", "generate"]) + + @pytest.mark.parametrize( + "node_name,relation_type", + [ + ("seed.test.my_seed", "BASE TABLE"), + ("model.test.my_table", "BASE TABLE"), + ("model.test.my_view", "VIEW"), + ("model.test.my_materialized_view", "MATERIALIZED VIEW"), + ], + ) + def test_relation_types_populate_correctly( + self, docs: CatalogArtifact, node_name: str, relation_type: str + ): + """ + This test addresses: https://github.com/dbt-labs/dbt-redshift/issues/652 + """ + assert node_name in docs.nodes + node = docs.nodes[node_name] + assert node.metadata.type == relation_type diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 000000000..808a7733c --- /dev/null +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit + + +class TestRedshiftShowLimit(BaseShowLimit): + pass + + +class TestRedshiftShowSqlHeader(BaseShowSqlHeader): + pass diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py new file mode 100644 index 000000000..3368a90f9 --- /dev/null +++ b/tests/functional/adapter/empty/test_empty.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty + + +class TestRedshiftEmpty(BaseTestEmpty): + pass diff --git a/tests/functional/adapter/expected_stats.py b/tests/functional/adapter/expected_stats.py index 9e635bdbb..265ae225a 100644 --- a/tests/functional/adapter/expected_stats.py +++ b/tests/functional/adapter/expected_stats.py @@ -1,4 +1,4 @@ -from dbt.tests.util import AnyStringWith, AnyFloat, AnyString +from dbt.tests.util import AnyStringWith, AnyInteger, AnyString, AnyFloat def redshift_stats(): @@ -27,14 +27,14 @@ def redshift_stats(): "max_varchar": { "id": "max_varchar", "label": "Max Varchar", - "value": AnyFloat(), + "value": AnyInteger(), "description": "Size of the largest column that uses a VARCHAR data type.", "include": True, }, "size": { "id": "size", "label": "Approximate Size", - "value": AnyFloat(), + "value": AnyInteger(), "description": "Approximate size of the table, calculated from a count of 1MB blocks", "include": True, }, diff --git a/tests/functional/adapter/incremental/test_incremental_strategies.py b/tests/functional/adapter/incremental/test_incremental_strategies.py index ed27be392..da5898bd6 100644 --- a/tests/functional/adapter/incremental/test_incremental_strategies.py +++ b/tests/functional/adapter/incremental/test_incremental_strategies.py @@ -1,6 +1,6 @@ import pytest from dbt.tests.util import run_dbt, get_manifest -from dbt.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError from dbt.context.providers import generate_runtime_model_context diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index d4c3e8a11..63bcede61 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -3,6 +3,7 @@ import pytest from dbt.adapters.base.relation import BaseRelation +from dbt.contracts.graph.model_config import OnConfigurationChangeOption from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic from dbt.tests.adapter.materialized_view.changes import ( @@ -11,17 +12,22 @@ MaterializedViewChangesContinueMixin, MaterializedViewChangesFailMixin, ) -from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW -from dbt.tests.util import get_model_file, set_model_file +from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW, MY_SEED +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + set_model_file, + run_dbt, +) from tests.functional.adapter.materialized_view_tests.utils import ( query_autorefresh, query_dist, query_relation_type, query_sort, + run_dbt_and_capture_with_retries_redshift_mv, ) - MY_MATERIALIZED_VIEW = """ {{ config( materialized='materialized_view', @@ -61,17 +67,13 @@ def query_row_count(project, relation: BaseRelation) -> int: def query_relation_type(project, relation: BaseRelation) -> Optional[str]: return query_relation_type(project, relation) - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with tables." - ) - def test_table_replaces_materialized_view(self, project, my_materialized_view): - super().test_table_replaces_materialized_view(project, my_materialized_view) - - @pytest.mark.skip( - "The current implementation does not support overwriting materialized views with views." - ) - def test_view_replaces_materialized_view(self, project, my_materialized_view): - super().test_view_replaces_materialized_view(project, my_materialized_view) + def test_materialized_view_create_idempotent(self, project, my_materialized_view): + # setup creates it once; verify it's there and run once + assert self.query_relation_type(project, my_materialized_view) == "materialized_view" + run_dbt_and_capture_with_retries_redshift_mv( + ["run", "--models", my_materialized_view.identifier] + ) + assert self.query_relation_type(project, my_materialized_view) == "materialized_view" class RedshiftMaterializedViewChanges(MaterializedViewChanges): @@ -99,10 +101,26 @@ def change_config_via_alter(project, materialized_view): new_model = initial_model.replace("dist='id',", "dist='id', auto_refresh=True") set_model_file(project, materialized_view, new_model) + @staticmethod + def change_config_via_alter_str_true(project, materialized_view): + initial_model = get_model_file(project, materialized_view) + new_model = initial_model.replace("dist='id',", "dist='id', auto_refresh='true'") + set_model_file(project, materialized_view, new_model) + + @staticmethod + def change_config_via_alter_str_false(project, materialized_view): + initial_model = get_model_file(project, materialized_view) + new_model = initial_model.replace("dist='id',", "dist='id', auto_refresh='False'") + set_model_file(project, materialized_view, new_model) + @staticmethod def check_state_alter_change_is_applied(project, materialized_view): assert query_autorefresh(project, materialized_view) is True + @staticmethod + def check_state_alter_change_is_applied_str_false(project, materialized_view): + assert query_autorefresh(project, materialized_view) is False + @staticmethod def change_config_via_replace(project, materialized_view): initial_model = get_model_file(project, materialized_view) @@ -120,16 +138,139 @@ def check_state_replace_change_is_applied(project, materialized_view): class TestRedshiftMaterializedViewChangesApply( RedshiftMaterializedViewChanges, MaterializedViewChangesApplyMixin ): - pass + def test_change_is_applied_via_alter(self, project, my_materialized_view): + self.check_start_state(project, my_materialized_view) + + self.change_config_via_alter(project, my_materialized_view) + _, logs = run_dbt_and_capture_with_retries_redshift_mv( + ["--debug", "run", "--models", my_materialized_view.name] + ) + + self.check_state_alter_change_is_applied(project, my_materialized_view) + + assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs) + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) + + def test_change_is_applied_via_alter_str_true(self, project, my_materialized_view): + self.check_start_state(project, my_materialized_view) + + self.change_config_via_alter_str_true(project, my_materialized_view) + _, logs = run_dbt_and_capture_with_retries_redshift_mv( + ["--debug", "run", "--models", my_materialized_view.name] + ) + + self.check_state_alter_change_is_applied(project, my_materialized_view) + + assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs) + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) + + def test_change_is_applied_via_replace(self, project, my_materialized_view): + self.check_start_state(project, my_materialized_view) + + self.change_config_via_alter(project, my_materialized_view) + self.change_config_via_replace(project, my_materialized_view) + _, logs = run_dbt_and_capture_with_retries_redshift_mv( + ["--debug", "run", "--models", my_materialized_view.name] + ) + + self.check_state_alter_change_is_applied(project, my_materialized_view) + self.check_state_replace_change_is_applied(project, my_materialized_view) + + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs) class TestRedshiftMaterializedViewChangesContinue( RedshiftMaterializedViewChanges, MaterializedViewChangesContinueMixin ): - pass + def test_change_is_not_applied_via_alter(self, project, my_materialized_view): + self.check_start_state(project, my_materialized_view) + + self.change_config_via_alter(project, my_materialized_view) + _, logs = run_dbt_and_capture_with_retries_redshift_mv( + ["--debug", "run", "--models", my_materialized_view.name] + ) + + self.check_start_state(project, my_materialized_view) + + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `continue` for `{my_materialized_view}`", + logs, + ) + assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs, False) + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) + + def test_change_is_not_applied_via_replace(self, project, my_materialized_view): + self.check_start_state(project, my_materialized_view) + + self.change_config_via_alter(project, my_materialized_view) + self.change_config_via_replace(project, my_materialized_view) + _, logs = run_dbt_and_capture_with_retries_redshift_mv( + ["--debug", "run", "--models", my_materialized_view.name] + ) + + self.check_start_state(project, my_materialized_view) + + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `continue` for `{my_materialized_view}`", + logs, + ) + assert_message_in_logs(f"Applying ALTER to: {my_materialized_view}", logs, False) + assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) class TestRedshiftMaterializedViewChangesFail( RedshiftMaterializedViewChanges, MaterializedViewChangesFailMixin ): + # Note: using retries doesn't work when we expect `dbt_run` to fail pass + + +NO_BACKUP_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + backup=False +) }} +select * from {{ ref('my_seed') }} +""" + + +class TestRedshiftMaterializedViewWithBackupConfig: + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_materialized_view.sql": NO_BACKUP_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": MY_SEED} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} + + @pytest.fixture(scope="function") + def dbt_run_results(self, project): + run_dbt(["seed"]) + yield run_dbt(["run", "--full-refresh"]) + + def test_running_mv_with_backup_false_succeeds(self, dbt_run_results): + assert dbt_run_results[0].node.config_call_dict["backup"] is False + + def test_running_mv_with_backup_false_is_idempotent(self, project, dbt_run_results): + """ + Addresses: https://github.com/dbt-labs/dbt-redshift/issues/621 + Context: + - The default for `backup` is `True` + - We cannot query `backup` for a materialized view in Redshift at the moment + Premise: + - Set `on_configuration_change` = 'fail' (via `project_config_update`) + - Set `backup` = False (via `NO_BACKUP_MATERIALIZED_VIEW`) + - Create the materialized view (via `dbt_run_results`) + - Run a second time forcing the configuration change monitoring + - There should be no changes monitored, hence the run should be successful + """ + results = run_dbt(["run"]) + assert results[0].node.config_call_dict["backup"] is False diff --git a/tests/functional/adapter/materialized_view_tests/utils.py b/tests/functional/adapter/materialized_view_tests/utils.py index 5ca1b939b..112ae3057 100644 --- a/tests/functional/adapter/materialized_view_tests/utils.py +++ b/tests/functional/adapter/materialized_view_tests/utils.py @@ -1,6 +1,7 @@ -from typing import Optional +from typing import List, Optional from dbt.adapters.base.relation import BaseRelation +from dbt.tests.util import run_dbt_and_capture from dbt.adapters.redshift.relation import RedshiftRelation @@ -67,3 +68,21 @@ def query_autorefresh(project, relation: RedshiftRelation) -> bool: and trim(mv.db_name) ilike '{ relation.database }' """ return project.run_sql(sql, fetch="one")[0] + + +def run_dbt_and_capture_with_retries_redshift_mv(args: List[str], max_retries: int = 10): + """ + We need to retry `run_dbt` calls on Redshift because we get sporadic failures of the form: + + Database Error in model my_materialized_view (models/my_materialized_view.sql) + could not open relation with OID 14957412 + """ + retries = 0 + while retries < max_retries: + try: + # there's no point to using this with expect_pass=False + return run_dbt_and_capture(args, expect_pass=True) + except AssertionError as e: + retries += 1 + if retries == max_retries: + raise e diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 8f8198a27..64ab24e42 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -92,6 +92,7 @@ class TestBaseAdapterMethod(BaseAdapterMethod): pass +@pytest.mark.skip(reason="Known flakey test to be reviewed") class TestDocsGenerateRedshift(BaseDocsGenerate): @pytest.fixture(scope="class") def expected_catalog(self, project, profile_user): diff --git a/tests/functional/adapter/test_store_test_failures.py b/tests/functional/adapter/test_store_test_failures.py index 5d6b70fbb..7f591654e 100644 --- a/tests/functional/adapter/test_store_test_failures.py +++ b/tests/functional/adapter/test_store_test_failures.py @@ -1,7 +1,32 @@ +from dbt.tests.adapter.store_test_failures_tests import basic from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( TestStoreTestFailures, ) -class RedshiftTestStoreTestFailures(TestStoreTestFailures): +class TestRedshiftTestStoreTestFailures(TestStoreTestFailures): + pass + + +class TestStoreTestFailuresAsInteractions(basic.StoreTestFailuresAsInteractions): + pass + + +class TestStoreTestFailuresAsProjectLevelOff(basic.StoreTestFailuresAsProjectLevelOff): + pass + + +class TestStoreTestFailuresAsProjectLevelView(basic.StoreTestFailuresAsProjectLevelView): + pass + + +class TestStoreTestFailuresAsGeneric(basic.StoreTestFailuresAsGeneric): + pass + + +class TestStoreTestFailuresAsProjectLevelEphemeral(basic.StoreTestFailuresAsProjectLevelEphemeral): + pass + + +class TestStoreTestFailuresAsExceptions(basic.StoreTestFailuresAsExceptions): pass diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 266103fbc..61a706f4d 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -8,9 +8,13 @@ from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect from dbt.tests.adapter.utils.test_last_day import BaseLastDay @@ -65,6 +69,10 @@ class TestDateDiff(BaseDateDiff): pass +class TestDateSpine(BaseDateSpine): + pass + + class TestDateTrunc(BaseDateTrunc): pass @@ -77,6 +85,18 @@ class TestExcept(BaseExcept): pass +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBeteween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + class TestHash(BaseHash): pass diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py new file mode 100644 index 000000000..5e454fe5e --- /dev/null +++ b/tests/unit/relation_configs/test_materialized_view.py @@ -0,0 +1,55 @@ +from unittest.mock import Mock + +import pytest + +from dbt.adapters.redshift.relation_configs import RedshiftMaterializedViewConfig + + +@pytest.mark.parametrize("bool_value", [True, False, "True", "False", "true", "false"]) +def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + config_dict = config.parse_relation_config(model_node) + assert isinstance(config_dict["autorefresh"], bool) + assert isinstance(config_dict["backup"], bool) + + +@pytest.mark.parametrize("bool_value", [1]) +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_types( + bool_value, +): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(TypeError): + config.parse_relation_config(model_node) + + +def test_redshift_materialized_view_config_throws_expected_exception_with_invalid_str(): + config = RedshiftMaterializedViewConfig( + database_name="somedb", + schema_name="public", + mv_name="someview", + query="select * from sometable", + ) + model_node = Mock() + model_node.config.extra.get = ( + lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + ) + with pytest.raises(ValueError): + config.parse_relation_config(model_node) diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py deleted file mode 100644 index 31c436d82..000000000 --- a/tests/unit/test_context.py +++ /dev/null @@ -1,231 +0,0 @@ -import os -import pytest -import unittest - -from unittest import mock - -from .utils import config_from_parts_or_dicts, inject_adapter, clear_plugin -from .mock_adapter import adapter_factory -import dbt.exceptions - -from dbt.adapters import ( - redshift, - factory, -) -from dbt.contracts.graph.model_config import ( - NodeConfig, -) -from dbt.contracts.graph.nodes import ModelNode, DependsOn, Macro -from dbt.context import providers -from dbt.node_types import NodeType - - -class TestRuntimeWrapper(unittest.TestCase): - def setUp(self): - self.mock_config = mock.MagicMock() - self.mock_config.quoting = {"database": True, "schema": True, "identifier": True} - adapter_class = adapter_factory() - self.mock_adapter = adapter_class(self.mock_config) - self.namespace = mock.MagicMock() - self.wrapper = providers.RuntimeDatabaseWrapper(self.mock_adapter, self.namespace) - self.responder = self.mock_adapter.responder - - -PROFILE_DATA = { - "target": "test", - "quoting": {}, - "outputs": { - "test": { - "type": "redshift", - "host": "localhost", - "schema": "analytics", - "user": "test", - "pass": "test", - "dbname": "test", - "port": 1, - } - }, -} - - -PROJECT_DATA = { - "name": "root", - "version": "0.1", - "profile": "test", - "project-root": os.getcwd(), - "config-version": 2, -} - - -def model(): - return ModelNode( - alias="model_one", - name="model_one", - database="dbt", - schema="analytics", - resource_type=NodeType.Model, - unique_id="model.root.model_one", - fqn=["root", "model_one"], - package_name="root", - original_file_path="model_one.sql", - root_path="/usr/src/app", - refs=[], - sources=[], - depends_on=DependsOn(), - config=NodeConfig.from_dict( - { - "enabled": True, - "materialized": "view", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "tags": [], - } - ), - tags=[], - path="model_one.sql", - raw_sql="", - description="", - columns={}, - ) - - -def mock_macro(name, package_name): - macro = mock.MagicMock( - __class__=Macro, - package_name=package_name, - resource_type="macro", - unique_id=f"macro.{package_name}.{name}", - ) - # Mock(name=...) does not set the `name` attribute, this does. - macro.name = name - return macro - - -def mock_manifest(config): - manifest_macros = {} - for name in ["macro_a", "macro_b"]: - macro = mock_macro(name, config.project_name) - manifest_macros[macro.unique_id] = macro - return mock.MagicMock(macros=manifest_macros) - - -def mock_model(): - return mock.MagicMock( - __class__=ModelNode, - alias="model_one", - name="model_one", - database="dbt", - schema="analytics", - resource_type=NodeType.Model, - unique_id="model.root.model_one", - fqn=["root", "model_one"], - package_name="root", - original_file_path="model_one.sql", - root_path="/usr/src/app", - refs=[], - sources=[], - depends_on=DependsOn(), - config=NodeConfig.from_dict( - { - "enabled": True, - "materialized": "view", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "tags": [], - } - ), - tags=[], - path="model_one.sql", - raw_sql="", - description="", - columns={}, - defer_relation=None, - ) - - -@pytest.fixture -def get_adapter(): - with mock.patch.object(providers, "get_adapter") as patch: - yield patch - - -@pytest.fixture -def get_include_paths(): - with mock.patch.object(factory, "get_include_paths") as patch: - patch.return_value = [] - yield patch - - -@pytest.fixture -def config(): - return config_from_parts_or_dicts(PROJECT_DATA, PROFILE_DATA) - - -@pytest.fixture -def manifest_fx(config): - return mock_manifest(config) - - -@pytest.fixture -def manifest_extended(manifest_fx): - dbt_macro = mock_macro("default__some_macro", "dbt") - # same namespace, same name, different pkg! - rs_macro = mock_macro("redshift__some_macro", "dbt_redshift") - # same name, different package - package_default_macro = mock_macro("default__some_macro", "root") - package_rs_macro = mock_macro("redshift__some_macro", "root") - manifest_fx.macros[dbt_macro.unique_id] = dbt_macro - manifest_fx.macros[rs_macro.unique_id] = rs_macro - manifest_fx.macros[package_default_macro.unique_id] = package_default_macro - manifest_fx.macros[package_rs_macro.unique_id] = package_rs_macro - return manifest_fx - - -@pytest.fixture -def redshift_adapter(config, get_adapter): - adapter = redshift.RedshiftAdapter(config) - inject_adapter(adapter, redshift.Plugin) - get_adapter.return_value = adapter - yield adapter - clear_plugin(redshift.Plugin) - - -def test_resolve_specific(config, manifest_extended, redshift_adapter, get_include_paths): - rs_macro = manifest_extended.macros["macro.dbt_redshift.redshift__some_macro"] - package_rs_macro = manifest_extended.macros["macro.root.redshift__some_macro"] - - ctx = providers.generate_runtime_model_context( - model=mock_model(), - config=config, - manifest=manifest_extended, - ) - - ctx["adapter"].config.dispatch - - # macro_a exists, but default__macro_a and redshift__macro_a do not - with pytest.raises(dbt.exceptions.CompilationError): - ctx["adapter"].dispatch("macro_a").macro - - # root namespace is always preferred, unless search order is explicitly defined in 'dispatch' config - assert ctx["adapter"].dispatch("some_macro").macro is package_rs_macro - assert ctx["adapter"].dispatch("some_macro", "dbt").macro is package_rs_macro - assert ctx["adapter"].dispatch("some_macro", "root").macro is package_rs_macro - - # override 'dbt' namespace search order, dispatch to 'root' first - ctx["adapter"].config.dispatch = [{"macro_namespace": "dbt", "search_order": ["root", "dbt"]}] - assert ctx["adapter"].dispatch("some_macro", macro_namespace="dbt").macro is package_rs_macro - - # override 'dbt' namespace search order, dispatch to 'dbt' only - ctx["adapter"].config.dispatch = [{"macro_namespace": "dbt", "search_order": ["dbt"]}] - assert ctx["adapter"].dispatch("some_macro", macro_namespace="dbt").macro is rs_macro - - # override 'root' namespace search order, dispatch to 'dbt' first - ctx["adapter"].config.dispatch = [{"macro_namespace": "root", "search_order": ["dbt", "root"]}] diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index c31366a1e..671e47032 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -1,5 +1,9 @@ import unittest + +from multiprocessing import get_context from unittest import mock + +from dbt_common.exceptions import DbtRuntimeError from unittest.mock import Mock, call import agate @@ -10,8 +14,8 @@ RedshiftAdapter, Plugin as RedshiftPlugin, ) -from dbt.clients import agate_helper -from dbt.exceptions import FailedToConnectError +from dbt_common.clients import agate_helper +from dbt.adapters.exceptions import FailedToConnectError from dbt.adapters.redshift.connections import RedshiftConnectMethodFactory, RedshiftSSLConfig from .utils import ( config_from_parts_or_dicts, @@ -59,7 +63,7 @@ def setUp(self): @property def adapter(self): if self._adapter is None: - self._adapter = RedshiftAdapter(self.config) + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) inject_adapter(self._adapter, RedshiftPlugin) return self._adapter @@ -144,7 +148,6 @@ def test_conn_timeout_30(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_explicit_iam_conn_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -173,7 +176,6 @@ def test_explicit_iam_conn_with_profile(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_explicit_iam_serverless_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -200,7 +202,6 @@ def test_explicit_iam_serverless_with_profile(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_explicit_region(self): # Successful test self.config.credentials = self.config.credentials.replace( @@ -229,7 +230,6 @@ def test_explicit_region(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_explicit_region_failure(self): # Failure test with no region self.config.credentials = self.config.credentials.replace( @@ -239,7 +239,7 @@ def test_explicit_region_failure(self): region=None, ) - with self.assertRaises(dbt.exceptions.FailedToConnectError): + with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( @@ -259,7 +259,6 @@ def test_explicit_region_failure(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_explicit_invalid_region(self): # Invalid region test self.config.credentials = self.config.credentials.replace( @@ -269,7 +268,7 @@ def test_explicit_invalid_region(self): region=None, ) - with self.assertRaises(dbt.exceptions.FailedToConnectError): + with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError): connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( @@ -384,14 +383,13 @@ def test_sslmode_prefer(self): ) @mock.patch("redshift_connector.connect", Mock()) - @mock.patch("boto3.Session", Mock()) def test_serverless_iam_failure(self): self.config.credentials = self.config.credentials.replace( method="iam", iam_profile="test", host="doesnotexist.1233.us-east-2.redshift-srvrlss.amazonaws.com", ) - with self.assertRaises(dbt.exceptions.FailedToConnectError) as context: + with self.assertRaises(dbt.adapters.exceptions.FailedToConnectError) as context: connection = self.adapter.acquire_connection("dummy") connection.handle redshift_connector.connect.assert_called_once_with( @@ -471,14 +469,13 @@ def test_cancel_open_connections_single(self): with mock.patch.object(self.adapter.connections, "add_query") as add_query: query_result = mock.MagicMock() cursor = mock.Mock() - cursor.fetchone.return_value = 42 + cursor.fetchone.return_value = (42,) add_query.side_effect = [(None, cursor), (None, query_result)] self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) add_query.assert_has_calls( [ call("select pg_backend_pid()"), - call("select pg_terminate_backend(42)"), ] ) @@ -514,12 +511,12 @@ def test_dbname_verification_is_case_insensitive(self): } self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) self.adapter.cleanup_connections() - self._adapter = RedshiftAdapter(self.config) + self._adapter = RedshiftAdapter(self.config, get_context("spawn")) self.adapter.verify_database("redshift") def test_execute_with_fetch(self): cursor = mock.Mock() - table = dbt.clients.agate_helper.empty_table() + table = agate_helper.empty_table() with mock.patch.object(self.adapter.connections, "add_query") as mock_add_query: mock_add_query.return_value = ( None, @@ -558,9 +555,7 @@ def test_add_query_with_no_cursor(self): self.adapter.connections, "get_thread_connection" ) as mock_get_thread_connection: mock_get_thread_connection.return_value = None - with self.assertRaisesRegex( - dbt.exceptions.DbtRuntimeError, "Tried to run invalid SQL: on " - ): + with self.assertRaisesRegex(DbtRuntimeError, "Tried to run invalid SQL: on "): self.adapter.connections.add_query(sql="") mock_get_thread_connection.assert_called_once() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f2ca418e3..3fc1d7ec6 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -9,7 +9,7 @@ import agate import pytest -from dbt.dataclass_schema import ValidationError +from dbt_common.dataclass_schema import ValidationError from dbt.config.project import PartialProject @@ -233,7 +233,7 @@ def assert_fails_validation(dct, cls): class TestAdapterConversions(TestCase): @staticmethod def _get_tester_for(column_type): - from dbt.clients import agate_helper + from dbt_common.clients import agate_helper if column_type is agate.TimeDelta: # dbt never makes this! return agate.TimeDelta() diff --git a/tox.ini b/tox.ini index e594c943a..c490fed9a 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,15 @@ passenv = DBT_* REDSHIFT_TEST_* PYTEST_ADDOPTS - DD_SERVICE + DD_CIVISIBILITY_AGENTLESS_ENABLED + DD_API_KEY + DD_SITE DD_ENV + DD_SERVICE commands = - {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils" + {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental" {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/utils + {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/incremental deps = -rdev-requirements.txt -e.