From 62af1b1e4705fda353d4aa8adc92e97500364842 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 4 Sep 2024 10:27:43 -0500 Subject: [PATCH] set up sample testing --- .changes/header.tpl.md | 6 - .changie.yaml | 33 --- .github/CODEOWNERS | 3 + .github/ISSUE_TEMPLATE/bug-report.yml | 65 ------ .github/ISSUE_TEMPLATE/config.yml | 8 - .github/ISSUE_TEMPLATE/feature-request.yml | 59 ------ .../ISSUE_TEMPLATE/implementation-ticket.yml | 58 ------ .github/ISSUE_TEMPLATE/regression-report.yml | 61 ------ .github/workflows/ci.yml | 144 +++++++++++++ CHANGELOG.md | 9 - CONTRIBUTING.md | 115 ++--------- README.md | 19 +- dbt_project.yml | 11 + dev-requirements.txt | 11 +- integration_tests/dbt_project.yml | 25 +++ .../macros}/.gitkeep | 0 integration_tests/models/.gitkeep | 0 integration_tests/packages.yml | 2 + integration_tests/profiles.yml | 111 ++++++++++ integration_tests/tests/.gitkeep | 0 macros/.gitkeep | 0 supported_adapters.env | 1 + tox.ini | 192 ++++++++++++++++++ 23 files changed, 509 insertions(+), 424 deletions(-) delete mode 100644 .changes/header.tpl.md delete mode 100644 .changie.yaml create mode 100644 .github/CODEOWNERS delete mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml delete mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml delete mode 100644 .github/ISSUE_TEMPLATE/implementation-ticket.yml delete mode 100644 .github/ISSUE_TEMPLATE/regression-report.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 CHANGELOG.md create mode 100644 dbt_project.yml create mode 100644 integration_tests/dbt_project.yml rename {.changes/unreleased => integration_tests/macros}/.gitkeep (100%) create mode 100644 integration_tests/models/.gitkeep create mode 100644 integration_tests/packages.yml create mode 100644 integration_tests/profiles.yml create mode 100644 integration_tests/tests/.gitkeep create mode 100644 macros/.gitkeep create mode 100644 supported_adapters.env create mode 100644 tox.ini diff --git a/.changes/header.tpl.md b/.changes/header.tpl.md deleted file mode 100644 index df8faa7..0000000 --- a/.changes/header.tpl.md +++ /dev/null @@ -1,6 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), -and is generated by [Changie](https://github.com/miniscruff/changie). diff --git a/.changie.yaml b/.changie.yaml deleted file mode 100644 index b1bd5ee..0000000 --- a/.changie.yaml +++ /dev/null @@ -1,33 +0,0 @@ -changesDir: .changes -unreleasedDir: unreleased -headerPath: header.tpl.md -changelogPath: CHANGELOG.md -versionExt: md -envPrefix: CHANGIE_ -versionFormat: '## dbt-oss-template {{.Version}} - {{.Time.Format "January 02, 2006"}}' -kindFormat: '### {{.Kind}}' -changeFormat: '* {{.Body}}' -kinds: - - label: Breaking Changes - - label: Features - - label: Fixes - - label: Docs - - label: Under the Hood - - label: Dependencies - - label: Security -newlines: - afterChangelogHeader: 1 - afterKind: 1 - afterChangelogVersion: 1 - beforeKind: 1 - endOfVersion: 1 - -custom: -- key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 -- key: Issue - label: GitHub Issue Number (separated by a single space if multiple) - type: string - minLength: 1 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..024ce49 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# This codeowners file is used to ensure all PRs require reviews from the core team + +* @dbt-labs/core-team diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml deleted file mode 100644 index 11de400..0000000 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: 🐞 Bug -description: Report a bug or an issue you've found with dbt-oss-template -title: "[Bug] " -labels: ["bug", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report! - - type: checkboxes - attributes: - label: Is this a new bug in dbt-oss-template? - description: > - In other words, is this an error, flaw, failure or fault in our software? - - If this is a bug that broke existing functionality that used to work, please open a regression issue. - If this is a request for help or troubleshooting code in your own dbt project, please join our [dbt Community Slack](https://www.getdbt.com/community/join-the-community/) or open a [Discussion question](https://github.com/dbt-labs/docs.getdbt.com/discussions). - - Please search to see if an issue already exists for the bug you encountered. - options: - - label: I believe this is a new bug in dbt-oss-template - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this bug - required: true - - type: textarea - attributes: - label: Current Behavior - description: A concise description of what you're experiencing. - validations: - required: true - - type: textarea - attributes: - label: Expected Behavior - description: A concise description of what you expected to happen. - validations: - required: true - - type: textarea - attributes: - label: Steps To Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. In this environment... - 2. With this config... - 3. Run '...' - 4. See error... - validations: - required: true - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - If applicable, log output to help explain your problem. - render: shell - validations: - required: false - - type: textarea - attributes: - label: Additional Context - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - - Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 3661b39..0000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Ask the community for help - url: https://github.com/dbt-labs/docs.getdbt.com/discussions - about: Need help troubleshooting? Check out our guide on how to ask - - name: Participate in Discussions - url: https://github.com/dbt-labs/dbt-oss-template/discussions - about: Do you have a Big Idea for dbt-oss-template? Read open discussions, or start a new one diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml deleted file mode 100644 index 2a88a62..0000000 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: ✨ Feature -description: Propose a straightforward extension of dbt-oss-template functionality -title: "[Feature] <title>" -labels: ["enhancement", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this feature request! - - type: checkboxes - attributes: - label: Is this your first time submitting a feature request? - description: > - We want to make sure that features are distinct and discoverable, - so that other members of the community can find them and offer their thoughts. - - Issues are the right place to request straightforward extensions of existing dbt-oss-template functionality. - For "big ideas" about future capabilities of dbt-oss-template, we ask that you open a - [discussion](https://github.com/dbt-labs/dbt-oss-template/discussions) in the "Ideas" category instead. - options: - - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this feature - required: true - - label: I am requesting a straightforward extension of existing dbt functionality, rather than a Big Idea better suited to a discussion - required: true - - type: textarea - attributes: - label: Describe the feature - description: A clear and concise description of what you want to happen. - validations: - required: true - - type: textarea - attributes: - label: Describe alternatives you've considered - description: | - A clear and concise description of any alternative solutions or features you've considered. - validations: - required: false - - type: textarea - attributes: - label: Who will this benefit? - description: | - What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. - validations: - required: false - - type: input - attributes: - label: Are you interested in contributing this feature? - description: Let us know if you want to write some code, and how we can help. - validations: - required: false - - type: textarea - attributes: - label: Anything else? - description: | - Links? References? Anything that will give us more context about the feature you are suggesting! - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/implementation-ticket.yml b/.github/ISSUE_TEMPLATE/implementation-ticket.yml deleted file mode 100644 index a3b052c..0000000 --- a/.github/ISSUE_TEMPLATE/implementation-ticket.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: 🛠️ Implementation -description: This is an implementation ticket intended for use by the maintainers of dbt-oss-template -title: "[<project>] <title>" -labels: ["user docs"] -body: - - type: markdown - attributes: - value: This is an implementation ticket intended for use by the maintainers of dbt-oss-template - - type: checkboxes - attributes: - label: Housekeeping - description: > - A couple friendly reminders: - 1. Remove the `user docs` label if the scope of this work does not require changes to https://docs.getdbt.com/docs: no end-user interface (e.g. yml spec, CLI, error messages, etc) or functional changes - 2. Link any blocking issues in the "Blocked on" field under the "Core devs & maintainers" project. - options: - - label: I am a maintainer of dbt-oss-template - required: true - - type: textarea - attributes: - label: Short description - description: | - Describe the scope of the ticket, a high-level implementation approach and any tradeoffs to consider - validations: - required: true - - type: textarea - attributes: - label: Acceptance criteria - description: | - What is the definition of done for this ticket? Include any relevant edge cases and/or test cases - validations: - required: true - - type: textarea - attributes: - label: Impact to Other Teams - description: | - Will this change impact other teams? Include details of the kinds of changes required (new tests, code changes, related tickets) and _add the relevant `Impact:[team]` label_. - placeholder: | - Example: This change impacts `dbt-redshift` because the tests will need to be modified. The `Impact:[Adapter]` label has been added. - validations: - required: true - - type: textarea - attributes: - label: Will backports be required? - description: | - Will this change need to be backported to previous versions? Add details, possible blockers to backporting and _add the relevant backport labels `backport 1.x.latest`_ - placeholder: | - Example: Backport to 1.6.latest, 1.5.latest and 1.4.latest. Since 1.4 isn't using click, the backport may be complicated. The `backport 1.6.latest`, `backport 1.5.latest` and `backport 1.4.latest` labels have been added. - validations: - required: true - - type: textarea - attributes: - label: Context - description: | - Provide the "why", motivation, and alternative approaches considered -- linking to previous refinement issues, spikes, docs as appropriate - validations: - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml deleted file mode 100644 index 766e4ef..0000000 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: ☣️ Regression -description: Report a regression you've observed in a newer version of dbt-oss-template -title: "[Regression] <title>" -labels: ["bug", "regression", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this regression report! - - type: checkboxes - attributes: - label: Is this a regression in a recent version of dbt-oss-template? - description: > - A regression is when documented functionality works as expected in an older version of dbt-oss-template, - and no longer works after upgrading to a newer version of dbt-oss-template - options: - - label: I believe this is a regression in dbt-oss-template functionality - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this regression - required: true - - type: textarea - attributes: - label: Current Behavior - description: A concise description of what you're experiencing. - validations: - required: true - - type: textarea - attributes: - label: Expected/Previous Behavior - description: A concise description of what you expected to happen. - validations: - required: true - - type: textarea - attributes: - label: Steps To Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. In this environment... - 2. With this config... - 3. Run '...' - 4. See error... - validations: - required: true - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - If applicable, log output to help explain your problem. - render: shell - validations: - required: false - - type: textarea - attributes: - label: Additional Context - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - - Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. - validations: - required: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c9475b8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,144 @@ +# **what?** +# Run tests for package-testing against supported adapters + +# **why?** +# To ensure that dbt-package-testing works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + # This repo does not have secrets set up so these triggers are disabled to prevent many failing tests on all PRs + # Packages would want to trigger these tests for pushes to main and pull_requests to ensure nothing breaks + # with any changes to the package + # push: + # branches: + # - main + # pull_request: + workflow_dispatch: + inputs: + adapter: + description: The adapter to test against. Defaults to all supported adapters when blank. + type: string + required: false + +env: + PYTHON_VERSION: "3.11" + +jobs: + determine-supported-adapters: + runs-on: ubuntu-latest + outputs: + adapters: ${{ steps.supported-adapters.outputs.adapters }} + steps: + - name: "Checkout ${{ github.event.repository }}" + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Get list of supported adapters or use input adapter only" + id: list-adapters + run: | + if [ -z "${{ inputs.adapter }}" ]; then + # github adds a pip freeze and a new line we need to strip out + source supported_adapters.env + echo $SUPPORTED_ADAPTERS + echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT + else + echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT + fi + + - name: "Format adapter list for use as the matrix" + id: supported-adapters + run: | + # Convert to JSON array and output + supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")') + echo $supported_adapters + echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT + + - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}" + run: | + title="${{ github.event.repository.name }} - adapters to test" + message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}" + echo "::notice $title::$message" + + run-tests: + runs-on: ubuntu-latest + needs: [determine-supported-adapters] + # you can install postgres as an additional service in the github container + services: + postgres: + image: postgres + env: + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }} + POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: + fail-fast: false + matrix: + adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}} + + steps: + - name: "Checkout ${{ github.event.repository }} " + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install ${{ matrix.adapter }}" + run: | + python -m pip install --upgrade pip + pip install dbt-${{ matrix.adapter }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Run integration tests with tox on ${{ matrix.adapter }}" + run: | + tox -e dbt_integration_${{ matrix.adapter }} + env: + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }} + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres_${{ github.run_number }}" + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" + REDSHIFT_PORT: 5439 + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} + BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" + # plus any other env vars needed for your supported adapter diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 5ca9d33..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), -and is generated by [Changie](https://github.com/miniscruff/changie). - - -No releases yet, this file will be updated when generating your first release. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36f8f95..c5121eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,15 +1,12 @@ -** replace `dbt-oss-template` with your repository name in all docs +# Contributing to `dbt-package-testing` - -# Contributing to `dbt-oss-template` - -`dbt-oss-template` is a template for open source software projects at dbt Labs. +`dbt-package-testing` is a template for package testing at dbt Labs. 1. [About this document](#about-this-document) 2. [Getting the code](#getting-the-code) 3. [Setting up an environment](#setting-up-an-environment) -4. [Running in development](#running-dbt-oss-template-in-development) +4. [Running in development](#running-dbt-package-testing-in-development) 5. [Testing](#testing) 6. [Debugging](#debugging) 7. [Adding or modifying a changelog entry](#adding-or-modifying-a-changelog-entry) @@ -18,15 +15,15 @@ ## About this document -There are many ways to contribute to the ongoing development of `dbt-oss-template`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations). +There are many ways to contribute to the ongoing development of `dbt-package-testing`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations). -The rest of this document serves as a more granular guide for contributing code changes to `dbt-oss-template` (this repository). It is not intended as a guide for using `dbt-oss-template`, and some pieces assume a level of familiarity with Python development (virtualenvs, `pip`, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line. +The rest of this document serves as a more granular guide for contributing code changes to `dbt-package-testing` (this repository). It is not intended as a guide for using `dbt-package-testing`, and some pieces assume a level of familiarity with Python development (virtualenvs, `pip`, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line. -If you get stuck, we're happy to help! Drop us a line in the `#dbt-oss-template-development` channel in the [dbt Community Slack](https://community.getdbt.com). +If you get stuck, we're happy to help! Drop us a line in the `#dbt-package-testing-development` channel in the [dbt Community Slack](https://community.getdbt.com). ### Notes -- **CLA:** Please note that anyone contributing code to `dbt-oss-template` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements). If you are unable to sign the CLA, the `dbt-oss-template` maintainers will unfortunately be unable to merge any of your Pull Requests. We welcome you to participate in discussions, open issues, and comment on existing ones. +- **CLA:** Please note that anyone contributing code to `dbt-package-testing` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements). If you are unable to sign the CLA, the `dbt-package-testing` maintainers will unfortunately be unable to merge any of your Pull Requests. We welcome you to participate in discussions, open issues, and comment on existing ones. - **Branches:** All pull requests from community contributors should target the `main` branch (default). - **Releases**: This repository is never released. @@ -38,67 +35,21 @@ You will need `git` in order to download and modify the source code. ### External contributors -If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-oss-template` by forking the `dbt-oss-template` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: +If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-package-testing` by forking the `dbt-package-testing` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: -1. Fork the `dbt-oss-template` repository +1. Fork the `dbt-package-testing` repository 2. Clone your fork locally 3. Check out a new branch for your proposed changes 4. Push changes to your fork -5. Open a pull request against `dbt-labs/dbt-oss-template` from your forked repository +5. Open a pull request against `dbt-labs/dbt-package-testing` from your forked repository ### dbt Labs contributors -If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-oss-template` repo. Rather than forking `dbt-oss-template` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. +If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-package-testing` repo. Rather than forking `dbt-package-testing` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. ## Setting up an environment -There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt-oss-template` development, many of these tools are used commonly across open-source python projects. - -### Tools - -These are the tools used in `dbt-oss-template` development and testing: - -- [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting -- [`black`](https://github.com/psf/black) for code formatting -- [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking -- [`pre-commit`](https://pre-commit.com) to easily run those checks -- [`changie`](https://changie.dev/) to create changelog entries, without merge conflicts - -A deep understanding of these tools in not required to effectively contribute to `dbt-oss-template`, but we recommend checking out the attached documentation if you're interested in learning more about each one. - -#### Virtual environments - -We strongly recommend using virtual environments when developing code in `dbt-oss-template`. We recommend creating this virtualenv -in the root of the `dbt-oss-template` repository. To create a new virtualenv, run: -```sh -python3 -m venv env -source env/bin/activate -``` - -This will create and activate a new Python virtual environment. - -## Running `dbt-oss-template` in development - -### Installation - -First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment). Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-oss-template` (and its dependencies): - -```sh -git -pre-commit install -``` - -### Running `dbt-oss-template` - -This repository is just a template and cannot be run. - -## Testing - -Once you're able to manually test that your code change is working as expected, it's important to run existing automated tests, as well as adding some new ones. These tests will ensure that: -- Your code changes do not unexpectedly break other established functionality -- Your code changes can handle all known edge cases -- The functionality you're adding will _keep_ working in the future - +This repo serves as a template and is no intedned to be run. ### Initial setup @@ -108,49 +59,13 @@ None needed. No tests included. - -### Unit, Integration, Functional? - -Here are some general rules for adding tests: -* unit tests (`tests/unit`) don’t need to access a database; "pure Python" tests should be written as unit tests -* functional tests (`tests/functional`) cover anything that interacts with a database, namely adapter - -## Debugging - -1. The logs for a `dbt run` have stack traces and other information for debugging errors (in `logs/dbt.log` in your project directory). -2. Try using a debugger, like `ipdb`. For pytest: `--pdb --pdbcls=IPython.terminal.debugger:pdb` -3. - -### Assorted development tips -* Append `# type: ignore` to the end of a line if you need to disable `mypy` on that line. -* Sometimes flake8 complains about lines that are actually fine, in which case you can put a comment on the line such as: # noqa or # noqa: ANNN, where ANNN is the error code that flake8 issues. -* To collect output for `CProfile`, run dbt with the `-r` option and the name of an output file, i.e. `dbt -r dbt.cprof run`. If you just want to profile parsing, you can do: `dbt -r dbt.cprof parse`. `pip` install `snakeviz` to view the output. Run `snakeviz dbt.cprof` and output will be rendered in a browser window. - -## Adding or modifying a CHANGELOG Entry - -We use [changie](https://changie.dev) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost. - -Follow the steps to [install `changie`](https://changie.dev/guide/installation/) for your system. - -Once changie is installed and your PR is created for a new feature, simply run the following command and changie will walk you through the process of creating a changelog entry: - -```shell -changie new -``` - -Commit the file that's created and your changelog entry is complete! - -If you are contributing to a feature already in progress, you will modify the changie yaml file in dbt/.changes/unreleased/ related to your change. If you need help finding this file, please ask within the discussion for the pull request! - -You don't need to worry about which `dbt-oss-template` version your change will go into. Just create the changelog entry with `changie`, and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-oss-template`. The Core maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-oss-template`. - ## Submitting a Pull Request -Code can be merged into the current development branch `main` by opening a pull request. A `dbt-oss-template` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. +Code can be merged into the current development branch `main` by opening a pull request. A `dbt-package-testing` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. -Automated tests run via GitHub Actions. If you're a first-time contributor, all tests (including code checks and unit tests) will require a maintainer to approve. Changes in the `dbt-oss-template` repository trigger integration tests against Postgres. dbt Labs also provides CI environments in which to test changes to other adapters, triggered by PRs in those adapters' repositories, as well as periodic maintenance checks of each adapter in concert with the latest `dbt-oss-template` code changes. +Automated tests run via GitHub Actions. If you're a first-time contributor, all tests (including code checks and unit tests) will require a maintainer to approve. Changes in the `dbt-package-testing` repository trigger integration tests against Postgres. dbt Labs also provides CI environments in which to test changes to other adapters, triggered by PRs in those adapters' repositories, as well as periodic maintenance checks of each adapter in concert with the latest `dbt-package-testing` code changes. -Once all tests are passing and your PR has been approved, a `dbt-oss-template` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: +Once all tests are passing and your PR has been approved, a `dbt-package-testing` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: ## Troubleshooting Tips - Sometimes, the content license agreement auto-check bot doesn't find a user's entry in its roster. If you need to force a rerun, add `@cla-bot check` in a comment on the pull request. diff --git a/README.md b/README.md index 023aadc..6120021 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,10 @@ -** replace `dbt-oss-template` with your repository name in all docs +## Understanding dbt-package-testing -## Understanding dbt-oss-template - -A short description of the purpose of this repository. Include relevant images. - -## Getting started - -- [Install dbt](https://docs.getdbt.com/docs/get-started/installation) -- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) - -## Join the dbt Community - -- Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) -- Read more on the [dbt Community Discourse](https://discourse.getdbt.com) +This is an example of how to test packages. ## Reporting bugs and contributing code -- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt-oss-template/issues/new) -- Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt-oss-template/blob/HEAD/CONTRIBUTING.md) +- Want to report a bug or request a feature? Let us know by opening [an issue](https://github.com/dbt-labs/dbt-package-testing/issues/new) ## Code of Conduct diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..211c284 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,11 @@ +name: 'dbt_package_testing' +version: '0.1.0' + +require-dbt-version: [">=1.3.0", "<2.0.0"] + +config-version: 2 + +target-path: "target" +clean-targets: ["target", "dbt_modules", "dbt_packages"] +macro-paths: ["macros"] +log-path: "logs" diff --git a/dev-requirements.txt b/dev-requirements.txt index 6f05e4a..df08dcd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,3 @@ -black==24.1.1 -bumpversion -flake8 -flaky -freezegun==1.4.0 -hypothesis -ipdb -mypy==1.8.0 +tox pip-tools -pre-commit + diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml new file mode 100644 index 0000000..bbe22f8 --- /dev/null +++ b/integration_tests/dbt_project.yml @@ -0,0 +1,25 @@ + +name: 'dbt_package_testing_integration_tests' +version: '1.0' + +profile: 'integration_tests' + +config-version: 2 + +model-paths: ["models"] +test-paths: ["tests"] +macro-paths: ["macros"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + - "dbt_packages" + +flags: + send_anonymous_usage_stats: False + use_colors: True + +dispatch: + - macro_namespace: 'dbt_package_testing' + search_order: ['dbt_package_testing_integration_tests', 'dbt_package_testing'] diff --git a/.changes/unreleased/.gitkeep b/integration_tests/macros/.gitkeep similarity index 100% rename from .changes/unreleased/.gitkeep rename to integration_tests/macros/.gitkeep diff --git a/integration_tests/models/.gitkeep b/integration_tests/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml new file mode 100644 index 0000000..424d083 --- /dev/null +++ b/integration_tests/packages.yml @@ -0,0 +1,2 @@ +packages: + - local: ../ diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 0000000..277596c --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,111 @@ +integration_tests: + target: postgres + outputs: + postgres: + type: "postgres" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 5 + + redshift: + type: "redshift" + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA') }}" + threads: 5 + + bigquery: + type: "bigquery" + method: "service-account-json" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + threads: 10 + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}" + job_retries: 3 + + snowflake: + type: "snowflake" + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" + threads: 10 + + trino: + type: "trino" + method: "{{ env_var('TRINO_METHOD') }}" + user: "{{ env_var('TRINO_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_TRINO_PASS') }}" + host: "{{ env_var('TRINO_HOST') }}" + port: "{{ env_var('TRINO_PORT') | as_number}}" + catalog: "{{ env_var('TRINO_CATALOG') }}" + schema: "{{ env_var('TRINO_SCHEMA') }}" + timezone: "{{ env_var('TRINO_TIMEZONE') }}" + threads: 12 + + databricks: + type: "databricks" + schema: "{{ env_var('DATABRICKS_SCHEMA') }}" + host: "{{ env_var('DATABRICKS_HOST') }}" + http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" + token: "{{ env_var('DBT_SECRET_ENV_DATABRICKS_TOKEN') }}" + threads: 8 + connect_retries: 5 + connect_timeout: 300 + + spark: + type: spark + host: "{{ env_var('SPARK_HOST') }}" + schema: "{{ env_var('SPARK_SCHEMA') }}" + user: "{{ env_var('SPARK_USER') }}" + method: "{{ env_var('SPARK_METHOD') }}" + port: "{{ env_var('SPARK_PORT') | as_number}}" + connect_retries: 3 + connect_timeout: 5 + + fabric: + type: fabric + driver: "{{ env_var('FABRIC_DRIVER') }}" + server: "{{ env_var('FABRIC_HOST') }}" + port: "{{ env_var('FABRIC_PORT') | as_number}}" + database: "{{ env_var('FABRIC_DATABASE') }}" + schema: "{{ env_var('FABRIC_SCHEMA') }}" + authentication: "{{ env_var('FABRIC_AUTHENTICATION') }}" + tenant_id: "{{ env_var('FABRIC_TENANT') }}" + client_id: "{{ env_var('FABRIC_CLIENT') }}" + client_secret: "{{ env_var('DBT_ENV_SECRET_FABRIC_CLIENT_SECRET') }}" + + synapse: + type: synapse + driver: "{{ env_var('SYNAPSE_DRIVER') }}" + server: "{{ env_var('SYNAPSE_HOST') }}" + port: "{{ env_var('SYNAPSE_PORT') | as_number}}" + database: "{{ env_var('SYNAPSE_DATABASE') }}" + schema: "{{ env_var('SYNAPSE_SCHEMA') }}" + authentication: "{{ env_var('SYNAPSE_AUTHENTICATION') }}" + tenant_id: "{{ env_var('SYNAPSE_TENANT_ID') }}" + client_id: "{{ env_var('SYNAPSE_CLIENT_ID') }}" + client_secret: "{{ env_var('DBT_ENV_SECRET_SYNAPSE_CLIENT_SECRET') }}" + + athena: + type: athena + s3_staging_dir: "{{ env_var('ATHENA_S3_STAGING_DIR') }}" + s3_data_dir: "{{ env_var('ATHENA_S3_DATA_DIR') }}" + s3_data_naming: "{{ env_var('ATHENA_S3_DATA_NAMING') }}" + region_name: "{{ env_var('ATHENA_REGION_NAME') }}" + schema: "{{ env_var('ATHENA_SCHEMA') }}" + database: "{{ env_var('ATHENA_DATABASE') }}" + threads: 4 + aws_access_key_id: "{{ env_var('DBT_ENV_SECRET_ATHENA_AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ env_var('DBT_ENV_SECRET_ATHENA_AWS_SECRET_ACCESS_KEY') }}" diff --git a/integration_tests/tests/.gitkeep b/integration_tests/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 0000000..5443231 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres,redshift,bigquery,snowflake,trino,databricks,spark,fabric,synapse,athena \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..9b9b021 --- /dev/null +++ b/tox.ini @@ -0,0 +1,192 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv + +[testenv] +passenv = + # postgres env vars + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA + # redshift + REDSHIFT_HOST + REDSHIFT_USER + DBT_ENV_SECRET_REDSHIFT_PASS + REDSHIFT_DATABASE + REDSHIFT_SCHEMA + REDSHIFT_PORT + # bigquery + BIGQUERY_PROJECT + BIGQUERY_KEYFILE_JSON + BIGQUERY_SCHEMA + # snowflake + SNOWFLAKE_ACCOUNT + SNOWFLAKE_USER + DBT_ENV_SECRET_SNOWFLAKE_PASS + SNOWFLAKE_ROLE + SNOWFLAKE_DATABASE + SNOWFLAKE_WAREHOUSE + SNOWFLAKE_SCHEMA + # trino + TRINO_METHOD + TRINO_USER + DBT_ENV_SECRET_TRINO_PASS + TRINO_HOST + TRINO_PORT + TRINO_CATALOG + TRINO_SCHEMA + TRINO_TIMEZONE + # databricks + DATABRICKS_SCHEMA + DATABRICKS_HOST + DATABRICKS_HTTP_PATH + DBT_SECRET_ENV_DATABRICKS_TOKEN + # spark + SPARK_HOST + SPARK_SCHEMA + SPARK_USER + SPARK_METHOD + SPARK_PORT + # fabric + FABRIC_DRIVER + FABRIC_HOST + FABRIC_PORT + FABRIC_DATABASE + FABRIC_SCHEMA + FABRIC_AUTHENTICATION + FABRIC_TENANT + FABRIC_CLIENT + DBT_ENV_SECRET_FABRIC_CLIENT_SECRET + # synapse + SYNAPSE_DRIVER + SYNAPSE_HOST + SYNAPSE_PORT + SYNAPSE_DATABASE + SYNAPSE_SCHEMA + SYNAPSE_AUTHENTICATION + SYNAPSE_TENANT_ID + SYNAPSE_CLIENT_ID + DBT_ENV_SECRET_SYNAPSE_CLIENT_SECRET + # athena + ATHENA_S3_STAGING_DIR + ATHENA_S3_DATA_DIR + ATHENA_S3_DATA_NAMING + ATHENA_REGION_NAME + ATHENA_SCHEMA + ATHENA_DATABASE + DBT_ENV_SECRET_ATHENA_AWS_ACCESS_KEY_ID + DBT_ENV_SECRET_ATHENA_AWS_SECRET_ACCESS_KEY + + +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target postgres + +# redshift integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_redshift] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target redshift + +# BigQuery integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target bigquery + +# Snowflake integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_snowflake] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target snowflake + +# Trino integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_trino] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target trino + +# Databricks integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_databricks] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target databricks + +# Spark integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_spark] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target spark + +# Fabric integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_fabric] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target fabric + +# Synapse integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_synapse] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target synapse + +# Athena integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_athena] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target athena