From bb763ec1041d4c2d40aab6406b359c4a3634351c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 21:04:19 -0500 Subject: [PATCH 1/7] move config into pyproject.toml or hatch.toml --- .bumpversion.cfg | 35 --------------- test.env.example => .env.example | 0 MANIFEST.in | 1 - Makefile | 77 -------------------------------- dev-requirements.txt | 24 ---------- hatch.toml | 61 +++++++++++++++++++++++++ pyproject.toml | 56 +++++++++++++++++++++++ pytest.ini | 11 ----- setup.py | 76 ------------------------------- tox.ini | 35 --------------- 10 files changed, 117 insertions(+), 259 deletions(-) delete mode 100644 .bumpversion.cfg rename test.env.example => .env.example (100%) delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 hatch.toml create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index f3d7dd51b..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.9.0b2 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/redshift/__version__.py] diff --git a/test.env.example b/.env.example similarity index 100% rename from test.env.example rename to .env.example diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index e443f8545..000000000 --- a/Makefile +++ /dev/null @@ -1,77 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-redshift - -.PHONY: mypy -mypy: ## Runs mypy against staged changes for static type checking. - @\ - pre-commit run --hook-stage manual mypy-check | grep -v "INFO" - -.PHONY: flake8 -flake8: ## Runs flake8 against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual flake8-check | grep -v "INFO" - -.PHONY: black -black: ## Runs black against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual black-check -v | grep -v "INFO" - -.PHONY: lint -lint: ## Runs flake8 and mypy code checks against staged changes. - @\ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: linecheck -linecheck: ## Checks for all Python lines 100 characters or more - @\ - find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; - -.PHONY: unit -unit: ## Runs unit tests with py39. - @\ - tox -e py39 - -.PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. - @\ - tox -p -e py39; \ - pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: integration -integration: ## Runs redshift integration tests with py38. - @\ - tox -e py39-redshift -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - - -.PHONY: docker-dev -docker-dev: - docker build -f docker/dev.Dockerfile -t dbt-redshift-dev . - docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev - -.PHONY: docker-prod -docker-prod: - docker build -f docker/Dockerfile -t dbt-redshift . diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 5a8b7f23f..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -# install latest changes in dbt-core + dbt-postgres -git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-postgres.git - -# dev -ipdb~=0.13.13 -pre-commit~=3.7.0 - -# test -ddtrace==2.3.0 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.5 -tox~=4.16 - -# build -bumpversion~=0.6.0 -twine~=5.1 -wheel~=0.42 diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 000000000..c33f5dac5 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,61 @@ +[version] +path = "dbt/adapters/redshift/__version__.py" + +[build.targets.sdist] +packages = ["dbt"] + +[build.targets.wheel] +packages = ["dbt"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "ddtrace==2.3.0", + "ipdb~=0.13.13", + "pre-commit==3.7.0", + "freezegun", + "pytest", + "pytest-csv~=3.0", + "pytest-dotenv", + "pytest-logbook~=1.2", + "pytest-mock", + "pytest-xdist", +] + +[envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-redshift-dev .", + "docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] + +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt_redshift-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_redshift-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-redshift ." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..51e48b2e6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-redshift" +description = "The Redshift adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "redshift", "aws", "amazon", "amazon web services"] +requires-python = ">=3.9.0" +authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.7,<2.0", + "dbt-postgres>=1.8,<1.10", + # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases + # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. + "redshift-connector<2.1.1,>=2.0.913,!=2.0.914", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0b3", + # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "sqlparse>=0.5.0,<0.6.0", + "agate", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-redshift" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-redshift.git" +Issues = "https://github.com/dbt-labs/dbt-redshift/issues" +Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" + +[tool.mypy] +mypy_path = "third-party-stubs/" + +[tool.pytest] +testpaths = ["tests/functional", "tests/unit"] +addopts = "-v -n auto" +color = true +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 8c290dc14..000000000 --- a/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/functional -markers = - flaky: marks tests as flaky so they run one at a time (de-select with '-m "not flaky"') diff --git a/setup.py b/setup.py deleted file mode 100644 index 675b8588e..000000000 --- a/setup.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -import sys - -if sys.version_info < (3, 9): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.9 or higher.") - sys.exit(1) - - -try: - from setuptools import find_namespace_packages -except ImportError: - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again') - sys.exit(1) - - -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/redshift/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -setup( - name="dbt-redshift", - version=_plugin_version(), - description="The Redshift adapter plugin for dbt", - long_description=README.read_text(), - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-redshift", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", - "dbt-postgres>=1.8,<1.10", - # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases - # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<2.1.1,>=2.0.913,!=2.0.914", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0b3", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.5.0,<0.6.0", - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], - python_requires=">=3.9", -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index f1929ff81..000000000 --- a/tox.ini +++ /dev/null @@ -1,35 +0,0 @@ -[tox] -skipsdist = True -envlist = py39,py310,py311,py312 - -[testenv:{unit,py39,py310,py311,py312,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py39,py310,py311,py312,py}-{redshift}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - REDSHIFT_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_INSTRUMENTATION_TELEMETRY_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental" - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/utils - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/incremental -deps = - -rdev-requirements.txt - -e. From a3e412f0b8ede65e4d022c9f1fad9ae5c8185ecc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:36:41 -0500 Subject: [PATCH 2/7] move config into pyproject.toml or hatch.toml --- .bumpversion.cfg | 0 .github/scripts/update_dependencies.sh | 8 +- .../scripts/update_dev_dependency_branches.sh | 4 +- .github/workflows/integration.yml | 94 ++-- .github/workflows/main.yml | 76 +-- .github/workflows/nightly-release.yml | 14 +- .github/workflows/release.yml | 80 ++- .github/workflows/release_prep_hatch.yml | 455 ++++++++++++++++++ hatch.toml | 2 +- setup.py | 0 10 files changed, 542 insertions(+), 191 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .github/workflows/release_prep_hatch.yml delete mode 100644 setup.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index 6000b5006..1205d80a8 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -2,10 +2,10 @@ set -e git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" +postgres_req_sed_pattern="s|dbt-postgres.git|dbt-postgres.git@${git_branch}" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 022df6a8a..9385cf885 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -5,8 +5,8 @@ set -e dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd952d8dd..e8d4911fe 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -100,14 +100,35 @@ jobs: python-version: "3.9" env: - TOXENV: integration-redshift - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 steps: - name: Check out the repository (push) @@ -144,14 +165,9 @@ jobs: ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt + cat hatch.toml - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version + - uses: pypa/hatch@install - name: Create AWS IAM profiles run: | @@ -169,52 +185,8 @@ jobs: aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION aws configure --profile $AWS_ROLE_PROFILE set output json - env: - AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} - AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} - AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} - AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user - AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} - AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} - AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} - AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} - AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - - - name: Run tox (redshift) - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} - REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} - REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} - REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- -m "not flaky" --ddtrace - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv - path: integration_results.csv + - run: hatch run integration-tests tests/functional -k "not flaky" test-flaky: name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky @@ -236,8 +208,6 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12"] env: - TOXENV: integration-redshift - PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_INSTRUMENTATION_TELEMETRY_ENABLED: false @@ -274,12 +244,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version + - uses: pypa/hatch@install - name: Update Adapters and Core branches (update dev_requirements.txt) if: github.event_name == 'workflow_dispatch' @@ -288,8 +253,7 @@ jobs: ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt - + cat hatch.toml - name: Run tox (redshift) env: @@ -307,7 +271,7 @@ jobs: DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- -m flaky --ddtrace + run: hatch run integration-tests tests/functional -k flaky --ddtrace require-label-comment: runs-on: ubuntu-22.04 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3ed8d8b27..d5c8dddf7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,19 +48,12 @@ jobs: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.9' - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - - - name: pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} @@ -72,10 +65,6 @@ jobs: matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - steps: - name: Check out the repository uses: actions/checkout@v4 @@ -88,27 +77,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - #no colons allowed for artifacts - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT + uses: pypa/hatch@install - - uses: actions/upload-artifact@v3 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv + - run: hatch run unit-tests build: name: build packages @@ -130,24 +101,16 @@ jobs: python-version: '3.9' - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version + uses: pypa/hatch@install - name: Build distributions - run: ./scripts/build-dist.sh + run: hatch build - name: Show distributions run: ls -lh dist/ - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 + run: hatch run build:check-all - name: Check if this is an alpha version id: check-is-alpha @@ -175,6 +138,7 @@ jobs: matrix: os: [ubuntu-22.04, macos-14, windows-2022] python-version: ['3.9', '3.10', '3.11', '3.12'] + dist-type: ["whl", "gz"] exclude: # psycopg2-binary doesn't have a precompiled wheel for python 3.9 for mac - os: macos-14 @@ -186,12 +150,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - uses: actions/download-artifact@v3 with: name: dist @@ -200,18 +158,10 @@ jobs: - name: Show distributions run: ls -lh dist/ - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check wheel distributions - run: | - python -c "import dbt.adapters.redshift" - - - name: Install source distributions + - name: Install ${{ matrix.dist-type }} distributions run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check source distributions + - name: Check ${{ matrix.dist-type }} distributions run: | - python -c "import dbt.adapters.redshift" + python -c "import dbt.adapters.bigquery" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index b38929417..e6366524d 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -50,11 +50,15 @@ jobs: commit_sha=$(git rev-parse HEAD) echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - uses: pypa/hatch@install + + - id: version-number-sources + run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" id: semver diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e01b961f2..1bb54cef5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,67 +119,45 @@ defaults: shell: bash jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - uses: ./.github/workflows/release-prep.yml + release-prep: + name: "Release prep: generate changelog, bump version" + uses: ./.github/workflows/release_prep_hatch.yml with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} steps: - - name: Print variables + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-prep.outputs.release-branch }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - uses: pypa/hatch@install + - id: archive run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + - run: hatch build && hatch run build:check-all + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.archive.outputs.name }} + path: dist/ + retention-days: 3 github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] + needs: [build-release, release-prep] uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 000000000..ee092600d --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,455 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string + deploy-to: + description: "Deploy to test or prod" + type: string + default: "prod" + is-nightly-release: + description: "Identify if this is a nightly release" + type: boolean + default: false + outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_DEFAULT_VERSION: 3.9 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_DEFAULT_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_current" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + core-team: + if: needs.release-inputs.outputs.changelog-exists == 'false' + needs: release-inputs + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + secrets: inherit + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - core-team + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - uses: pypa/hatch@install + + - name: "Run unit tests" + run: hatch run unit-tests + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install + + - name: "Run integration tests" + run: hatch run integration-tests + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.result == 'success' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.event.repository.name }}" + uses: actions/checkout@v4 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in + run: | + if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then + branch="${{ needs.release-branch.outputs.name }}" + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.branch.outputs.name }} + + - name: "Set release SHA" + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} diff --git a/hatch.toml b/hatch.toml index c33f5dac5..3a3990a6c 100644 --- a/hatch.toml +++ b/hatch.toml @@ -17,7 +17,7 @@ dependencies = [ "ipdb~=0.13.13", "pre-commit==3.7.0", "freezegun", - "pytest", + "pytest>=7.0,<8.0", "pytest-csv~=3.0", "pytest-dotenv", "pytest-logbook~=1.2", diff --git a/setup.py b/setup.py deleted file mode 100644 index e69de29bb..000000000 From 3119b571edf731fcb0aefa20778b5bead4ddbf6c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:38:11 -0500 Subject: [PATCH 3/7] reduce PR noise --- .env.example => test.env.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .env.example => test.env.example (100%) diff --git a/.env.example b/test.env.example similarity index 100% rename from .env.example rename to test.env.example From 6a659b7de3d24b211f153edd8eb9376e30909861 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:44:48 -0500 Subject: [PATCH 4/7] reduce PR noise --- .github/workflows/integration.yml | 48 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e8d4911fe..46c47fbf6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -100,35 +100,12 @@ jobs: python-version: "3.9" env: - AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} - AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} - AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} - AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user - AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} - AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} - AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} - AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} - AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} - REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} - REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} - REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 steps: - name: Check out the repository (push) @@ -185,8 +162,33 @@ jobs: aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - run: hatch run integration-tests tests/functional -k "not flaky" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 test-flaky: name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky From 7e1c854776fb3c4fd3938d764d4e621ac2f802a8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:57:46 -0500 Subject: [PATCH 5/7] reduce PR noise --- .github/workflows/integration.yml | 4 ++-- .github/workflows/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 46c47fbf6..7f04b1994 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -173,7 +173,7 @@ jobs: AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - - run: hatch run integration-tests tests/functional -k "not flaky" + - run: hatch run integration-tests tests/functional -m "not flaky" env: REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} @@ -273,7 +273,7 @@ jobs: DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 - run: hatch run integration-tests tests/functional -k flaky --ddtrace + run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace require-label-comment: runs-on: ubuntu-22.04 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d5c8dddf7..a036b2efc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: with: python-version: '3.9' - - name: Run pre-commit hooks + - name: pre-commit hooks uses: pre-commit/action@v3.0.1 unit: From 35562c26e157160cad10308cf04492d76f65e7db Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:59:27 -0500 Subject: [PATCH 6/7] changelog --- .changes/unreleased/Under the Hood-20241207-165918.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20241207-165918.yaml diff --git a/.changes/unreleased/Under the Hood-20241207-165918.yaml b/.changes/unreleased/Under the Hood-20241207-165918.yaml new file mode 100644 index 000000000..96d7da708 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241207-165918.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml and to hatch as a dev tool +time: 2024-12-07T16:59:18.731819-05:00 +custom: + Author: mikealfare + Issue: "951" From 34a812e658376c4aad1c23913df770e228acf670 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:42:23 -0500 Subject: [PATCH 7/7] fix pytest config in pyproject.toml --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 51e48b2e6..210dbe478 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,9 @@ Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" [tool.mypy] mypy_path = "third-party-stubs/" -[tool.pytest] +[tool.pytest.ini_options] testpaths = ["tests/functional", "tests/unit"] +env_files = ["test.env"] addopts = "-v -n auto" color = true filterwarnings = [