From 793ed23953c98b781404c661123faf4c560f7236 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:18:50 -0500 Subject: [PATCH 01/20] move setup.py to pyproject.toml --- pyproject.toml | 57 ++++++++++++++++++++++++++++++++++++ setup.py | 78 -------------------------------------------------- 2 files changed, 57 insertions(+), 78 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e04e6590b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-bigquery" +description = "The BigQuery adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "bigquery", "google"] +requires-python = ">=3.9.0" +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.7,<2.0", + # 3.20 introduced pyarrow>=3.0 under the `pandas` extra + "google-cloud-bigquery[pandas]>=3.0,<4.0", + "google-cloud-storage~=2.4", + "google-cloud-dataproc~=5.0", + # ---- + # Expect compatibility with all new versions of these packages, so lower bounds only. + "google-api-core>=2.11.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-bigquery" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-bigquery.git" +Issues = "https://github.com/dbt-labs/dbt-bigquery/issues" +Changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/main/CHANGELOG.md" + +[tool.pytest] +testpaths = ["tests/functional", "tests/unit"] +addopts = "-v -n auto" +color = true +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 79f6025ea..000000000 --- a/setup.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -import sys - -# require a supported version of Python -if sys.version_info < (3, 9): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.9 or higher.") - sys.exit(1) - -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again') - sys.exit(1) - -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/bigquery/__version__.py" - - -def _dbt_bigquery_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -package_name = "dbt-bigquery" -description = """The BigQuery adapter plugin for dbt""" - -setup( - name="dbt-bigquery", - version=_dbt_bigquery_version(), - description="The Bigquery adapter plugin for dbt", - long_description=README.read_text(), - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-bigquery", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", - # 3.20 introduced pyarrow>=3.0 under the `pandas` extra - "google-cloud-bigquery[pandas]>=3.0,<4.0", - "google-cloud-storage~=2.4", - "google-cloud-dataproc~=5.0", - # ---- - # Expect compatibility with all new versions of these packages, so lower bounds only. - "google-api-core>=2.11.0", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], - python_requires=">=3.9", -) From 256ff9c43d96e2422cf6780bb6c65c3ed8eae45e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:25:09 -0500 Subject: [PATCH 02/20] move dev tool config to pyproject.toml --- .bumpversion.cfg | 35 ------------------ test.env.example => .env.example | 0 MANIFEST.in | 1 - Makefile | 21 ----------- dev-requirements.txt | 20 ----------- hatch.toml | 61 ++++++++++++++++++++++++++++++++ mypy.ini | 2 -- pyproject.toml | 3 ++ pytest.ini | 10 ------ tox.ini | 54 ---------------------------- 10 files changed, 64 insertions(+), 143 deletions(-) delete mode 100644 .bumpversion.cfg rename test.env.example => .env.example (100%) delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 hatch.toml delete mode 100644 mypy.ini delete mode 100644 pytest.ini delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index bd9430cbe..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.9.0b1 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/bigquery/__version__.py] diff --git a/test.env.example b/.env.example similarity index 100% rename from test.env.example rename to .env.example diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index bdacb538b..000000000 --- a/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-bigquery - -.PHONY: docker-dev -docker-dev: - docker build -f docker/dev.Dockerfile -t dbt-bigquery-dev . - docker run --rm -it --name dbt-bigquery-dev -v $(shell pwd):/opt/code dbt-bigquery-dev - -.PHONY: docker-prod -docker-prod: - docker build -f docker/Dockerfile -t dbt-bigquery . diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 2c0134110..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# install latest changes in dbt-core -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core - -# dev -ddtrace==2.3.0 -pre-commit~=3.7.0 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.6 -tox~=4.11 - -# build -bumpversion~=0.6.0 -twine~=5.1 -wheel~=0.43 diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 000000000..e65f81313 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,61 @@ +[version] +path = "dbt/adapters/bigquery/__version__.py" + +[build.targets.sdist] +packages = ["dbt"] + +[build.targets.wheel] +packages = ["dbt"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "ddtrace==2.3.0", + "ipdb~=0.13.13", + "pre-commit==3.7.0", + "freezegun", + "pytest", + "pytest-csv~=3.0", + "pytest-dotenv", + "pytest-logbook~=1.2", + "pytest-mock", + "pytest-xdist", +] + +[envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-bigquery-dev .", + "docker run --rm -it --name dbt-bigquery-dev -v $(shell pwd):/opt/code dbt-bigquery-dev", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] + +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt_bigquery-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-bigquery", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_bigquery-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-bigquery", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-bigquery ." diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 247a47fec..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -mypy_path = third-party-stubs/ diff --git a/pyproject.toml b/pyproject.toml index e04e6590b..c710084a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,9 @@ Repository = "https://github.com/dbt-labs/dbt-bigquery.git" Issues = "https://github.com/dbt-labs/dbt-bigquery/issues" Changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/main/CHANGELOG.md" +[tool.mypy] +mypy_path = "third-party-stubs/" + [tool.pytest] testpaths = ["tests/functional", "tests/unit"] addopts = "-v -n auto" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b04a6ccf3..000000000 --- a/pytest.ini +++ /dev/null @@ -1,10 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/integration - tests/functional diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 240d85e34..000000000 --- a/tox.ini +++ /dev/null @@ -1,54 +0,0 @@ -[tox] -skipsdist = True -envlist = py39,py310,py311,py312 - -[testenv:{unit,py39,py310,py311,py312,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py39,py310,py311,py312,py}-{bigquery}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - BIGQUERY_TEST_* - PYTEST_ADDOPTS - DATAPROC_* - GCS_BUCKET - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - bigquery: {envpython} -m pytest -n auto {posargs} -vv tests/functional -k "not TestPython" --profile service_account -deps = - -rdev-requirements.txt - . - -[testenv:{python-tests,py39,py310,py311,py312,py}] -description = python integration testing -skip_install = true -passenv = - DBT_* - BIGQUERY_TEST_* - PYTEST_ADDOPTS - DATAPROC_* - GCS_BUCKET - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - {envpython} -m pytest {posargs} -vv tests/functional -k "TestPython" --profile service_account -deps = - -rdev-requirements.txt - -e. From b0ba2b9f96f5e5a07544b04302fee92f79f656c3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:34:52 -0500 Subject: [PATCH 03/20] update integration.yml to use hatch commands --- .github/workflows/integration.yml | 62 +++++++------------------------ 1 file changed, 13 insertions(+), 49 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a9179f9ce..2401c4fec 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -127,7 +127,8 @@ jobs: bigquery: - 'dbt/**' - 'tests/**' - - 'dev-requirements.txt' + - 'hatch.toml' + - 'pyproject.toml' - '.github/**' - '*.py' @@ -164,8 +165,6 @@ jobs: matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} @@ -196,18 +195,6 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - name: Update Adapters and Core branches (update dev_requirements.txt) if: github.event_name == 'workflow_dispatch' run: | @@ -217,8 +204,13 @@ jobs: ${{ inputs.dbt_common_branch }} cat dev-requirements.txt - - name: Run tox (bigquery) - if: matrix.adapter == 'bigquery' + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - uses: pypa/hatch@install + + - run: hatch run integration-tests -k "not TestPython" --ddtrace env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} @@ -229,27 +221,6 @@ jobs: DATAPROC_REGION: us-central1 DATAPROC_CLUSTER_NAME: dbt-test-1 GCS_BUCKET: dbt-ci - run: tox -- --ddtrace - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }} - path: ./logs - overwrite: true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - overwrite: true # python integration tests are slow so we only run them seperately and for a single OS / python version test-python: @@ -281,19 +252,13 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python 3.9 - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version + - uses: pypa/hatch@install - - name: Run tox (python models) + - run: hatch run integration-tests -k "TestPython" --ddtrace env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} @@ -304,7 +269,6 @@ jobs: DATAPROC_REGION: us-central1 DATAPROC_CLUSTER_NAME: dbt-test-1 GCS_BUCKET: dbt-ci - run: tox -e python-tests -- --ddtrace require-label-comment: runs-on: ubuntu-latest From fde4361aea1e0b14401b87cb205461ff46056a47 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:40:56 -0500 Subject: [PATCH 04/20] update main.yml to use hatch commands --- .github/workflows/main.yml | 58 +++++--------------------------------- 1 file changed, 7 insertions(+), 51 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b82f3e0f..9e5ab6342 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,15 +52,8 @@ jobs: with: python-version: '3.9' - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - dbt --version - - name: Run pre-comit hooks - run: pre-commit run --all-files --show-diff-on-failure + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} @@ -72,10 +65,6 @@ jobs: matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - steps: - name: Check out the repository uses: actions/checkout@v4 @@ -88,27 +77,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox + uses: pypa/hatch@install - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - overwrite: true + - run: hatch run unit-tests build: name: build packages @@ -129,25 +100,16 @@ jobs: with: python-version: '3.9' - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version + - uses: pypa/hatch@install - name: Build distributions - run: ./scripts/build-dist.sh + run: hatch build - name: Show distributions run: ls -lh dist/ - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 + run: hatch run build:check-all - name: Check if this is an alpha version id: check-is-alpha @@ -184,12 +146,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - uses: actions/download-artifact@v4 with: name: dist From bfddb55a6a962c612a2bda1848035ec381f94842 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:46:59 -0500 Subject: [PATCH 05/20] update scripts for pyproject.toml and hatch.toml references --- .github/scripts/update_dependencies.sh | 6 +++--- .github/scripts/update_dev_dependency_branches.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index c3df48e52..fabdadff2 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -2,9 +2,9 @@ set -e git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 022df6a8a..9385cf885 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -5,8 +5,8 @@ set -e dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then From 304e7a33fb42a8e7b271174c26389c327b51f96b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:47:59 -0500 Subject: [PATCH 06/20] changelog --- .changes/unreleased/Under the Hood-20241117-194746.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20241117-194746.yaml diff --git a/.changes/unreleased/Under the Hood-20241117-194746.yaml b/.changes/unreleased/Under the Hood-20241117-194746.yaml new file mode 100644 index 000000000..e8658ee20 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241117-194746.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml and to hatch as a dev tool +time: 2024-11-17T19:47:46.341-05:00 +custom: + Author: mikealfare + Issue: "1407" From 6faf54d1f53fb17e52c046754070b7508f109e58 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 20:15:48 -0500 Subject: [PATCH 07/20] update release workflow to use hatch --- .github/workflows/release.yml | 108 +++--- .github/workflows/release_prep_hatch.yml | 440 +++++++++++++++++++++++ 2 files changed, 486 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad7cf76b4..18fa346e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,86 +119,70 @@ defaults: shell: bash jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Only Docker: ${{ inputs.only_docker }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + release-prep: + name: "Release prep: generate changelog, bump version" + uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} steps: - - name: Print variables + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-prep.outputs.release-branch }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - uses: pypa/hatch@install + - id: archive run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + - run: hatch build && hatch run build:check-all + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.archive.outputs.name }} + path: dist/ + retention-days: 3 github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + needs: [build-release, release-prep] + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} test_run: ${{ inputs.test_run }} + archive_name: ${{ needs.build-release.outputs.archive-name }} pypi-release: name: PyPI Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + needs: build-release + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ needs.build-release.outputs.archive-name }} docker-release: name: "Docker Release" @@ -206,7 +190,7 @@ jobs: # what we need to release but draft releases don't actually tag the commit so it # finds nothing to release if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} - needs: [bump-version-generate-changelog, build-test-package, github-release] + needs: github-release permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 000000000..dde861251 --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,440 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string + deploy-to: + description: "Deploy to test or prod" + type: string + default: "prod" + is-nightly-release: + description: "Identify if this is a nightly release" + type: boolean + default: false + outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.12 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_current" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + core-team: + if: needs.release-inputs.outputs.changelog-exists == 'false' + needs: release-inputs + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + secrets: inherit + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - core-team + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run unit tests" + run: hatch run unit-tests + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Set up `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run integration tests" + run: hatch run integration-tests + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.result == 'success' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.event.repository.name }}" + uses: actions/checkout@v4 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in + run: | + if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then + branch="${{ needs.release-branch.outputs.name }}" + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.branch.outputs.name }} + + - name: "Set release SHA" + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} From 6c7fd0074a3746b63849d0c2c912afbf3e733d72 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 20:22:18 -0500 Subject: [PATCH 08/20] update ref to dev-requirements.txt to be hatch.toml --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2401c4fec..96d4309ae 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -202,7 +202,7 @@ jobs: ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt + cat hatch.toml - uses: actions/setup-python@v5 with: From cc9da7cd45ac5b38ef07a68ea9b12b77b41f0bee Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 20:40:18 -0500 Subject: [PATCH 09/20] remove dd from pytest call --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 96d4309ae..cd6d31943 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -210,7 +210,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests -k "not TestPython" --ddtrace + - run: hatch run integration-tests -k "not TestPython" env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} From 6b74160c1377582a91917fedfec2c3199821abe5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 20:48:36 -0500 Subject: [PATCH 10/20] simplify pyproject.toml --- pyproject.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c710084a2..014f211fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,8 @@ description = "The BigQuery adapter plugin for dbt" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "bigquery", "google"] requires-python = ">=3.9.0" -authors = [ - { name = "dbt Labs", email = "info@dbtlabs.com" }, -] -maintainers = [ - { name = "dbt Labs", email = "info@dbtlabs.com" }, -] +authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", From d92f63ebe2342511dca0b096ff93d599c55eb48a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 00:54:01 -0500 Subject: [PATCH 11/20] more hatch updates --- .bumpversion.cfg | 0 {scripts => .github/scripts}/env-setup.sh | 0 .github/workflows/nightly-release.yml | 19 ++++++----- .github/workflows/release.yml | 6 ++-- .github/workflows/release_prep_hatch.yml | 39 ++++++++++++++++------- .github/workflows/version-bump.yml | 28 ---------------- scripts/build-dist.sh | 20 ------------ 7 files changed, 39 insertions(+), 73 deletions(-) delete mode 100644 .bumpversion.cfg rename {scripts => .github/scripts}/env-setup.sh (100%) delete mode 100644 .github/workflows/version-bump.yml delete mode 100755 scripts/build-dist.sh diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/env-setup.sh b/.github/scripts/env-setup.sh similarity index 100% rename from scripts/env-setup.sh rename to .github/scripts/env-setup.sh diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 1dc396154..c986a4d0f 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -50,11 +50,15 @@ jobs: commit_sha=$(git rev-parse HEAD) echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - uses: pypa/hatch@install + + - id: version-number-sources + run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" id: semver @@ -108,10 +112,5 @@ jobs: sha: ${{ needs.aggregate-release-data.outputs.commit_sha }} target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - build_script_path: "scripts/build-dist.sh" - env_setup_script_path: "scripts/env-setup.sh" - s3_bucket_name: "core-team-artifacts" - package_test_command: "dbt -h" - test_run: true nightly_release: true secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18fa346e6..15840e5ed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,7 @@ on: env_setup_script_path: description: "Environment setup script path" type: string - default: "scripts/env-setup.sh" + default: "./.github/scripts/env-setup.sh" required: false s3_bucket_name: description: "AWS S3 bucket name" @@ -88,7 +88,7 @@ on: env_setup_script_path: description: "Environment setup script path" type: string - default: "scripts/env-setup.sh" + default: "./.github/scripts/env-setup.sh" required: false s3_bucket_name: description: "AWS S3 bucket name" @@ -121,7 +121,7 @@ defaults: jobs: release-prep: name: "Release prep: generate changelog, bump version" - uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main + uses: ./.github/workflows/release_prep_hatch.yml with: branch: ${{ inputs.branch }} version: ${{ inputs.version }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index dde861251..ee092600d 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -77,7 +77,7 @@ defaults: shell: bash env: - PYTHON_TARGET_VERSION: 3.12 + PYTHON_DEFAULT_VERSION: 3.9 NOTIFICATION_PREFIX: "[Release Prep]" jobs: @@ -100,7 +100,7 @@ jobs: echo Deploy to: ${{ inputs.deploy-to }} echo Nightly release: ${{ inputs.is-nightly-release }} # ENVIRONMENT VARIABLES - echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Python version: ${{ env.PYTHON_DEFAULT_VERSION }} echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" @@ -108,8 +108,11 @@ jobs: with: ref: ${{ inputs.branch }} - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install - name: "Parse input version" id: semver @@ -223,8 +226,11 @@ jobs: with: ref: ${{ needs.release-branch.outputs.name }} - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install - name: "Install `changie`" run: | @@ -296,8 +302,11 @@ jobs: with: ref: ${{ needs.release-branch.outputs.name }} - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install - name: "Bump version to ${{ inputs.version }}" run: hatch version ${{ inputs.version }} @@ -338,8 +347,11 @@ jobs: with: ref: ${{ needs.release-branch.outputs.name }} - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - uses: pypa/hatch@install - name: "Run unit tests" run: hatch run unit-tests @@ -361,8 +373,11 @@ jobs: with: ref: ${{ needs.release-branch.outputs.name }} - - name: "Set up `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + + - uses: pypa/hatch@install - name: "Run integration tests" run: hatch run integration-tests diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml deleted file mode 100644 index bde34d683..000000000 --- a/.github/workflows/version-bump.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# This workflow will take the new version number to bump to. With that -# it will run versionbump to update the version number everywhere in the -# code base and then run changie to create the corresponding changelog. -# A PR will be created with the changes that can be reviewed before committing. - -# **why?** -# This is to aid in releasing dbt and making sure we have updated -# the version in all places and generated the changelog. - -# **when?** -# This is triggered manually - -name: Version Bump - -on: - workflow_dispatch: - inputs: - version_number: - description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' - required: true - -jobs: - version_bump_and_changie: - uses: dbt-labs/actions/.github/workflows/version-bump.yml@main - with: - version_number: ${{ inputs.version_number }} - secrets: inherit # ok since what we are calling is internally maintained diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh deleted file mode 100755 index 3c3808399..000000000 --- a/scripts/build-dist.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" - -PYTHON_BIN=${PYTHON_BIN:-python} - -echo "$PYTHON_BIN" - -set -x - -rm -rf "$DBT_PATH"/dist -rm -rf "$DBT_PATH"/build -mkdir -p "$DBT_PATH"/dist - -cd "$DBT_PATH" -$PYTHON_BIN setup.py sdist bdist_wheel - -set +x From f297c23472dbaea33effb839a68a9362a10ca3c7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 00:59:22 -0500 Subject: [PATCH 12/20] revert 3.9 abstraction --- .github/workflows/integration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cd6d31943..730126c6c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -204,9 +204,10 @@ jobs: ${{ inputs.dbt_common_branch }} cat hatch.toml - - uses: actions/setup-python@v5 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.9" - uses: pypa/hatch@install From 2f0b18cb3b4497c7c755f8ea82ab43700de5fcac Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 01:00:49 -0500 Subject: [PATCH 13/20] make updates for PR readability --- .github/workflows/integration.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 730126c6c..3932e2620 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -195,6 +195,13 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - uses: pypa/hatch@install + - name: Update Adapters and Core branches (update dev_requirements.txt) if: github.event_name == 'workflow_dispatch' run: | @@ -204,13 +211,6 @@ jobs: ${{ inputs.dbt_common_branch }} cat hatch.toml - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - uses: pypa/hatch@install - - run: hatch run integration-tests -k "not TestPython" env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} From a30dd19aad767ade1032e917ceb22d6db5232adc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 01:02:40 -0500 Subject: [PATCH 14/20] make updates for PR readability --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3932e2620..8e2ff404c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -195,10 +195,10 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python 3.9 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: ${{ matrix.python-version }} - uses: pypa/hatch@install From 0e15b74e0a7e6d25425b54ac2a7bd46b589a064c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 01:05:31 -0500 Subject: [PATCH 15/20] make updates for PR readability --- .github/workflows/integration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8e2ff404c..86601df4a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -253,9 +253,10 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-python@v5 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.9" - uses: pypa/hatch@install From 05f0375f44b697b15728b270639633ebda1a783d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 02:52:59 -0500 Subject: [PATCH 16/20] update runners for main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e5ab6342..9ad70ce5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -136,7 +136,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-14, windows-latest] + os: [ubuntu-22.04, macos-14, windows-2022] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ["whl", "gz"] From 32301141e0b2a1f62f2015b54ca892a8a47704da Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:13:09 -0500 Subject: [PATCH 17/20] pin pytest for ddtrace --- hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch.toml b/hatch.toml index e65f81313..5515a81f0 100644 --- a/hatch.toml +++ b/hatch.toml @@ -17,7 +17,7 @@ dependencies = [ "ipdb~=0.13.13", "pre-commit==3.7.0", "freezegun", - "pytest", + "pytest>=7.0,<8.0", "pytest-csv~=3.0", "pytest-dotenv", "pytest-logbook~=1.2", From a08c5650c68eedd41bf336463b3593a55f2a82bb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:14:27 -0500 Subject: [PATCH 18/20] explicitly limit integration tests to tests/functional --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 86601df4a..34ebadcfe 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -211,7 +211,7 @@ jobs: ${{ inputs.dbt_common_branch }} cat hatch.toml - - run: hatch run integration-tests -k "not TestPython" + - run: hatch run integration-tests tests/functional -k "not TestPython" env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} @@ -260,7 +260,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests -k "TestPython" --ddtrace + - run: hatch run integration-tests tests/functional -k "TestPython" --ddtrace env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} From 6073a1f3fd16c0a1212022dc4965120f38304b30 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:27:01 -0500 Subject: [PATCH 19/20] add the service-account profile flag for integration tests --- hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch.toml b/hatch.toml index 5515a81f0..34ba6d2a3 100644 --- a/hatch.toml +++ b/hatch.toml @@ -29,7 +29,7 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" -integration-tests = "python -m pytest {args:tests/functional}" +integration-tests = "python -m pytest --profile service_account {args:tests/functional}" docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-bigquery-dev .", "docker run --rm -it --name dbt-bigquery-dev -v $(shell pwd):/opt/code dbt-bigquery-dev", From d2174f816e529e8aa32a3f535ede49556670b9ce Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:46:21 -0500 Subject: [PATCH 20/20] fix pytest config in pyproject.toml --- .github/workflows/integration.yml | 6 ++---- pyproject.toml | 3 ++- .env.example => test.env.example | 0 3 files changed, 4 insertions(+), 5 deletions(-) rename .env.example => test.env.example (100%) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 34ebadcfe..32d937ef8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -226,11 +226,9 @@ jobs: # python integration tests are slow so we only run them seperately and for a single OS / python version test-python: name: "test-python" - needs: test-metadata + needs: test runs-on: ubuntu-latest if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && ( github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || @@ -260,7 +258,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests tests/functional -k "TestPython" --ddtrace + - run: hatch run integration-tests tests/functional -n1 -k "TestPython" --ddtrace env: BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} diff --git a/pyproject.toml b/pyproject.toml index 014f211fc..b2d55b25f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,9 @@ Changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/main/CHANGELOG.md" [tool.mypy] mypy_path = "third-party-stubs/" -[tool.pytest] +[tool.pytest.ini_options] testpaths = ["tests/functional", "tests/unit"] +env_files = ["test.env"] addopts = "-v -n auto" color = true filterwarnings = [ diff --git a/.env.example b/test.env.example similarity index 100% rename from .env.example rename to test.env.example