Move to hatch and pyproject.toml

.changes/unreleased/Under the Hood-20241117-184430.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-11-17T18:44:30.85288-05:00
+custom:
+  Author: mikealfare
+  Issue: "1250"

.github/scripts/update_dependencies.sh

@@ -2,9 +2,9 @@
 set -e
 
 git_branch=$1
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
-tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g"
+tests_req_sed_pattern="s|dbt-adapters.git#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g"
 if [[ "$OSTYPE" == darwin* ]]; then
  # mac ships with a different version of sed that requires a delimiter arg
  sed -i "" "$core_req_sed_pattern" $target_req_file

.github/scripts/update_dev_dependency_branches.sh

@@ -5,8 +5,8 @@ set -e
 dbt_adapters_branch=$1
 dbt_core_branch=$2
 dbt_common_branch=$3
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
 adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
 common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
 if [[ "$OSTYPE" == darwin* ]]; then

.github/workflows/integration.yml

@@ -119,7 +119,8 @@ jobs:
               - '.github/**/*.sh'
               - 'dbt/**'
               - 'tests/**'
-              - 'dev-requirements.txt'
+              - 'hatch.toml'
+              - 'pyproject.toml'
               - '*.py'
       - name: Generate integration test matrix
         id: generate-matrix
@@ -153,8 +154,6 @@ jobs:
       matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
 
     env:
-      TOXENV: integration-${{ matrix.adapter }}
-      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
       DBT_INVOCATION_ENV: github-actions
       DD_CIVISIBILITY_AGENTLESS_ENABLED: true
       DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
@@ -190,24 +189,22 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Update Adapters and Core branches (update dev_requirements.txt)
+      - name: Update Adapters and Core branches (update hatch.toml)
         if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
           ./.github/scripts/update_dev_dependency_branches.sh \
             ${{ inputs.dbt_adapters_branch }} \
             ${{ inputs.dbt_core_branch }} \
             ${{ inputs.dbt_common_branch }}
-          cat dev-requirements.txt
+          cat hatch.toml
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install tox
-          python -m pip --version
-          tox --version
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
-      - name: Run tox (snowflake)
-        if: matrix.adapter == 'snowflake'
+      - uses: pypa/hatch@install
+
+      - run: hatch run integration-tests --ddtrace
         env:
           SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
           SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
@@ -226,26 +223,6 @@ jobs:
           DBT_TEST_USER_1: dbt_test_role_1
           DBT_TEST_USER_2: dbt_test_role_2
           DBT_TEST_USER_3: dbt_test_role_3
-        run: tox -- --ddtrace
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}
-          path: ./logs
-          overwrite: true
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
-          path: integration_results.csv
-          overwrite: true
 
   require-label-comment:
     runs-on: ubuntu-latest

.github/workflows/main.yml

@@ -52,16 +52,8 @@ jobs:
         with:
           python-version: '3.9'
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install -r dev-requirements.txt
-          python -m pip --version
-          pre-commit --version
-          dbt --version
-
       - name: Run pre-commit hooks
-        run: pre-commit run --all-files --show-diff-on-failure
+        uses: pre-commit/[email protected]
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -73,10 +65,6 @@ jobs:
       matrix:
         python-version: ['3.9', '3.10', '3.11', '3.12']
 
-    env:
-      TOXENV: "unit"
-      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
-
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -89,26 +77,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install tox
-          python -m pip --version
-          tox --version
+        uses: pypa/hatch@install
 
-      - name: Run tox
-        run: tox
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: unit_results.csv
-          overwrite: true
+      - run: hatch run unit-tests
 
   build:
     name: build packages
@@ -129,25 +100,16 @@ jobs:
         with:
           python-version: '3.9'
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
-          python -m pip --version
+      - uses: pypa/hatch@install
 
       - name: Build distributions
-        run: ./scripts/build-dist.sh
+        run: hatch build
 
-      - name: Show distributions
+      - name: Show distribution contents
         run: ls -lh dist/
 
       - name: Check distribution descriptions
-        run: |
-          twine check dist/*
-
-      - name: Check wheel contents
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
+        run: hatch run build:check-all
 
       - name: Check if this is an alpha version
         id: check-is-alpha
@@ -184,12 +146,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade wheel
-          python -m pip --version
-
       - uses: actions/download-artifact@v4
         with:
           name: dist

.github/workflows/release.yml

@@ -119,94 +119,78 @@ defaults:
     shell: bash
 
 jobs:
-  log-inputs:
-    name: Log Inputs
-    runs-on: ubuntu-latest
-    steps:
-      - name: "[DEBUG] Print Variables"
-        run: |
-          echo The last commit sha in the release: ${{ inputs.sha }}
-          echo The branch to release from:         ${{ inputs.target_branch }}
-          echo The release version number:         ${{ inputs.version_number }}
-          echo Build script path:                  ${{ inputs.build_script_path }}
-          echo Environment setup script path:      ${{ inputs.env_setup_script_path }}
-          echo AWS S3 bucket name:                 ${{ inputs.s3_bucket_name }}
-          echo Package test command:               ${{ inputs.package_test_command }}
-          echo Test run:                           ${{ inputs.test_run }}
-          echo Nightly release:                    ${{ inputs.nightly_release }}
-          echo Only Docker:                        ${{ inputs.only_docker }}
-
-  bump-version-generate-changelog:
-    name: Bump package version, Generate changelog
-    uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
+  release-prep:
+    name: "Release prep: generate changelog, bump version"
+    uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main
     with:
-      sha: ${{ inputs.sha }}
-      version_number: ${{ inputs.version_number }}
-      target_branch: ${{ inputs.target_branch }}
-      env_setup_script_path: ${{ inputs.env_setup_script_path }}
-      test_run: ${{ inputs.test_run }}
-      nightly_release: ${{ inputs.nightly_release }}
+      branch: ${{ inputs.branch }}
+      version: ${{ inputs.version }}
+      deploy-to: ${{ inputs.deploy-to }}
     secrets: inherit
 
-  log-outputs-bump-version-generate-changelog:
-    name: "[Log output] Bump package version, Generate changelog"
-    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
-    needs: [bump-version-generate-changelog]
+  build-release:
+    name: "Build release"
+    needs: release-prep
     runs-on: ubuntu-latest
+    outputs:
+      archive-name: ${{ steps.archive.outputs.name }}
     steps:
-      - name: Print variables
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.release-prep.outputs.release-branch }}
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python_version }}
+      - uses: pypa/hatch@install
+      - id: archive
         run: |
-          echo Final SHA     : ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
-          echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
-
-  build-test-package:
-    name: Build, Test, Package
-    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
-    needs: [bump-version-generate-changelog]
-    uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
-    with:
-      sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
-      version_number: ${{ inputs.version_number }}
-      changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
-      build_script_path: ${{ inputs.build_script_path }}
-      s3_bucket_name: ${{ inputs.s3_bucket_name }}
-      package_test_command: ${{ inputs.package_test_command }}
-      test_run: ${{ inputs.test_run }}
-      nightly_release: ${{ inputs.nightly_release }}
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }}
+          echo "name=$archive_name" >> $GITHUB_OUTPUT
+      - run: hatch build && hatch run build:check-all
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.archive.outputs.name }}
+          path: dist/
+          retention-days: 3
 
   github-release:
     name: GitHub Release
     if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
-    needs: [bump-version-generate-changelog, build-test-package]
-    uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
+    needs: [build-release, release-prep]
+    uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main
     with:
       sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
       changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
       test_run: ${{ inputs.test_run }}
+      archive_name: ${{ needs.build-release.outputs.archive-name }}
 
   pypi-release:
     name: PyPI Release
     if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
-    needs: [bump-version-generate-changelog, build-test-package]
-    uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
-    with:
-      version_number: ${{ inputs.version_number }}
-      test_run: ${{ inputs.test_run }}
-    secrets:
-      PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
-      TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
+    needs: build-release
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.deploy-to }}
+      url: ${{ vars.PYPI_PROJECT_URL }}
+    permissions:
+      # this permission is required for trusted publishing
+      # see https://github.com/marketplace/actions/pypi-publish
+      id-token: write
+    steps:
+      - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main
+        with:
+          repository-url: ${{ vars.PYPI_REPOSITORY_URL }}
+          archive-name: ${{ needs.build-release.outputs.archive-name }}
 
   docker-release:
     name: "Docker Release"
     # We cannot release to docker on a test run because it uses the tag in GitHub as
     # what we need to release but draft releases don't actually tag the commit so it
     # finds nothing to release
     if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
-    needs: [bump-version-generate-changelog, build-test-package, github-release]
+    needs: github-release
     permissions:
       packages: write
     uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main