diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..bb4659c8a --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,44 @@ +name: "Build artifacts" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run integration tests on" + type: string + required: true + archive-name: + description: "Name of the archive for build artifacts, used during a release" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + build-artifacts: + name: "Build artifacts" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: "Setup environment" + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions + with: + python-version: ${{ matrix.python-version }} + + - name: "Build artifacts" + uses: dbt-labs/actions/hatch/artifacts/create.yml@add-hatch-actions + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/ci-test-suite.yml b/.github/workflows/ci-test-suite.yml index ada117771..8fde14f37 100644 --- a/.github/workflows/ci-test-suite.yml +++ b/.github/workflows/ci-test-suite.yml @@ -24,30 +24,22 @@ jobs: uses: dbt-labs/actions/.github/workflows/code-quality.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - check-command: "hatch run code-quality" - python-version: ${{ vars.DBT_PYTHON_VERSION }} unit-tests: name: "Run unit tests" uses: dbt-labs/actions/.github/workflows/unit-tests.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - test-command: "hatch run unit-tests:all" - python-versions: ${{ vars.DBT_PYTHON_VERSIONS }} integration-tests: name: "Run integration tests" uses: dbt-labs/actions/.github/workflows/integration-tests.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - test-command: "hatch run integration-tests:all" - python-versions-ubuntu: ${{ vars.DBT_PYTHON_VERSIONS }} - python-versions-macos: ${{ vars.DBT_PYTHON_VERSION }} - python-versions-windows: ${{ vars.DBT_PYTHON_VERSION }} build-artifacts: name: "Verify build artifacts" - uses: dbt-labs/actions/.github/workflows/build-release-artifacts.yml@add-hatch-actions + uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 1acd20586..7f32602e5 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -17,14 +17,6 @@ on: description: "The branch/tag/commit to run code quality on" type: string required: true - check-command: - description: "The command to run code quality checks" - type: string - required: true - python-version: - description: "The python version to test against" - type: string - required: true permissions: read-all @@ -44,18 +36,17 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.DBT_PYTHON_VERSION }} - name: "Run code quality" shell: bash - run: ${{ inputs.check-command }} + run: hatch run code-quality - name: "[INFO] Run code quality" shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Run code quality" MESSAGE: "Code quality checks completed successfully!" diff --git a/.github/workflows/daily-dev-releases.yml b/.github/workflows/daily-dev-releases.yml index d288532bf..3c961eb2b 100644 --- a/.github/workflows/daily-dev-releases.yml +++ b/.github/workflows/daily-dev-releases.yml @@ -36,6 +36,11 @@ jobs: - name: "Checkout `${{ github.repository }}`" uses: actions/checkout@v3 + - name: "Setup environment" + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions + with: + ref: ${{ vars.DBT_PYTHON_VERSION }} + - name: "Set: commit" id: commit run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT @@ -45,7 +50,7 @@ jobs: run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" - id: semver + id: version uses: dbt-labs/actions/parse-semver@v1.1.0 with: version: ${{ steps.current-version.outputs.version }} @@ -53,9 +58,9 @@ jobs: - name: "Set: release version" id: release-version run: | - version="${{ steps.semver.outputs.major }}" - version+=".${{ steps.semver.outputs.minor }}" - version+=".$(${{ steps.semver.outputs.patch }}+1)" + version="${{ steps.version.outputs.major }}" + version+=".${{ steps.version.outputs.minor }}" + version+=".$(${{ steps.version.outputs.patch }}+1)" version+="dev$(date +'%m%d%Y')" echo "version=$version" >> $GITHUB_OUTPUT diff --git a/.github/workflows/dev-branch-integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml deleted file mode 100644 index ace39b778..000000000 --- a/.github/workflows/dev-branch-integration-tests.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: "Dev branch integration tests" - -on: - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }}" - cancel-in-progress: true - -jobs: - require-approval-on-forks: - # if it's a pull request from a fork that has not been approved yet, don't run tests - if: >- - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository && - !contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: "Author requires permission to run tests" - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "Thanks for your contribution! "\ - "This PR was created from a fork, integration tests need to be approved prior to running. "\ - "@dbt-labs/core will review this PR and approve running integration tests." - check_for_duplicate_msg: true - - integration-tests: - name: "Integration tests" - needs: require-approval-on-forks - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} - include: - - os: macos-latest - python-version: ${{ vars.DBT_PYTHON_VERSION }} - - os: windows-latest - python-version: ${{ vars.DBT_PYTHON_VERSION }} - env: - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} - REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} - DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} - DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} - DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: ${{ vars.DD_SITE }} - DD_ENV: ${{ vars.DD_ENV }} - DD_SERVICE: ${{ github.event.repository.name }} - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/actions/setup-environment@add-hatch-actions - with: - python-version: ${{ matrix.python-version }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - dbt-core-branch: ${{ inputs.dbt-core-branch }} - - - name: Create AWS IAM profile - run: | - aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID - aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY - aws configure set region $REDSHIFT_TEST_REGION - aws configure set output json - - - name: "Run integration tests" - shell: bash - run: hatch run integration-tests:all - - aggregate-results: - name: "Collect integration tests results" - needs: integration-tests - if: always() && !cancelled() - runs-on: ubuntu-latest - steps: - - name: "[INFO] Integration tests - FAILED" - if: failure() - shell: bash - run: | - echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" - exit 1 - env: - WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} - TITLE: "Integration tests - FAILED" - MESSAGE: "Integration tests failed!" - - - name: "[INFO] Integration tests - PASSED" - if: ${{ !failure() }} # includes skipped - shell: bash - run: | - echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" - env: - WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} - TITLE: "Integration tests - PASSED" - MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3024d3fb3..e92168069 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,22 +7,36 @@ on: description: "The branch/tag/commit to run integration tests on" type: string required: true - test-command: - description: "The command to run unit tests" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" type: string - required: true - python-versions-ubuntu: - description: "The python versions to test against for ubuntu" + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" type: string - required: true - python-versions-macos: - description: "The python version to test against for macos" + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" type: string - required: true - python-versions-windows: - description: "The python version to test against for windows" + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" type: string - required: true + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "" permissions: read-all @@ -52,6 +66,10 @@ jobs: "@dbt-labs/core will review this PR and approve running integration tests." check_for_duplicate_msg: true + - name: "Exit" + shell: bash + run: exit 0 + integration-tests: name: "Integration tests" needs: require-approval-on-forks @@ -60,19 +78,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ${{ fromJSON(inputs.python-versions-ubuntu) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} include: - os: macos-latest - python-version: ${{ fromJSON(inputs.python-versions-macos) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSION) }} - os: windows-latest - python-version: ${{ fromJSON(inputs.python-versions-windows) }} - env: - DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: ${{ vars.DD_SITE }} - DD_ENV: ${{ vars.DD_ENV }} - DD_SERVICE: ${{ github.event.repository.name }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSION) }} steps: - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" uses: actions/checkout@v4 @@ -81,13 +92,47 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment@add-hatch-actions + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: python-version: ${{ matrix.python-version }} + - name: "Update development branches" + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Create AWS IAM profile" + run: | + aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID + aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY + aws configure set region $REDSHIFT_TEST_REGION + aws configure set output json + - name: "Run integration tests" shell: bash - run: ${{ inputs.test-command }} + run: hatch run integration-tests:all + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} + REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} aggregate-results: name: "Collect integration tests results" @@ -98,9 +143,7 @@ jobs: - name: "[INFO] Integration tests - FAILED" if: failure() shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" - exit 1 + run: echo "::notice title=$TITLE::$MESSAGE" && exit 1 env: TITLE: "Integration tests - FAILED" MESSAGE: "Integration tests failed!" @@ -108,8 +151,7 @@ jobs: - name: "[INFO] Integration tests - PASSED" if: ${{ !failure() }} # includes skipped shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Integration tests - PASSED" MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 768805ac2..1305d9c80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,37 +111,27 @@ jobs: - name: "Create release branch" id: branch - uses: dbt-labs/actions/create-temp-branch@add-hatch-actions + uses: dbt-labs/actions/github/branch/create@add-hatch-actions with: - branch-stub: "release/${{ inputs.version }}/" + branch-group: "release/${{ inputs.version }}" - bump-version: - name: "Bump version" - needs: release-branch - uses: dbt-labs/actions/.github/workflows/build-bump-version.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - version: ${{ inputs.version }} - secrets: inherit + - name: "Bump version" + uses: dbt-labs/actions/hatch/version/build.yml@add-hatch-actions + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} - changelog: - name: "Generate a new changelog" - needs: - - release-branch - - bump-version - uses: dbt-labs/actions/.github/workflows/build-changelog.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - version: ${{ inputs.version }} - secrets: inherit + - name: "Generate a new changelog" + uses: dbt-labs/actions/changie/changelog/create.yml@add-hatch-actions + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} ci-test-suite: name: "Run code quality, unit tests, and integration tests, and build artifacts" needs: - release-inputs - release-branch - - version-bump - - changelog uses: ./.github/workflows/ci-test-suite.yml with: ref: ${{ needs.release-branch.outputs.branch }} @@ -164,7 +154,7 @@ jobs: - name: "Merge `${{ needs.release-branch.outputs.branch }}` into `${{ inputs.branch }}`" if: ${{ inputs.deploy-environment == 'prod' }} id: merge - uses: dbt-labs/actions/github-merge@add-hatch-actions + uses: dbt-labs/actions/github/branch/merge@add-hatch-actions with: source-branch: ${{ needs.release-branch.outputs.branch }} target-branch: ${{ inputs.branch }} @@ -195,9 +185,7 @@ jobs: publish-pypi: name: "Publish to PyPI" - needs: - - release-inputs - - merge-changes + needs: merge-changes if: ${{ !failure() && !cancelled() }} environment: name: ${{ inputs.deploy-environment }} @@ -205,17 +193,15 @@ jobs: runs-on: ubuntu-latest steps: - name: "Publish to PyPI" - uses: .github/actions/publish-pypi + uses: dbt-labs/actions/pypi/release/create with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ inputs.archive-name }} + index-url: ${{ vars.PYPI_REPOSITORY_URL }} publish-pypi-internal: name: "Publish to internal PyPI" if: ${{ !failure() && !cancelled() }} - needs: - - release-inputs - - merge-changes + needs: merge-changes uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main with: version_number: "${{ inputs.version }}" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 314cc5345..00ae81917 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,14 +7,6 @@ on: description: "The branch/tag/commit to run integration tests on" type: string required: true - test-command: - description: "The command to run unit tests" - type: string - required: true - python-versions: - description: "The python versions to test against" - type: string - required: true permissions: read-all @@ -29,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(inputs.python-versions) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} steps: - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" uses: actions/checkout@v4 @@ -38,13 +30,13 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: python-version: ${{ matrix.python-version }} - name: "Run unit tests" shell: bash - run: ${{ inputs.test-command }} + run: hatch run unit-tests:all aggregate-results: name: "Collect unit tests results" @@ -55,9 +47,7 @@ jobs: - name: "[INFO] Unit tests - FAILED" if: failure() shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" - exit 1 + run: echo "::notice title=$TITLE::$MESSAGE" && exit 1 env: TITLE: "Unit tests - FAILED" MESSAGE: "Unit tests completed with failures!" @@ -65,8 +55,7 @@ jobs: - name: "[INFO] Unit tests - PASSED" if: ${{ !failure() }} # includes skipped shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Unit tests - PASSED" MESSAGE: "Unit tests completed successfully!"