Merge branch 'main' of https://github.com/dbt-labs/dbt-snowflake into iceberg_poc

Author: dataders

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.8.0b1
+current_version = 1.8.0b2
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.8.0-b2.md

@@ -0,0 +1,25 @@
+## dbt-snowflake 1.8.0-b2 - April 03, 2024
+
+### Features
+
+-  Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-snowflake/issues/38))
+
+### Fixes
+
+- Acknowledge `send_anonymous_usage_stats` setting for python models ([#830](https://github.com/dbt-labs/dbt-snowflake/issues/830))
+- modify metadata queries to take into account object type of BASE TABLE being assigned to dynamic tables when was null previously ([#934](https://github.com/dbt-labs/dbt-snowflake/issues/934))
+- remove `private_key_path` from connection keys ([#949](https://github.com/dbt-labs/dbt-snowflake/issues/949))
+
+### Under the Hood
+
+- Add unit test for transaction semantics. ([#912](https://github.com/dbt-labs/dbt-snowflake/issues/912))
+
+### Dependencies
+
+- hard pin ddtrace to 2.3.0 ([#932](https://github.com/dbt-labs/dbt-snowflake/pull/932))
+- bump cryptography to 42.0.4 or higher for security callouts ([#938](https://github.com/dbt-labs/dbt-snowflake/pull/938))
+- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#964](https://github.com/dbt-labs/dbt-snowflake/pull/964))
+
+### Security
+
+- Pin `black>=24.3` in `dev-requirements.txt` ([#959](https://github.com/dbt-labs/dbt-snowflake/pull/959))

.changes/unreleased/Dependencies-20240319-102636.yaml renamed to .changes/1.8.0/Dependencies-20240319-102636.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Add `dbt-core` as a dependency to preserve backwards compatibility for installation
+time: 2024-04-03T13:39:36.783478-04:00
+custom:
+  Author: mikealfare
+  PR: "964"

.changes/unreleased/Dependencies-20240320-111128.yaml renamed to .changes/1.8.0/Dependencies-20240320-111128.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add external_access_integration and secret to python models
+time: 2024-04-02T13:13:30.952425+01:00
+custom:
+  Author: Lindblomsebastian
+  Issue: "955"

.changes/1.8.0/Dependencies-20240403-133936.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support TableLastModifiedMetadataBatch capability
+time: 2024-04-04T17:17:04.853047-07:00
+custom:
+  Author: michelleark
+  Issue: "965"

.changes/unreleased/Features-20240318-033136.yaml renamed to .changes/1.8.0/Features-20240318-033136.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Update dependabot to cover GHA
+time: 2024-04-10T18:16:35.621793-04:00
+custom:
+  Author: mikealfare
+  Issue: "967"

.changes/unreleased/Fixes-20240318-203635.yaml renamed to .changes/1.8.0/Fixes-20240318-203635.yaml

@@ -1,8 +1,17 @@
 version: 2
 updates:
-  # python dependencies
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
       interval: "daily"
     rebase-strategy: "disabled"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    rebase-strategy: "disabled"
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "weekly"
+    rebase-strategy: "disabled"

.changes/unreleased/Fixes-20240320-154054.yaml renamed to .changes/1.8.0/Fixes-20240320-154054.yaml

@@ -13,7 +13,8 @@
 #
 # **when?**
 # This workflow can be run manually on demand or can be called by other workflows
-name: Release to GitHub and PyPI
+name: "Release to GitHub, PyPI, and Docker"
+run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker"
 
 on:
   workflow_dispatch:
@@ -131,27 +132,21 @@ jobs:
 
   bump-version-generate-changelog:
     name: Bump package version, Generate changelog
-
     uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
-
     with:
       sha: ${{ inputs.sha }}
       version_number: ${{ inputs.version_number }}
       target_branch: ${{ inputs.target_branch }}
       env_setup_script_path: ${{ inputs.env_setup_script_path }}
       test_run: ${{ inputs.test_run }}
       nightly_release: ${{ inputs.nightly_release }}
-
     secrets: inherit
 
   log-outputs-bump-version-generate-changelog:
     name: "[Log output] Bump package version, Generate changelog"
     if: ${{ !failure() && !cancelled() }}
-
     needs: [bump-version-generate-changelog]
-
     runs-on: ubuntu-latest
-
     steps:
       - name: Print variables
         run: |
@@ -162,9 +157,7 @@ jobs:
     name: Build, Test, Package
     if: ${{ !failure() && !cancelled() }}
     needs: [bump-version-generate-changelog]
-
     uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
-
     with:
       sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -174,19 +167,15 @@ jobs:
       package_test_command: ${{ inputs.package_test_command }}
       test_run: ${{ inputs.test_run }}
       nightly_release: ${{ inputs.nightly_release }}
-
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   github-release:
     name: GitHub Release
     if: ${{ !failure() && !cancelled() }}
-
     needs: [bump-version-generate-changelog, build-test-package]
-
     uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
-
     with:
       sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -195,34 +184,38 @@ jobs:
 
   pypi-release:
     name: PyPI Release
-
-    needs: [github-release]
-
+    if: ${{ !failure() && !cancelled() }}
+    needs: [bump-version-generate-changelog, build-test-package]
     uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
-
     with:
       version_number: ${{ inputs.version_number }}
       test_run: ${{ inputs.test_run }}
-
     secrets:
       PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
       TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
 
+  docker-release:
+    name: "Docker Release"
+    if: ${{ !failure() && !cancelled() }}
+    needs: [bump-version-generate-changelog, build-test-package, github-release]
+    permissions:
+      packages: write
+    uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main
+    with:
+      version_number: ${{ inputs.version_number }}
+      test_run: ${{ inputs.test_run }}
+
   slack-notification:
     name: Slack Notification
     if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
-
     needs:
       [
-        bump-version-generate-changelog,
-        build-test-package,
         github-release,
         pypi-release,
+        docker-release,
       ]
-
     uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
     with:
       status: "failure"
-
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}

.changes/unreleased/Fixes-20240322-104553.yaml renamed to .changes/1.8.0/Fixes-20240322-104553.yaml

@@ -5,6 +5,34 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-snowflake 1.8.0-b2 - April 03, 2024
+
+### Features
+
+-  Add new workflow for internal patch releases ([#38](https://github.com/dbt-labs/dbt-snowflake/issues/38))
+
+### Fixes
+
+- Acknowledge `send_anonymous_usage_stats` setting for python models ([#830](https://github.com/dbt-labs/dbt-snowflake/issues/830))
+- modify metadata queries to take into account object type of BASE TABLE being assigned to dynamic tables when was null previously ([#934](https://github.com/dbt-labs/dbt-snowflake/issues/934))
+- remove `private_key_path` from connection keys ([#949](https://github.com/dbt-labs/dbt-snowflake/issues/949))
+
+### Under the Hood
+
+- Add unit test for transaction semantics. ([#912](https://github.com/dbt-labs/dbt-snowflake/issues/912))
+
+### Dependencies
+
+- hard pin ddtrace to 2.3.0 ([#932](https://github.com/dbt-labs/dbt-snowflake/pull/932))
+- bump cryptography to 42.0.4 or higher for security callouts ([#938](https://github.com/dbt-labs/dbt-snowflake/pull/938))
+- Add `dbt-core` as a dependency to preserve backwards compatibility for installation ([#964](https://github.com/dbt-labs/dbt-snowflake/pull/964))
+
+### Security
+
+- Pin `black>=24.3` in `dev-requirements.txt` ([#959](https://github.com/dbt-labs/dbt-snowflake/pull/959))
+
+
+
 ## dbt-snowflake 1.8.0-b1 - March 02, 2024
 
 ### Features
@@ -46,8 +74,6 @@
 
 - DDOS vulnerability in 41.0.5. Uptick to avoid ([#852](https://github.com/dbt-labs/dbt-snowflake/pull/852))
 
-
-
 ## Previous Releases
 For information on prior major and minor releases, see their changelogs:
 - [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md)

.changes/unreleased/Security-20240327-190647.yaml renamed to .changes/1.8.0/Security-20240327-190647.yaml

@@ -66,32 +66,11 @@ help: ## Show this help message.
 	@echo 'targets:'
 	@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: ubuntu-py38
-ubuntu-py38:
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py38 . --build-arg version=3.8
-	docker run --rm -it --name dbt-snowflake-ubuntu-py38 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py38
-
-.PHONY: ubuntu-py39
-ubuntu-py39:
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py39 . --build-arg version=3.9
-	docker run --rm -it --name dbt-snowflake-ubuntu-py39 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py39
-
-.PHONY: ubuntu-py310
-ubuntu-py310:
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py310 . --build-arg version=3.10
-	docker run --rm -it --name dbt-snowflake-ubuntu-py310 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py310
-
-.PHONY: ubuntu-py311
-ubuntu-py311:
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py311 . --build-arg version=3.11
-	docker run --rm -it --name dbt-snowflake-ubuntu-py311 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py311
-
-.PHONY: debian-py38
-debian-py38:
-	docker build -f docker/debian-py38.Dockerfile -t dbt-snowflake-debian-py38 . --build-arg version=3.8.15
-	docker run --rm -it --name dbt-snowflake-debian-py38 -v $(shell pwd):/opt/code dbt-snowflake-debian-py38
-
-.PHONY: dev-env-default
-dev-env-default:
-	docker build -f docker/dev-env-default.Dockerfile -t dbt-snowflake-dev-env-default .
-	docker run --rm -it --name dbt-snowflake-dev-env-default -v $(shell pwd):/opt/code dbt-snowflake-dev-env-default
+.PHONY: docker-dev
+docker-dev:
+	docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .
+	docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev
+
+.PHONY: docker-prod
+docker-prod:
+	docker build -f docker/Dockerfile -t dbt-snowflake .

.changes/unreleased/Under the Hood-20240227-010428.yaml renamed to .changes/1.8.0/Under the Hood-20240227-010428.yaml

@@ -1 +1 @@
-version = "1.8.0b1"
+version = "1.8.0b2"

.changes/unreleased/Features-20240402-131330.yaml

@@ -53,6 +53,7 @@ class SnowflakeAdapter(SQLAdapter):
         {
             Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full),
             Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full),
+            Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full),
         }
     )
 
@@ -205,6 +206,10 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str):
 
         packages = parsed_model["config"].get("packages", [])
         imports = parsed_model["config"].get("imports", [])
+        external_access_integrations = parsed_model["config"].get(
+            "external_access_integrations", []
+        )
+        secrets = parsed_model["config"].get("secrets", {})
         # adding default packages we need to make python model work
         default_packages = ["snowflake-snowpark-python"]
         package_names = [package.split("==")[0] for package in packages]
@@ -213,9 +218,18 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str):
                 packages.append(default_package)
         packages = "', '".join(packages)
         imports = "', '".join(imports)
-        # we can't pass empty imports clause to snowflake
+        external_access_integrations = ", ".join(external_access_integrations)
+        secrets = ", ".join(f"'{key}' = {value}" for key, value in secrets.items())
+
+        # we can't pass empty imports, external_access_integrations or secrets clause to snowflake
         if imports:
             imports = f"IMPORTS = ('{imports}')"
+        if external_access_integrations:
+            # Black is trying to make this a tuple.
+            # fmt: off
+            external_access_integrations = f"EXTERNAL_ACCESS_INTEGRATIONS = ({external_access_integrations})"
+        if secrets:
+            secrets = f"SECRETS = ({secrets})"
 
         if self.config.args.SEND_ANONYMOUS_USAGE_STATS:
             snowpark_telemetry_string = "dbtLabs_dbtPython"
@@ -230,6 +244,8 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str):
 LANGUAGE PYTHON
 RUNTIME_VERSION = '{python_version}'
 PACKAGES = ('{packages}')
+{external_access_integrations}
+{secrets}
 {imports}
 HANDLER = 'main'
 EXECUTE AS CALLER

.changes/unreleased/Features-20240404-171704.yaml

@@ -0,0 +1,37 @@
+# this image gets published to GHCR for production use
+ARG py_version=3.10.7
+
+FROM python:$py_version-slim-bullseye as base
+
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    build-essential=12.9 \
+    ca-certificates=20210119 \
+    git=1:2.30.2-1+deb11u2 \
+    libpq-dev=13.14-0+deb11u1 \
+    make=4.3-4.1 \
+    openssh-client=1:8.4p1-5+deb11u3 \
+    software-properties-common=0.96.20.2-2.1 \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+ENV PYTHONIOENCODING=utf-8
+ENV LANG=C.UTF-8
+
+RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir
+
+
+FROM base as dbt-snowflake
+
+ARG commit_ref=main
+
+HEALTHCHECK CMD dbt --version || exit 1
+
+WORKDIR /usr/app/dbt/
+ENTRYPOINT ["dbt"]
+
+RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/dbt-snowflake@${commit_ref}"