diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml new file mode 100644 index 000000000..9b82b15af --- /dev/null +++ b/.github/workflows/terraform-apply.yml @@ -0,0 +1,31 @@ +name: "Terraform apply" + +on: + workflow_dispatch: + +defaults: + run: + shell: bash + working-directory: ./.terraform + +jobs: + apply: + name: "Apply" + runs-on: ubuntu-latest + environment: "Snowflake: KTB38830" + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + ref: main + + - name: "Setup Terraform" + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ vars.TERRAFORM_VERSION }} + + - name: "Terraform init" + run: terraform init + + - name: "Terraform apply" + run: terraform apply -auto-approve diff --git a/.github/workflows/terraform-code-quality.yml b/.github/workflows/terraform-code-quality.yml new file mode 100644 index 000000000..9cde6ec82 --- /dev/null +++ b/.github/workflows/terraform-code-quality.yml @@ -0,0 +1,34 @@ +name: "Terraform code quality" + +on: + pull_request: + branches: + - main + +defaults: + run: + shell: bash + working-directory: ./.terraform + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + environment: "Snowflake: KTB38830" + steps: + - name: "Checkout" + uses: actions/checkout@v4 + + - name: "Setup Terraform" + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ vars.TERRAFORM_VERSION }} + + - name: "Terraform init" + run: terraform init + + - name: "Terraform format" + run: terraform fmt -check + + - name: "Terraform validate" + run: terraform validate diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml new file mode 100644 index 000000000..7b74a59c1 --- /dev/null +++ b/.github/workflows/terraform-plan.yml @@ -0,0 +1,36 @@ +name: "Terraform plan" + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to run plan on" + type: string + default: main + +defaults: + run: + shell: bash + working-directory: ./.terraform + +jobs: + plan: + name: "Plan" + runs-on: ubuntu-latest + environment: "Snowflake: KTB38830" + steps: + - name: "Checkout `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup Terraform" + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ vars.TERRAFORM_VERSION }} + + - name: "Terraform init" + run: terraform init + + - name: "Terraform plan" + run: terraform plan diff --git a/.gitignore b/.gitignore index 780d98f70..6ce7f70c3 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,27 @@ venv/ # vscode .vscode/ + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Terraform lock files +*.terraform.lock.hcl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6b26b559..a28c60a87 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,3 +55,11 @@ repos: additional_dependencies: - types-pytz - types-requests + +- repo: local + hooks: + - id: terraform-lint + name: terraform-lint + entry: bash -c 'cd infra && terraform fmt && terraform validate' + language: system + pass_filenames: false diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 000000000..3a2f8f5cf --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,19 @@ +terraform { + required_version = "1.8.3" + + required_providers { + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "0.91.0" + } + } +} + +provider "snowflake" { + alias = "security_admin" + role = "SECURITYADMIN" + # SNOWFLAKE_ACCOUNT + # SNOWFLAKE_USER + # SNOWFLAKE_AUTHENTICATOR + # SNOWFLAKE_PRIVATE_KEY +} diff --git a/infra/snowflake.tf b/infra/snowflake.tf new file mode 100644 index 000000000..669298e0a --- /dev/null +++ b/infra/snowflake.tf @@ -0,0 +1,73 @@ +# Resources needed to run dbt-snowflake + +resource "snowflake_database" "dbt_snowflake_db" { + name = "DBT_SNOWFLAKE_DB" + data_retention_time_in_days = 0 + comment = "Used by `dbt-snowflake` for CI" +} + +resource "snowflake_warehouse" "dbt_snowflake_wh" { + name = "DBT_SNOWFLAKE_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + comment = "Used by `dbt-snowflake` for CI" +} + +resource "snowflake_role" "dbt_snowflake_role" { + provider = snowflake.security_admin + name = "DBT_SNOWFLAKE_ROLE" + comment = "Application role for `dbt_snowflake`" +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_snowflake_db" { + provider = snowflake.security_admin + privileges = ["USAGE", "MODIFY", "CREATE SCHEMA"] + account_role_name = snowflake_role.dbt_snowflake_role.name + + on_account_object { + object_type = "DATABASE" + object_name = snowflake_database.dbt_snowflake_db.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_snowflake_wh" { + provider = snowflake.security_admin + privileges = ["USAGE"] + account_role_name = snowflake_role.dbt_snowflake_role.name + + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.dbt_snowflake_wh.name + } +} + +resource "snowflake_user" "dbt_snowflake" { + provider = snowflake.security_admin + name = "DBT_SNOWFLAKE" + display_name = "dbt-snowflake" + default_warehouse = snowflake_warehouse.dbt_snowflake_wh.name + default_role = snowflake_role.dbt_snowflake_role.name + default_namespace = snowflake_database.dbt_snowflake_db.name + comment = "Application user for `dbt_snowflake`" +} + +resource "snowflake_grant_account_role" "dbt_snowflake" { + provider = snowflake.security_admin + role_name = snowflake_role.dbt_snowflake_role.name + user_name = snowflake_user.dbt_snowflake.name +} + +# Additional resources required for integration tests + +resource "snowflake_database" "dbt_snowflake_db_alt" { + name = "DBT_SNOWFLAKE_DB_ALT" + data_retention_time_in_days = 0 + comment = "Used by `dbt-snowflake` for CI" +} + +resource "snowflake_warehouse" "dbt_snowflake_wh_alt" { + name = "DBT_SNOWFLAKE_WH_ALT" + warehouse_size = "XSMALL" + auto_suspend = 60 + comment = "Used by `dbt-snowflake` for CI" +} diff --git a/test.env.example b/test.env.example index bdf5d68e1..092a159f2 100644 --- a/test.env.example +++ b/test.env.example @@ -6,30 +6,28 @@ # These will all be gathered from account information or created by you. # SNOWFLAKE_TEST_ACCOUNT: The name that uniquely identifies your Snowflake account. -# SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database. -# SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing. # SNOWFLAKE_TEST_DATABASE: Name of the primary database to use for testing. +# SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary. +# SNOWFLAKE_TEST_USER: Username of database user +# SNOWFLAKE_TEST_PASSWORD:Password used for your database user. + # SNOWFLAKE_TEST_OAUTH_CLIENT_ID: Client ID of the OAuth client integration. (only for oauth authentication) # SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: Client secret of your OAuth client id. (only for oauth authentication) # SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: Boolean value defaulted to True keep connection alive. (only for oauth authentication) -# SNOWFLAKE_TEST_PASSWORD:Password used for your database user. -# SNOWFLAKE_TEST_QUOTED_DATABASE: Name of database to be used from warehouse. -# SNOWFLAKE_TEST_USER: Username of database user -# SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary. + +# SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database. +# SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing. # Copy the following to a test.env, and replace example values with your information. -SNOWFLAKE_TEST_ACCOUNT=my_account_id -SNOWFLAKE_TEST_ALT_DATABASE=my_alt_database_name -SNOWFLAKE_TEST_ALT_WAREHOUSE=my_alt_warehouse_name -SNOWFLAKE_TEST_DATABASE=my_database_name +SNOWFLAKE_TEST_ACCOUNT=my_account +SNOWFLAKE_TEST_DATABASE=DBT_SNOWFLAKE_DB +SNOWFLAKE_TEST_WAREHOUSE=DBT_SNOWFLAKE_WH +SNOWFLAKE_TEST_USER=DBT_SNOWFLAKE +SNOWFLAKE_TEST_PASSWORD=my_password + SNOWFLAKE_TEST_OAUTH_CLIENT_ID=my_oauth_id SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=my_oauth_secret SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=TRUE -SNOWFLAKE_TEST_PASSWORD=my_password -SNOWFLAKE_TEST_QUOTED_DATABASE=my_quoted_database_name -SNOWFLAKE_TEST_USER=my_username -SNOWFLAKE_TEST_WAREHOUSE=my_warehouse_name -DBT_TEST_USER_1=dbt_test_role_1 -DBT_TEST_USER_2=dbt_test_role_2 -DBT_TEST_USER_3=dbt_test_role_3 +SNOWFLAKE_TEST_ALT_DATABASE=DBT_SNOWFLAKE_DB_ALT +SNOWFLAKE_TEST_ALT_WAREHOUSE=DBT_SNOWFLAKE_WH_ALT