feat(TPG >=5.3)!: add new features. update provider requirements (#98)

terraform-google-modules · Nov 16, 2023 · ea834f7 · ea834f7
1 parent e975c68
commit ea834f7
Show file tree

Hide file tree

Showing 27 changed files with 806 additions and 972 deletions.
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.5
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.17
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 

diff --git a/README.md b/README.md
@@ -6,6 +6,16 @@ The resources/services/activations/deletions that this module will create/trigge
 
 - Create a GCP Composer Environment
 
+## Compatibility
+
+This module is meant for use with Terraform 1.3+ and tested using Terraform 1.3+. If you find incompatibilities using Terraform >=1.3, please open an issue.
+
+## Version
+
+Current version is 4.0. Upgrade guides:
+
+- [3.X -> 4.0.](/docs/upgrading_to_v4.0.md)
+
 ## Usage
 Cloud Composer currently has two versions: V2 has greater flexibility in the Airflow core services
 (scheduler, webserver, worker) and has a more practical and scalable infrastructure. Therefore, we recommend prioritizing the

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
@@ -28,52 +28,6 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
 
-# ----- SUITE simple-composer-env-v1-local
-
-- id: init-simple-composer-env-v1
-  waitFor:
-    - prepare
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage init --verbose']
-- id: apply-simple-composer-env-v1
-  waitFor:
-    - init-simple-composer-env-v1
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage apply --verbose']
-- id: verify-simple-composer-env-v1
-  waitFor:
-    - apply-simple-composer-env-v1
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage verify --verbose']
-- id: destroy-simple-composer-env-v1
-  waitFor:
-    - verify-simple-composer-env-v1
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage destroy --verbose']
-
-  # ----- SUITE simple-composer-env-v2-local
-
-- id: init-simple-composer-env-v2
-  waitFor:
-    - prepare
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
-- id: apply-simple-composer-env-v2
-  waitFor:
-    - init-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
-- id: verify-simple-composer-env-v2
-  waitFor:
-    - apply-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
-- id: destroy-simple-composer-env-v2
-  waitFor:
-    - verify-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
-
 # ----- SUITE airflow-connection-local
 
 - id: converge airflow-connection-local
@@ -111,11 +65,34 @@ steps:
 #   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
 #   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-pool-local']
 
-  # ----- SUITE composer-v2-sharedvpc-prereq-local
+  # ----- SUITE simple-composer-env-v2-local
+
+- id: init-simple-composer-env-v2
+  waitFor:
+    - verify airflow-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
+- id: apply-simple-composer-env-v2
+  waitFor:
+    - init-simple-composer-env-v2
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
+- id: verify-simple-composer-env-v2
+  waitFor:
+    - apply-simple-composer-env-v2
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
+- id: destroy-simple-composer-env-v2
+  waitFor:
+    - verify-simple-composer-env-v2
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
+
+# ----- SUITE composer-v2-sharedvpc-prereq-local
 
 - id: init-composer-v2-sharedvpc-prereq
   waitFor:
-    - prepare
+    - destroy-simple-composer-env-v2
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose']
 - id: apply-composer-v2-sharedvpc-prereq
@@ -133,9 +110,32 @@ steps:
     - verify-composer-v2-sharedvpc-prereq
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose']
+
+# ----- SUITE simple-composer-env-v1-local
+
+- id: init-simple-composer-env-v1
+  waitFor:
+    - destroy-composer-v2-sharedvpc-prereq
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage init --verbose']
+- id: apply-simple-composer-env-v1
+  waitFor:
+    - init-simple-composer-env-v1
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage apply --verbose']
+- id: verify-simple-composer-env-v1
+  waitFor:
+    - apply-simple-composer-env-v1
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage verify --verbose']
+- id: destroy-simple-composer-env-v1
+  waitFor:
+    - verify-simple-composer-env-v1
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage destroy --verbose']
 tags:
 - 'ci'
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.17'
diff --git a/docs/upgrading_to_v4.0.md b/docs/upgrading_to_v4.0.md
@@ -0,0 +1,7 @@
+# Upgrading to v4.0.0
+
+The v4.0 release contains backwards-incompatible changes.
+
+This update requires upgrading the minimum provider version to `5.3`.
+
+### TPG max version is bumped to 6.x
diff --git a/examples/airflow_connection/main.tf b/examples/airflow_connection/main.tf
@@ -27,6 +27,7 @@ module "simple-composer-environment" {
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
   node_count                       = 3
   machine_type                     = "n1-standard-1"
+  image_version                    = "composer-1.20.12-airflow-1.10.15"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

diff --git a/examples/airflow_pool/main.tf b/examples/airflow_pool/main.tf
@@ -27,6 +27,7 @@ module "simple-composer-environment" {
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
   node_count                       = 3
   machine_type                     = "n1-standard-1"
+  image_version                    = "composer-1.20.12-airflow-1.10.15"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

diff --git a/examples/composer_v2_sharedvpc_prereq/composer.tf b/examples/composer_v2_sharedvpc_prereq/composer.tf
@@ -22,7 +22,6 @@ module "composer_net" {
   subnetwork                             = "composer-subnet"
   cloud_composer_network_ipv4_cidr_block = "192.168.192.0/24"
   master_ipv4_cidr                       = "192.168.193.0/28"
-  cloud_sql_ipv4_cidr                    = "192.168.0.0/17"
   gke_subnet_ip_range                    = ["10.100.232.0/27"]
   gke_pods_services_ip_ranges            = ["10.1.0.0/16", "10.4.0.0/16", "10.10.10.0/24", "10.10.14.0/24"]
 }

diff --git a/examples/simple_composer_env_v1/main.tf b/examples/simple_composer_env_v1/main.tf
@@ -27,4 +27,5 @@ module "simple-composer-environment" {
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
   node_count                       = 3
   machine_type                     = "n1-standard-1"
+  image_version                    = "composer-1.20.12-airflow-1.10.15"
 }
diff --git a/examples/simple_composer_env_v2/main.tf b/examples/simple_composer_env_v2/main.tf
@@ -14,6 +14,27 @@
  * limitations under the License.
  */
 
+resource "random_string" "key_suffix" {
+  length  = 5
+  special = false
+  upper   = false
+}
+
+# Create a bucket to store the snapshots
+resource "google_storage_bucket" "my_bucket" {
+  project                     = var.project_id
+  name                        = "snapshot-bucket-${random_string.key_suffix.result}"
+  location                    = var.region
+  force_destroy               = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_iam_member" "object_admin" {
+  bucket = google_storage_bucket.my_bucket.name
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${var.composer_service_account}"
+}
+
 module "simple-composer-environment" {
   source                           = "../../modules/create_environment_v2"
   project_id                       = var.project_id
@@ -26,11 +47,14 @@ module "simple-composer-environment" {
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
   grant_sa_agent_permission        = false
   environment_size                 = "ENVIRONMENT_SIZE_SMALL"
+  enable_private_endpoint          = true
+  use_private_environment          = true
+
   scheduler = {
     cpu        = 0.5
     memory_gb  = 1.875
     storage_gb = 1
-    count      = 1
+    count      = 2
   }
   web_server = {
     cpu        = 0.5
@@ -41,7 +65,24 @@ module "simple-composer-environment" {
     cpu        = 0.5
     memory_gb  = 1.875
     storage_gb = 1
-    min_count  = 1
+    min_count  = 2
     max_count  = 3
   }
+
+  scheduled_snapshots_config = {
+    enabled                    = true
+    snapshot_location          = google_storage_bucket.my_bucket.url
+    snapshot_creation_schedule = "0 4 * * *"
+    time_zone                  = "UTC+01"
+  }
+
+  maintenance_window = {
+    start_time = "2023-01-01T00:00:00Z"
+    end_time   = "2023-01-01T12:00:00Z"
+    recurrence = "FREQ=WEEKLY;BYDAY=SU,WE,SA"
+  }
+
+  depends_on = [
+    google_storage_bucket_iam_member.object_admin,
+  ]
 }
diff --git a/modules/airflow_storage/README.md b/modules/airflow_storage/README.md
@@ -24,14 +24,14 @@ See the examples for cloudsql support.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | destination\_path | The optional destination path | `string` | `null` | no |
-| environment | n/a | `string` | n/a | yes |
-| location | n/a | `string` | n/a | yes |
-| project\_id | n/a | `string` | n/a | yes |
+| environment | Environment | `string` | n/a | yes |
+| location | Location of the resource | `string` | n/a | yes |
+| project\_id | Project ID | `string` | n/a | yes |
 | source\_path | The source on the local file system | `string` | n/a | yes |
 | type | The type of resource to upload. Either dag, plugin or data | `string` | n/a | yes |
 
 ## Outputs
 
-No output.
+No outputs.
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/airflow_storage/variables.tf b/modules/airflow_storage/variables.tf
@@ -15,15 +15,18 @@
  */
 
 variable "project_id" {
-  type = string
+  type        = string
+  description = "Project ID"
 }
 
 variable "location" {
-  type = string
+  type        = string
+  description = "Location of the resource"
 }
 
 variable "environment" {
-  type = string
+  type        = string
+  description = "Environment"
 }
 
 variable "type" {

diff --git a/modules/airflow_storage/versions.tf b/modules/airflow_storage/versions.tf
@@ -20,7 +20,7 @@ terraform {
 
     google = {
       source  = "hashicorp/google"
-      version = ">= 3.53, < 5.0"
+      version = ">= 3.53, < 6"
     }
   }
 

diff --git a/modules/composer_net/README.md b/modules/composer_net/README.md
@@ -8,7 +8,6 @@ This example illustrates how to use the `composer-net` module. Please see exampl
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. | `string` | `null` | no |
-| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL. | `string` | `null` | no |
 | composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes |
 | gke\_pods\_services\_ip\_ranges | The secondary IP ranges for the GKE Pods and Services IP ranges | `list(string)` | n/a | yes |
 | gke\_subnet\_ip\_range | The GKE subnet IP range | `list(string)` | n/a | yes |
@@ -23,7 +22,7 @@ This example illustrates how to use the `composer-net` module. Please see exampl
 
 | Name | Description |
 |------|-------------|
-| composer\_sa\_email | n/a |
+| composer\_sa\_email | composer service account email |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

diff --git a/modules/composer_net/outputs.tf b/modules/composer_net/outputs.tf
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 output "composer_sa_email" {
-  value = google_service_account.composer_sa.email
+  description = "composer service account email"
+  value       = google_service_account.composer_sa.email
 }
 
diff --git a/modules/composer_net/variables.tf b/modules/composer_net/variables.tf
@@ -42,13 +42,6 @@ variable "subnetwork" {
   description = "The subnetwork to host the composer cluster."
 }
 
-
-variable "cloud_sql_ipv4_cidr" {
-  description = "The CIDR block from which IP range in tenant project will be reserved for Cloud SQL."
-  type        = string
-  default     = null
-}
-
 variable "master_ipv4_cidr" {
   description = "The CIDR block from which IP range in tenant project will be reserved for the master."
   type        = string

diff --git a/modules/composer_net/versions.tf b/modules/composer_net/versions.tf
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 3.53, < 6"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 3.53, < 6"
+    }
+  }
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-composer:composer_net/v3.4.0"
+  }
+}