feat!: Cloud composer v2 (#36)

* refactor: separate packages by Airflow version

* feature: new cloud composer (v2) version

* feature: license

* feature: add roles/composer.ServiceAgentV2Ext to composer api agent

* fix: rename simple composer env dir

* fix: cloud build kitchen-tf test integration

* fix: cloud build kitchen-tf test integration

* fix: cloud build kitchen-tf test integration

* refactor: license year + docs

* fix: lint

* fix: master_authorized_networks_config flag

* update

README.md

@@ -7,8 +7,14 @@ The resources/services/activations/deletions that this module will create/trigge
 - Create a GCP Composer Environment
 
 ## Usage
+Cloud Composer currently has two versions: V2 has greater flexibility in the Airflow core services
+(scheduler, webserver, worker) and has a more practical and scalable infrastructure. Therefore, we recommend prioritizing the
+use of V2 for new environments.
 
-Basic usage of this module is as follows:
+You can find an overview of the product [here](https://cloud.google.com/composer/docs/composer-2/composer-overview)
+and the [list of major differences](https://cloud.google.com/composer/docs/concepts/versioning/composer-versioning-overview).
+
+Simple usage is as follows:
 
 ```hcl
 module "composer" {

build/int.cloudbuild.yaml

@@ -28,23 +28,41 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
 
-# ----- SUITE simple-composer-env-local
+# ----- SUITE simple-composer-env-v1-local
 
-- id: converge simple-composer-env-local
+- id: converge simple-composer-env-v1-local
   waitFor:
     - create all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-local']
-- id: verify simple-composer-env-local
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v1-local']
+- id: verify simple-composer-env-v1-local
   waitFor:
-    - converge simple-composer-env-local
+    - converge simple-composer-env-v1-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-local']
-- id: destroy simple-composer-env-local
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v1-local']
+- id: destroy simple-composer-env-v1-local
   waitFor:
-    - verify simple-composer-env-local
+    - verify simple-composer-env-v1-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-local']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v1-local']
+
+  # ----- SUITE simple-composer-env-v2-local
+
+- id: converge simple-composer-env-v2-local
+  waitFor:
+    - create all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v2-local']
+- id: verify simple-composer-env-v2-local
+  waitFor:
+    - converge simple-composer-env-v2-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v2-local']
+- id: destroy simple-composer-env-v2-local
+  waitFor:
+    - verify simple-composer-env-v2-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v2-local']
 
 # ----- SUITE airflow-connection-local
 

examples/airflow_connection/main.tf

@@ -30,7 +30,7 @@ provider "external" {
 }
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -40,6 +40,8 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

examples/airflow_pool/main.tf

@@ -30,7 +30,7 @@ provider "external" {
 }
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -40,6 +40,8 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

examples/simple_composer_env/README.md → examples/simple_composer_env_v1/README.md

@@ -1,4 +1,4 @@
-# Simple Cloud Composer Environment Example
+# Simple Cloud Composer Environment (V1) Example
 
 This example illustrates how to use the `composer` module.
 

examples/simple_composer_env/main.tf → examples/simple_composer_env_v1/main.tf

@@ -15,7 +15,7 @@
  */
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -25,4 +25,6 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }

examples/simple_composer_env_v2/README.md

@@ -0,0 +1,35 @@
+# Simple Cloud Composer Environment (V2) Example
+
+This example illustrates how to use the `composer` module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no |
+| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
+| network | Network where Cloud Composer is created. | `string` | n/a | yes |
+| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | n/a | yes |
+| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
+| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes |
+| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | n/a | yes |
+| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
+| composer\_env\_id | ID of Cloud Composer Environment. |
+| composer\_env\_name | Name of the Cloud Composer Environment. |
+| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
+| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/simple_composer_env_v2/main.tf

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "simple-composer-environment" {
+  source                           = "../../modules/create_environment_v2"
+  project_id                       = var.project_id
+  composer_env_name                = var.composer_env_name
+  region                           = var.region
+  composer_service_account         = var.composer_service_account
+  network                          = var.network
+  subnetwork                       = var.subnetwork
+  pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
+  service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  grant_sa_agent_permission        = false
+  environment_size                 = "ENVIRONMENT_SIZE_SMALL"
+  scheduler = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+    count      = 1
+  }
+  web_server = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+  }
+  worker = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+    min_count  = 1
+    max_count  = 3
+  }
+}

examples/simple_composer_env_v2/outputs.tf

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "composer_env_name" {
+  description = "Name of the Cloud Composer Environment."
+  value       = module.simple-composer-environment.composer_env_name
+}
+
+output "composer_env_id" {
+  description = "ID of Cloud Composer Environment."
+  value       = module.simple-composer-environment.composer_env_id
+}
+
+output "gke_cluster" {
+  description = "Google Kubernetes Engine cluster used to run the Cloud Composer Environment."
+  value       = module.simple-composer-environment.gke_cluster
+}
+
+output "gcs_bucket" {
+  description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment."
+  value       = module.simple-composer-environment.gcs_bucket
+}
+
+output "airflow_uri" {
+  description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
+  value       = module.simple-composer-environment.airflow_uri
+}

examples/simple_composer_env_v2/variables.tf

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "Project ID where Cloud Composer Environment is created."
+  type        = string
+}
+
+variable "composer_env_name" {
+  description = "Name of Cloud Composer Environment."
+  default     = "ci-composer"
+  type        = string
+}
+
+variable "region" {
+  description = "Region where Cloud Composer Environment is created."
+  type        = string
+}
+
+variable "composer_service_account" {
+  description = "Service Account to be used for running Cloud Composer Environment."
+  type        = string
+}
+
+variable "network" {
+  description = "Network where Cloud Composer is created."
+  type        = string
+}
+
+variable "subnetwork" {
+  description = "Subetwork where Cloud Composer is created."
+  type        = string
+}
+
+variable "pod_ip_allocation_range_name" {
+  description = "The name of the cluster's secondary range used to allocate IP addresses to pods."
+  type        = string
+}
+
+variable "service_ip_allocation_range_name" {
+  type        = string
+  description = "The name of the services' secondary range used to allocate IP addresses to the cluster."
+}

examples/simple_composer_env_v2/versions.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.8.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 4.8.0"
+    }
+  }
+}

kitchen.yml

@@ -20,15 +20,27 @@ platforms:
   - name: local
 
 suites:
-  - name: simple-composer-env
+  - name: simple-composer-env-v1
     driver:
       name: terraform
       command_timeout: 3600
-      root_module_directory: test/fixtures/simple-composer-env
+      root_module_directory: test/fixtures/simple-composer-env-v1
     verifier:
       name: terraform
       systems:
-        - name: simple-composer-env
+        - name: simple-composer-env-v1
+          backend: local
+    provisioner:
+      name: terraform
+  - name: simple-composer-env-v2
+    driver:
+      name: terraform
+      command_timeout: 3600
+      root_module_directory: test/fixtures/simple-composer-env-v2
+    verifier:
+      name: terraform
+      systems:
+        - name: simple-composer-env-v2
           backend: local
     provisioner:
       name: terraform

main.tf

@@ -15,7 +15,7 @@
  */
 
 module "composer-environment" {
-  source = "./modules/create_environment"
+  source = "./modules/create_environment_v1"
 
   project_id        = var.project_id
   composer_env_name = var.composer_env_name

modules/create_environment/README.md → modules/create_environment_v1/README.md

@@ -1,10 +1,10 @@
-# Module Cloud Composer Environment
+# Module Cloud Composer Environment ([V1](https://cloud.google.com/composer/docs/concepts/overview))
 
 This optional module is used to create a Cloud Composer environment.
 
 ```hcl
 module "composer" {
-  source = "terraform-google-modules/composer/google//modules/create_environment"
+  source = "terraform-google-modules/composer/google//modules/create_environment_v1"
 
   project = "project-123"
   name    = "Composer-Prod-Env"

modules/create_environment/versions.tf → modules/create_environment_v1/versions.tf

@@ -30,7 +30,7 @@ terraform {
   }
 
   provider_meta "google" {
-    module_name = "blueprints/terraform/terraform-google-composer:create_environment/v2.4.0"
+    module_name = "blueprints/terraform/terraform-google-composer:create_environment_v1/v2.4.0"
   }
 
 }