From 959db7d36dca43fa74866d79a91f1d5d1121fa16 Mon Sep 17 00:00:00 2001 From: Diego Lima Date: Tue, 29 Mar 2022 01:26:04 -0300 Subject: [PATCH] feat!: Cloud composer v2 (#36) * refactor: separate packages by Airflow version * feature: new cloud composer (v2) version * feature: license * feature: add roles/composer.ServiceAgentV2Ext to composer api agent * fix: rename simple composer env dir * fix: cloud build kitchen-tf test integration * fix: cloud build kitchen-tf test integration * fix: cloud build kitchen-tf test integration * refactor: license year + docs * fix: lint * fix: master_authorized_networks_config flag * update --- README.md | 8 +- build/int.cloudbuild.yaml | 36 ++- examples/airflow_connection/main.tf | 4 +- examples/airflow_pool/main.tf | 4 +- .../README.md | 2 +- .../main.tf | 4 +- .../outputs.tf | 0 .../variables.tf | 0 .../versions.tf | 0 examples/simple_composer_env_v2/README.md | 35 +++ examples/simple_composer_env_v2/main.tf | 47 ++++ examples/simple_composer_env_v2/outputs.tf | 40 +++ examples/simple_composer_env_v2/variables.tf | 56 +++++ examples/simple_composer_env_v2/versions.tf | 29 +++ kitchen.yml | 18 +- main.tf | 2 +- .../README.md | 4 +- .../main.tf | 0 .../outputs.tf | 0 .../variables.tf | 0 .../versions.tf | 2 +- modules/create_environment_v2/README.md | 61 +++++ modules/create_environment_v2/iam.tf | 26 ++ modules/create_environment_v2/main.tf | 151 ++++++++++++ modules/create_environment_v2/outputs.tf | 40 +++ modules/create_environment_v2/variables.tf | 233 ++++++++++++++++++ modules/create_environment_v2/versions.tf | 40 +++ .../main.tf | 2 +- .../network.tf | 0 .../outputs.tf | 0 .../variables.tf | 0 test/fixtures/simple-composer-env-v2/main.tf | 32 +++ .../simple-composer-env-v2/network.tf | 46 ++++ .../simple-composer-env-v2/outputs.tf | 65 +++++ .../simple-composer-env-v2/variables.tf | 31 +++ .../controls/simple-composer.rb | 0 .../inspec.yml | 0 .../controls/simple-composer.rb | 49 ++++ .../simple-composer-env-v2/inspec.yml | 34 +++ test/setup/iam.tf | 6 + 40 files changed, 1085 insertions(+), 22 deletions(-) rename examples/{simple_composer_env => simple_composer_env_v1}/README.md (97%) rename examples/{simple_composer_env => simple_composer_env_v1}/main.tf (92%) rename examples/{simple_composer_env => simple_composer_env_v1}/outputs.tf (100%) rename examples/{simple_composer_env => simple_composer_env_v1}/variables.tf (100%) rename examples/{simple_composer_env => simple_composer_env_v1}/versions.tf (100%) create mode 100644 examples/simple_composer_env_v2/README.md create mode 100644 examples/simple_composer_env_v2/main.tf create mode 100644 examples/simple_composer_env_v2/outputs.tf create mode 100644 examples/simple_composer_env_v2/variables.tf create mode 100644 examples/simple_composer_env_v2/versions.tf rename modules/{create_environment => create_environment_v1}/README.md (97%) rename modules/{create_environment => create_environment_v1}/main.tf (100%) rename modules/{create_environment => create_environment_v1}/outputs.tf (100%) rename modules/{create_environment => create_environment_v1}/variables.tf (100%) rename modules/{create_environment => create_environment_v1}/versions.tf (96%) create mode 100644 modules/create_environment_v2/README.md create mode 100644 modules/create_environment_v2/iam.tf create mode 100644 modules/create_environment_v2/main.tf create mode 100644 modules/create_environment_v2/outputs.tf create mode 100644 modules/create_environment_v2/variables.tf create mode 100644 modules/create_environment_v2/versions.tf rename test/fixtures/{simple-composer-env => simple-composer-env-v1}/main.tf (95%) rename test/fixtures/{simple-composer-env => simple-composer-env-v1}/network.tf (100%) rename test/fixtures/{simple-composer-env => simple-composer-env-v1}/outputs.tf (100%) rename test/fixtures/{simple-composer-env => simple-composer-env-v1}/variables.tf (100%) create mode 100644 test/fixtures/simple-composer-env-v2/main.tf create mode 100644 test/fixtures/simple-composer-env-v2/network.tf create mode 100644 test/fixtures/simple-composer-env-v2/outputs.tf create mode 100644 test/fixtures/simple-composer-env-v2/variables.tf rename test/integration/{simple-composer-env => simple-composer-env-v1}/controls/simple-composer.rb (100%) rename test/integration/{simple-composer-env => simple-composer-env-v1}/inspec.yml (100%) create mode 100644 test/integration/simple-composer-env-v2/controls/simple-composer.rb create mode 100644 test/integration/simple-composer-env-v2/inspec.yml diff --git a/README.md b/README.md index d60eb54d..3d45fe58 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,14 @@ The resources/services/activations/deletions that this module will create/trigge - Create a GCP Composer Environment ## Usage +Cloud Composer currently has two versions: V2 has greater flexibility in the Airflow core services +(scheduler, webserver, worker) and has a more practical and scalable infrastructure. Therefore, we recommend prioritizing the +use of V2 for new environments. -Basic usage of this module is as follows: +You can find an overview of the product [here](https://cloud.google.com/composer/docs/composer-2/composer-overview) +and the [list of major differences](https://cloud.google.com/composer/docs/concepts/versioning/composer-versioning-overview). + +Simple usage is as follows: ```hcl module "composer" { diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index bc5345c5..1ecacbe1 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -28,23 +28,41 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] -# ----- SUITE simple-composer-env-local +# ----- SUITE simple-composer-env-v1-local -- id: converge simple-composer-env-local +- id: converge simple-composer-env-v1-local waitFor: - create all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-local'] -- id: verify simple-composer-env-local + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v1-local'] +- id: verify simple-composer-env-v1-local waitFor: - - converge simple-composer-env-local + - converge simple-composer-env-v1-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-local'] -- id: destroy simple-composer-env-local + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v1-local'] +- id: destroy simple-composer-env-v1-local waitFor: - - verify simple-composer-env-local + - verify simple-composer-env-v1-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-local'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v1-local'] + + # ----- SUITE simple-composer-env-v2-local + +- id: converge simple-composer-env-v2-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v2-local'] +- id: verify simple-composer-env-v2-local + waitFor: + - converge simple-composer-env-v2-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v2-local'] +- id: destroy simple-composer-env-v2-local + waitFor: + - verify simple-composer-env-v2-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v2-local'] # ----- SUITE airflow-connection-local diff --git a/examples/airflow_connection/main.tf b/examples/airflow_connection/main.tf index 12870577..048c5dc4 100644 --- a/examples/airflow_connection/main.tf +++ b/examples/airflow_connection/main.tf @@ -30,7 +30,7 @@ provider "external" { } module "simple-composer-environment" { - source = "../../modules/create_environment" + source = "../../modules/create_environment_v1" project_id = var.project_id composer_env_name = var.composer_env_name region = var.region @@ -40,6 +40,8 @@ module "simple-composer-environment" { use_ip_aliases = true pod_ip_allocation_range_name = var.pod_ip_allocation_range_name service_ip_allocation_range_name = var.service_ip_allocation_range_name + node_count = 3 + machine_type = "n1-standard-1" } # Making the k8s master globally available is only to make the integration testing portable and should be removed diff --git a/examples/airflow_pool/main.tf b/examples/airflow_pool/main.tf index 46d78d2d..b18afbb9 100644 --- a/examples/airflow_pool/main.tf +++ b/examples/airflow_pool/main.tf @@ -30,7 +30,7 @@ provider "external" { } module "simple-composer-environment" { - source = "../../modules/create_environment" + source = "../../modules/create_environment_v1" project_id = var.project_id composer_env_name = var.composer_env_name region = var.region @@ -40,6 +40,8 @@ module "simple-composer-environment" { use_ip_aliases = true pod_ip_allocation_range_name = var.pod_ip_allocation_range_name service_ip_allocation_range_name = var.service_ip_allocation_range_name + node_count = 3 + machine_type = "n1-standard-1" } # Making the k8s master globally available is only to make the integration testing portable and should be removed diff --git a/examples/simple_composer_env/README.md b/examples/simple_composer_env_v1/README.md similarity index 97% rename from examples/simple_composer_env/README.md rename to examples/simple_composer_env_v1/README.md index 0223de5a..45f04a6e 100644 --- a/examples/simple_composer_env/README.md +++ b/examples/simple_composer_env_v1/README.md @@ -1,4 +1,4 @@ -# Simple Cloud Composer Environment Example +# Simple Cloud Composer Environment (V1) Example This example illustrates how to use the `composer` module. diff --git a/examples/simple_composer_env/main.tf b/examples/simple_composer_env_v1/main.tf similarity index 92% rename from examples/simple_composer_env/main.tf rename to examples/simple_composer_env_v1/main.tf index 1a364a10..70991e3b 100644 --- a/examples/simple_composer_env/main.tf +++ b/examples/simple_composer_env_v1/main.tf @@ -15,7 +15,7 @@ */ module "simple-composer-environment" { - source = "../../modules/create_environment" + source = "../../modules/create_environment_v1" project_id = var.project_id composer_env_name = var.composer_env_name region = var.region @@ -25,4 +25,6 @@ module "simple-composer-environment" { use_ip_aliases = true pod_ip_allocation_range_name = var.pod_ip_allocation_range_name service_ip_allocation_range_name = var.service_ip_allocation_range_name + node_count = 3 + machine_type = "n1-standard-1" } diff --git a/examples/simple_composer_env/outputs.tf b/examples/simple_composer_env_v1/outputs.tf similarity index 100% rename from examples/simple_composer_env/outputs.tf rename to examples/simple_composer_env_v1/outputs.tf diff --git a/examples/simple_composer_env/variables.tf b/examples/simple_composer_env_v1/variables.tf similarity index 100% rename from examples/simple_composer_env/variables.tf rename to examples/simple_composer_env_v1/variables.tf diff --git a/examples/simple_composer_env/versions.tf b/examples/simple_composer_env_v1/versions.tf similarity index 100% rename from examples/simple_composer_env/versions.tf rename to examples/simple_composer_env_v1/versions.tf diff --git a/examples/simple_composer_env_v2/README.md b/examples/simple_composer_env_v2/README.md new file mode 100644 index 00000000..a60b4e37 --- /dev/null +++ b/examples/simple_composer_env_v2/README.md @@ -0,0 +1,35 @@ +# Simple Cloud Composer Environment (V2) Example + +This example illustrates how to use the `composer` module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no | +| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | +| network | Network where Cloud Composer is created. | `string` | n/a | yes | +| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | n/a | yes | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes | +| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | n/a | yes | +| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_composer_env_v2/main.tf b/examples/simple_composer_env_v2/main.tf new file mode 100644 index 00000000..630d158c --- /dev/null +++ b/examples/simple_composer_env_v2/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "simple-composer-environment" { + source = "../../modules/create_environment_v2" + project_id = var.project_id + composer_env_name = var.composer_env_name + region = var.region + composer_service_account = var.composer_service_account + network = var.network + subnetwork = var.subnetwork + pod_ip_allocation_range_name = var.pod_ip_allocation_range_name + service_ip_allocation_range_name = var.service_ip_allocation_range_name + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + scheduler = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 1 + } + web_server = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + worker = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 1 + max_count = 3 + } +} diff --git a/examples/simple_composer_env_v2/outputs.tf b/examples/simple_composer_env_v2/outputs.tf new file mode 100644 index 00000000..9f77920d --- /dev/null +++ b/examples/simple_composer_env_v2/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_id +} + +output "gke_cluster" { + description = "Google Kubernetes Engine cluster used to run the Cloud Composer Environment." + value = module.simple-composer-environment.gke_cluster +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer-environment.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer-environment.airflow_uri +} diff --git a/examples/simple_composer_env_v2/variables.tf b/examples/simple_composer_env_v2/variables.tf new file mode 100644 index 00000000..559ac558 --- /dev/null +++ b/examples/simple_composer_env_v2/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment." + default = "ci-composer" + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string +} + +variable "composer_service_account" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} + +variable "network" { + description = "Network where Cloud Composer is created." + type = string +} + +variable "subnetwork" { + description = "Subetwork where Cloud Composer is created." + type = string +} + +variable "pod_ip_allocation_range_name" { + description = "The name of the cluster's secondary range used to allocate IP addresses to pods." + type = string +} + +variable "service_ip_allocation_range_name" { + type = string + description = "The name of the services' secondary range used to allocate IP addresses to the cluster." +} diff --git a/examples/simple_composer_env_v2/versions.tf b/examples/simple_composer_env_v2/versions.tf new file mode 100644 index 00000000..1510530c --- /dev/null +++ b/examples/simple_composer_env_v2/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.8.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.8.0" + } + } +} diff --git a/kitchen.yml b/kitchen.yml index e7e9c5fa..115bf84d 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -20,15 +20,27 @@ platforms: - name: local suites: - - name: simple-composer-env + - name: simple-composer-env-v1 driver: name: terraform command_timeout: 3600 - root_module_directory: test/fixtures/simple-composer-env + root_module_directory: test/fixtures/simple-composer-env-v1 verifier: name: terraform systems: - - name: simple-composer-env + - name: simple-composer-env-v1 + backend: local + provisioner: + name: terraform + - name: simple-composer-env-v2 + driver: + name: terraform + command_timeout: 3600 + root_module_directory: test/fixtures/simple-composer-env-v2 + verifier: + name: terraform + systems: + - name: simple-composer-env-v2 backend: local provisioner: name: terraform diff --git a/main.tf b/main.tf index d1aeeec6..72835a2f 100644 --- a/main.tf +++ b/main.tf @@ -15,7 +15,7 @@ */ module "composer-environment" { - source = "./modules/create_environment" + source = "./modules/create_environment_v1" project_id = var.project_id composer_env_name = var.composer_env_name diff --git a/modules/create_environment/README.md b/modules/create_environment_v1/README.md similarity index 97% rename from modules/create_environment/README.md rename to modules/create_environment_v1/README.md index 3f2b2eb5..6fb010f9 100644 --- a/modules/create_environment/README.md +++ b/modules/create_environment_v1/README.md @@ -1,10 +1,10 @@ -# Module Cloud Composer Environment +# Module Cloud Composer Environment ([V1](https://cloud.google.com/composer/docs/concepts/overview)) This optional module is used to create a Cloud Composer environment. ```hcl module "composer" { - source = "terraform-google-modules/composer/google//modules/create_environment" + source = "terraform-google-modules/composer/google//modules/create_environment_v1" project = "project-123" name = "Composer-Prod-Env" diff --git a/modules/create_environment/main.tf b/modules/create_environment_v1/main.tf similarity index 100% rename from modules/create_environment/main.tf rename to modules/create_environment_v1/main.tf diff --git a/modules/create_environment/outputs.tf b/modules/create_environment_v1/outputs.tf similarity index 100% rename from modules/create_environment/outputs.tf rename to modules/create_environment_v1/outputs.tf diff --git a/modules/create_environment/variables.tf b/modules/create_environment_v1/variables.tf similarity index 100% rename from modules/create_environment/variables.tf rename to modules/create_environment_v1/variables.tf diff --git a/modules/create_environment/versions.tf b/modules/create_environment_v1/versions.tf similarity index 96% rename from modules/create_environment/versions.tf rename to modules/create_environment_v1/versions.tf index b73e3fdb..e679fb3f 100644 --- a/modules/create_environment/versions.tf +++ b/modules/create_environment_v1/versions.tf @@ -30,7 +30,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-composer:create_environment/v2.4.0" + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v1/v2.4.0" } } diff --git a/modules/create_environment_v2/README.md b/modules/create_environment_v2/README.md new file mode 100644 index 00000000..70ecc1e0 --- /dev/null +++ b/modules/create_environment_v2/README.md @@ -0,0 +1,61 @@ +# Module Cloud Composer Environment ([V2](https://cloud.google.com/composer/docs/composer-2/composer-overview)) + +This optional module is used to create a Cloud Composer environment. + +```hcl +module "composer" { + source = "terraform-google-modules/composer/google//modules/create_environment_v2" + + project = "project-123" + name = "Composer-Prod-Env" + region = "us-central1" +} +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | +| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. | `string` | `null` | no | +| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL. | `string` | `null` | no | +| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes | +| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | +| enable\_private\_endpoint | Configure public access to the cluster endpoint. | `bool` | `false` | no | +| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no | +| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: ENVIRONMENT\_SIZE\_SMALL, ENVIRONMENT\_SIZE\_MEDIUM, and ENVIRONMENT\_SIZE\_LARGE. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no | +| grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no | +| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.0.2-airflow-2.1.4"` | no | +| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the master. | `string` | `null` | no | +| network | The VPC network to host the composer cluster. | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | `null` | no | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | +| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no | +| scheduler | Configuration for resources used by Airflow schedulers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    count      = number
  })
| 
{
  "count": 2,
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | `null` | no | +| subnetwork | The subnetwork to host the composer cluster. | `string` | n/a | yes | +| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| use\_private\_environment | Enable private environment. | `bool` | `false` | no | +| web\_server | Configuration for resources used by Airflow web server. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
  })
| 
{
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| web\_server\_allowed\_ip\_ranges | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied. | 
list(object({
    value       = string,
    description = string
  }))
| `null` | no | +| web\_server\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the web server. | `string` | `null` | no | +| worker | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    min_count  = number
    max_count  = number
  })
| 
{
  "cpu": 2,
  "max_count": 6,
  "memory_gb": 7.5,
  "min_count": 2,
  "storage_gb": 5
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | + + diff --git a/modules/create_environment_v2/iam.tf b/modules/create_environment_v2/iam.tf new file mode 100644 index 00000000..f0897301 --- /dev/null +++ b/modules/create_environment_v2/iam.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "composer_agent_service_account" { + count = var.grant_sa_agent_permission ? 1 : 0 + project = data.google_project.project.project_id + role = "roles/composer.ServiceAgentV2Ext" + member = format("serviceAccount:%s", local.cloud_composer_sa) +} diff --git a/modules/create_environment_v2/main.tf b/modules/create_environment_v2/main.tf new file mode 100644 index 00000000..bc7c4aa4 --- /dev/null +++ b/modules/create_environment_v2/main.tf @@ -0,0 +1,151 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + subnetwork_region = var.subnetwork_region != "" ? var.subnetwork_region : var.region + cloud_composer_sa = format("service-%s@cloudcomposer-accounts.iam.gserviceaccount.com", data.google_project.project.number) + + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] +} + +resource "google_composer_environment" "composer_env" { + provider = google-beta + + project = var.project_id + name = var.composer_env_name + region = var.region + labels = var.labels + + config { + + environment_size = var.environment_size + + node_config { + network = "projects/${local.network_project_id}/global/networks/${var.network}" + subnetwork = "projects/${local.network_project_id}/regions/${local.subnetwork_region}/subnetworks/${var.subnetwork}" + service_account = var.composer_service_account + + dynamic "ip_allocation_policy" { + for_each = (var.pod_ip_allocation_range_name != null || var.service_ip_allocation_range_name != null) ? [1] : [] + content { + cluster_secondary_range_name = var.pod_ip_allocation_range_name + services_secondary_range_name = var.service_ip_allocation_range_name + } + } + } + + dynamic "software_config" { + for_each = [ + { + airflow_config_overrides = var.airflow_config_overrides + pypi_packages = var.pypi_packages + env_variables = var.env_variables + image_version = var.image_version + }] + content { + airflow_config_overrides = software_config.value["airflow_config_overrides"] + pypi_packages = software_config.value["pypi_packages"] + env_variables = software_config.value["env_variables"] + image_version = software_config.value["image_version"] + } + } + + dynamic "private_environment_config" { + for_each = var.use_private_environment ? [ + { + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr + cloud_sql_ipv4_cidr_block = var.cloud_sql_ipv4_cidr + web_server_ipv4_cidr_block = var.web_server_ipv4_cidr + cloud_composer_network_ipv4_cidr_block = var.cloud_composer_network_ipv4_cidr_block + }] : [] + content { + enable_private_endpoint = private_environment_config.value["enable_private_endpoint"] + master_ipv4_cidr_block = private_environment_config.value["master_ipv4_cidr_block"] + cloud_sql_ipv4_cidr_block = private_environment_config.value["cloud_sql_ipv4_cidr_block"] + web_server_ipv4_cidr_block = private_environment_config.value["web_server_ipv4_cidr_block"] + cloud_composer_network_ipv4_cidr_block = private_environment_config.value["cloud_composer_network_ipv4_cidr_block"] + } + } + + dynamic "maintenance_window" { + for_each = (var.maintenance_end_time != null && var.maintenance_recurrence != null) ? [ + { + start_time = var.maintenance_start_time + end_time = var.maintenance_end_time + recurrence = var.maintenance_recurrence + }] : [] + content { + start_time = maintenance_window.value["start_time"] + end_time = maintenance_window.value["end_time"] + recurrence = maintenance_window.value["recurrence"] + } + } + + workloads_config { + + dynamic "scheduler" { + for_each = var.scheduler != null ? [var.scheduler] : [] + content { + cpu = scheduler.value["cpu"] + memory_gb = scheduler.value["memory_gb"] + storage_gb = scheduler.value["storage_gb"] + count = scheduler.value["count"] + } + } + + dynamic "web_server" { + for_each = var.web_server != null ? [var.web_server] : [] + content { + cpu = web_server.value["cpu"] + memory_gb = web_server.value["memory_gb"] + storage_gb = web_server.value["storage_gb"] + } + } + + dynamic "worker" { + for_each = var.worker != null ? [var.worker] : [] + content { + cpu = worker.value["cpu"] + memory_gb = worker.value["memory_gb"] + storage_gb = worker.value["storage_gb"] + min_count = worker.value["min_count"] + max_count = worker.value["max_count"] + } + } + } + + dynamic "master_authorized_networks_config" { + for_each = local.master_authorized_networks_config + content { + enabled = length(var.master_authorized_networks) > 0 + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value["cidr_blocks"] + content { + cidr_block = master_authorized_networks_config.value["cidr_block"] + display_name = master_authorized_networks_config.value["display_name"] + } + } + } + } + } + + depends_on = [google_project_iam_member.composer_agent_service_account] + +} diff --git a/modules/create_environment_v2/outputs.tf b/modules/create_environment_v2/outputs.tf new file mode 100644 index 00000000..fbc538c6 --- /dev/null +++ b/modules/create_environment_v2/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_env_name" { + value = google_composer_environment.composer_env.name + description = "Name of the Cloud Composer Environment." +} + +output "composer_env_id" { + value = google_composer_environment.composer_env.id + description = "ID of Cloud Composer Environment." +} + +output "gke_cluster" { + value = google_composer_environment.composer_env.config.0.gke_cluster + description = "Google Kubernetes Engine cluster used to run the Cloud Composer Environment." +} + +output "gcs_bucket" { + value = google_composer_environment.composer_env.config.0.dag_gcs_prefix + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." +} + +output "airflow_uri" { + value = google_composer_environment.composer_env.config.0.airflow_uri + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." +} diff --git a/modules/create_environment_v2/variables.tf b/modules/create_environment_v2/variables.tf new file mode 100644 index 00000000..9b3f865b --- /dev/null +++ b/modules/create_environment_v2/variables.tf @@ -0,0 +1,233 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment" + type = string +} + +variable "region" { + description = "Region where the Cloud Composer Environment is created." + type = string + default = "us-central1" +} + +variable "labels" { + type = map(string) + description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer." + default = {} +} + +variable "network" { + type = string + description = "The VPC network to host the composer cluster." +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the composer cluster." +} + +variable "subnetwork_region" { + type = string + description = "The subnetwork region of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "composer_service_account" { + description = "Service Account for running Cloud Composer." + type = string + default = null +} + +variable "pod_ip_allocation_range_name" { + description = "The name of the cluster's secondary range used to allocate IP addresses to pods." + type = string + default = null +} + +variable "service_ip_allocation_range_name" { + type = string + description = "The name of the services' secondary range used to allocate IP addresses to the cluster." + default = null +} + +variable "airflow_config_overrides" { + type = map(string) + description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"." + default = {} +} + +variable "env_variables" { + type = map(string) + description = "Variables of the airflow environment." + default = {} +} + +variable "image_version" { + type = string + description = "The version of the aiflow running in the cloud composer environment." + default = "composer-2.0.2-airflow-2.1.4" +} + +variable "pypi_packages" { + type = map(string) + description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." + default = {} +} + +variable "use_private_environment" { + description = "Enable private environment." + type = bool + default = false +} + +variable "cloud_sql_ipv4_cidr" { + description = "The CIDR block from which IP range in tenant project will be reserved for Cloud SQL." + type = string + default = null +} + +variable "web_server_ipv4_cidr" { + description = "The CIDR block from which IP range in tenant project will be reserved for the web server." + type = string + default = null +} + +variable "master_ipv4_cidr" { + description = "The CIDR block from which IP range in tenant project will be reserved for the master." + type = string + default = null +} + +variable "enable_private_endpoint" { + description = "Configure public access to the cluster endpoint." + type = bool + default = false +} + +variable "cloud_composer_network_ipv4_cidr_block" { + description = "The CIDR block from which IP range in tenant project will be reserved." + type = string + default = null +} + +variable "web_server_allowed_ip_ranges" { + description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied." + default = null + type = list(object({ + value = string, + description = string + })) +} + +variable "maintenance_start_time" { + description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" + type = string + default = "05:00" +} + +variable "maintenance_end_time" { + description = "Time window specified for recurring maintenance operations in RFC3339 format" + type = string + default = null +} + +variable "maintenance_recurrence" { + description = "Frequency of the recurring maintenance window in RFC5545 format." + type = string + default = null +} + +variable "environment_size" { + type = string + description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: ENVIRONMENT_SIZE_SMALL, ENVIRONMENT_SIZE_MEDIUM, and ENVIRONMENT_SIZE_LARGE." + default = "ENVIRONMENT_SIZE_MEDIUM" +} + +variable "scheduler" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + count = 2 + } + description = "Configuration for resources used by Airflow schedulers." +} + +variable "web_server" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + } + description = "Configuration for resources used by Airflow web server." +} + +variable "worker" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + min_count = number + max_count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + min_count = 2 + max_count = 6 + } + description = "Configuration for resources used by Airflow workers." +} + +variable "master_authorized_networks" { + type = list(object({ + cidr_block = string + display_name = string + })) + default = [] + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "grant_sa_agent_permission" { + type = bool + default = true + description = "Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. " +} diff --git a/modules/create_environment_v2/versions.tf b/modules/create_environment_v2/versions.tf new file mode 100644 index 00000000..f73cd777 --- /dev/null +++ b/modules/create_environment_v2/versions.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 4.8.0, < 5.0" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.8.0, < 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v2/v2.4.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v2/v2.4.0" + } + +} diff --git a/test/fixtures/simple-composer-env/main.tf b/test/fixtures/simple-composer-env-v1/main.tf similarity index 95% rename from test/fixtures/simple-composer-env/main.tf rename to test/fixtures/simple-composer-env-v1/main.tf index b7c02f93..c146f3a2 100644 --- a/test/fixtures/simple-composer-env/main.tf +++ b/test/fixtures/simple-composer-env-v1/main.tf @@ -15,7 +15,7 @@ */ module "simple-composer" { - source = "../../../examples/simple_composer_env" + source = "../../../examples/simple_composer_env_v1" project_id = var.project_id composer_env_name = "composer-env-${random_id.random_suffix.hex}" diff --git a/test/fixtures/simple-composer-env/network.tf b/test/fixtures/simple-composer-env-v1/network.tf similarity index 100% rename from test/fixtures/simple-composer-env/network.tf rename to test/fixtures/simple-composer-env-v1/network.tf diff --git a/test/fixtures/simple-composer-env/outputs.tf b/test/fixtures/simple-composer-env-v1/outputs.tf similarity index 100% rename from test/fixtures/simple-composer-env/outputs.tf rename to test/fixtures/simple-composer-env-v1/outputs.tf diff --git a/test/fixtures/simple-composer-env/variables.tf b/test/fixtures/simple-composer-env-v1/variables.tf similarity index 100% rename from test/fixtures/simple-composer-env/variables.tf rename to test/fixtures/simple-composer-env-v1/variables.tf diff --git a/test/fixtures/simple-composer-env-v2/main.tf b/test/fixtures/simple-composer-env-v2/main.tf new file mode 100644 index 00000000..242926a6 --- /dev/null +++ b/test/fixtures/simple-composer-env-v2/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "simple-composer" { + source = "../../../examples/simple_composer_env_v2" + + project_id = var.project_id + composer_env_name = "composer-env-${random_id.random_suffix.hex}" + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + pod_ip_allocation_range_name = google_compute_subnetwork.main.secondary_ip_range[0].range_name + service_ip_allocation_range_name = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +resource "random_id" "random_suffix" { + byte_length = 2 +} diff --git a/test/fixtures/simple-composer-env-v2/network.tf b/test/fixtures/simple-composer-env-v2/network.tf new file mode 100644 index 00000000..c2ad05cd --- /dev/null +++ b/test/fixtures/simple-composer-env-v2/network.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + private_ip_google_access = true + + secondary_ip_range { + range_name = "ci-composer-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "ci-composer-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} diff --git a/test/fixtures/simple-composer-env-v2/outputs.tf b/test/fixtures/simple-composer-env-v2/outputs.tf new file mode 100644 index 00000000..991ce3d4 --- /dev/null +++ b/test/fixtures/simple-composer-env-v2/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + description = "Project ID where Cloud Composer Environment is created." + value = var.project_id +} + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer.composer_env_name +} + +output "network" { + description = "The Cloud Composer Network." + value = google_compute_network.main.name +} + +output "subnetwork" { + description = "The Cloud Composer Subnetwork." + value = google_compute_subnetwork.main.name +} + +output "pod_ip_allocation_range_name" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "service_ip_allocation_range_name" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer.composer_env_id +} + +output "gke_cluster" { + description = "Google Kubernetes Engine cluster used to run the Cloud Composer Environment." + value = module.simple-composer.gke_cluster +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer.airflow_uri +} diff --git a/test/fixtures/simple-composer-env-v2/variables.tf b/test/fixtures/simple-composer-env-v2/variables.tf new file mode 100644 index 00000000..ff99cfed --- /dev/null +++ b/test/fixtures/simple-composer-env-v2/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_sa" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string + default = "us-central1" +} diff --git a/test/integration/simple-composer-env/controls/simple-composer.rb b/test/integration/simple-composer-env-v1/controls/simple-composer.rb similarity index 100% rename from test/integration/simple-composer-env/controls/simple-composer.rb rename to test/integration/simple-composer-env-v1/controls/simple-composer.rb diff --git a/test/integration/simple-composer-env/inspec.yml b/test/integration/simple-composer-env-v1/inspec.yml similarity index 100% rename from test/integration/simple-composer-env/inspec.yml rename to test/integration/simple-composer-env-v1/inspec.yml diff --git a/test/integration/simple-composer-env-v2/controls/simple-composer.rb b/test/integration/simple-composer-env-v2/controls/simple-composer.rb new file mode 100644 index 00000000..43165547 --- /dev/null +++ b/test/integration/simple-composer-env-v2/controls/simple-composer.rb @@ -0,0 +1,49 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cloud Composer Environment + +project_id = attribute('project_id') +composer_env_name = attribute('composer_env_name') +composer_env_id = attribute('composer_env_id') +gke_cluster = attribute('gke_cluster') +gcs_bucket = attribute('gcs_bucket') +airflow_uri = attribute('airflow_uri') + +control "Cloud Composer Environment" do + title "Simple Cloud Composer" + + describe command("gcloud composer environments describe #{attribute("composer_env_name")} --location=us-central1 --project=#{attribute("project_id")} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq "" } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "composer_environment_config" do + it "has config" do + config_data = data['config'] + expect(data["name"]).to include("projects/#{attribute("project_id")}/locations/us-central1/environments/#{attribute("composer_env_name")}") + expect(config_data["gkeCluster"]). to include(attribute("gke_cluster")) + expect(config_data["dagGcsPrefix"]). to include(attribute("gcs_bucket")) + expect(config_data["airflowUri"]). to include(attribute("airflow_uri")) + end + end + end +end diff --git a/test/integration/simple-composer-env-v2/inspec.yml b/test/integration/simple-composer-env-v2/inspec.yml new file mode 100644 index 00000000..6c95f717 --- /dev/null +++ b/test/integration/simple-composer-env-v2/inspec.yml @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: simple-composer-env +attributes: + - name: project_id + required: true + type: string + - name: composer_env_name + required: true + type: string + - name: composer_env_id + required: true + type: string + - name: gke_cluster + required: true + type: string + - name: gcs_bucket + required: true + type: string + - name: airflow_uri + required: true + type: string diff --git a/test/setup/iam.tf b/test/setup/iam.tf index b18b21cc..fcff3206 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -24,6 +24,12 @@ locals { ] } +resource "google_project_iam_member" "composer_agent_service_account" { + project = module.project.project_id + role = "roles/composer.ServiceAgentV2Ext" + member = format("serviceAccount:service-%s@cloudcomposer-accounts.iam.gserviceaccount.com", module.project.project_number) +} + resource "google_service_account" "int_test" { project = module.project.project_id account_id = "ci-composer"