From a5d12b3bf77c6acb47eb2d817b9f115e831a3947 Mon Sep 17 00:00:00 2001 From: Caetano Colin <164910343+caetano-colin@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:44:56 -0300 Subject: [PATCH] chore(refactor): Removing data filter by labels and updating outdated README.md on service_catalog/modules/composer (#56) * composer module fix * adjust readme * update readme * update readme * update readme --- .../modules/composer/README.md | 129 +++++------------- .../service-catalog/modules/composer/data.tf | 12 +- .../modules/composer/locals.tf | 2 - .../modules/composer/variables.tf | 44 +++--- 4 files changed, 58 insertions(+), 129 deletions(-) diff --git a/5-app-infra/source_repos/service-catalog/modules/composer/README.md b/5-app-infra/source_repos/service-catalog/modules/composer/README.md index 87e59ee6..28f038fa 100644 --- a/5-app-infra/source_repos/service-catalog/modules/composer/README.md +++ b/5-app-infra/source_repos/service-catalog/modules/composer/README.md @@ -1,92 +1,11 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +# Composer ## Requirements -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | -| [random](#provider\_random) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 8.1 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_cloudbuildv2_connection.repo_connect](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_cloudbuildv2_connection) | resource | -| [google-beta_google_cloudbuildv2_repository.repo](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_cloudbuildv2_repository) | resource | -| [google-beta_google_composer_environment.cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_composer_environment) | resource | -| [google_cloudbuild_trigger.zip_files](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | -| [google_secret_manager_secret_iam_policy.policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_policy) | resource | -| [google_service_account.trigger_sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | -| [google_service_account_iam_member.trigger_sa_impersonate](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_iam_policy.serviceagent_secretAccessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_netblock_ip_ranges.health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.iap_forwarders](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.legacy_health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_pubsub_topic.secret_rotations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/pubsub_topic) | data source | -| [google_secret_manager_secret.github_api_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/secret_manager_secret) | data source | -| [google_secret_manager_secret_version.github_api](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/secret_manager_secret_version) | data source | -| [google_service_account.composer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/service_account) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [airflow\_config\_overrides](#input\_airflow\_config\_overrides) | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | -| [env\_variables](#input\_env\_variables) | Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z\_][a-zA-Z0-9\_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW\_\_[A-Z0-9\_]+\_\_[A-Z0-9\_]+), and they cannot match any of the following reserved names: [AIRFLOW\_HOME,C\_FORCE\_ROOT,CONTAINER\_NAME,DAGS\_FOLDER,GCP\_PROJECT,GCS\_BUCKET,GKE\_CLUSTER\_NAME,SQL\_DATABASE,SQL\_INSTANCE,SQL\_PASSWORD,SQL\_PROJECT,SQL\_REGION,SQL\_USER] | `map(any)` | `{}` | no | -| [github\_app\_installation\_id](#input\_github\_app\_installation\_id) | The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build | `number` | n/a | yes | -| [github\_name\_prefix](#input\_github\_name\_prefix) | A name for your github connection to cloubuild | `string` | `"github-modules"` | no | -| [github\_remote\_uri](#input\_github\_remote\_uri) | Url of your github repo | `string` | n/a | yes | -| [github\_secret\_name](#input\_github\_secret\_name) | Name of the github secret to extract github token info | `string` | `"github-api-token"` | no | -| [image\_version](#input\_image\_version) | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.5.2-airflow-2.6.3"` | no | -| [labels](#input\_labels) | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | -| [maintenance\_window](#input\_maintenance\_window) | The configuration settings for Cloud Composer maintenance window. | 
object({
    start_time = string
    end_time   = string
    recurrence = string
  })
| 
{
  "end_time": "2021-01-01T13:00:00Z",
  "recurrence": "FREQ=WEEKLY;BYDAY=SU",
  "start_time": "2021-01-01T01:00:00Z"
}
| no | -| [name](#input\_name) | name of the Composer environment | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Optional project ID where Cloud Composer Environment is created. | `string` | `null` | no | -| [pypi\_packages](#input\_pypi\_packages) | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | -| [python\_version](#input\_python\_version) | The default version of Python used to run the Airflow scheduler, worker, and webserver processes. | `string` | `"3"` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| [service\_account\_prefix](#input\_service\_account\_prefix) | Name prefix to use for service accounts. | `string` | `"sa"` | no | -| [web\_server\_allowed\_ip\_ranges](#input\_web\_server\_allowed\_ip\_ranges) | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied. | 
list(object({
    value       = string
    description = string
  }))
| `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [airflow\_uri](#output\_airflow\_uri) | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | -| [composer\_env\_id](#output\_composer\_env\_id) | ID of Cloud Composer Environment. | -| [composer\_env\_name](#output\_composer\_env\_name) | Name of the Cloud Composer Environment. | -| [gcs\_bucket](#output\_gcs\_bucket) | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | -| [gke\_cluster](#output\_gke\_cluster) | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | - +- Githup Cloud Build App installed. `https://github.com/settings/installations` +- Github repository created. +- Github Token secret version created on Secret Manager. `https://console.cloud.google.com/security/secret-manager/secret/github-api-token/versions?project=ML_PROJECT_ID` +- Composer SA created. ## Security Controls @@ -94,22 +13,24 @@ The following table outlines which of the suggested controls for Vertex Generati | Name | Control ID | NIST 800-53 | CRI Profile | Category | Source Blueprint |------|------------|-------------|-------------|----------| ----------------| |Customer Managed Encryption Keys| COM-CO-2.3| SC-12
SC-13| PR.DS-1.1
PR.DS-2.1
PR.DS-2.2
PR.DS-5.1 | Recommended | Secure Foundation v4 + ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | -| env\_variables | Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z\_][a-zA-Z0-9\_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW\_\_[A-Z0-9\_]+\_\_[A-Z0-9\_]+), and they cannot match any of the following reserved names: [AIRFLOW\_HOME,C\_FORCE\_ROOT,CONTAINER\_NAME,DAGS\_FOLDER,GCP\_PROJECT,GCS\_BUCKET,GKE\_CLUSTER\_NAME,SQL\_DATABASE,SQL\_INSTANCE,SQL\_PASSWORD,SQL\_PROJECT,SQL\_REGION,SQL\_USER] | `map(any)` | `{}` | no | -| github\_app\_installation\_id | The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build | `number` | n/a | yes | -| github\_name\_prefix | A name for your github connection to cloubuild | `string` | `"github-modules"` | no | -| github\_remote\_uri | Url of your github repo | `string` | n/a | yes | -| github\_secret\_name | Name of the github secret to extract github token info | `string` | `"github-api-token"` | no | -| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.5.2-airflow-2.6.3"` | no | +| env\_variables | Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z\_][a-zA-Z0-9\_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW\_\_[A-Z0-9\_]+\_\_[A-Z0-9\_]+), and they cannot match any of the following reserved names: [AIRFLOW\_HOME,C\_FORCE\_ROOT,CONTAINER\_NAME,DAGS\_FOLDER,GCP\_PROJECT,GCS\_BUCKET,GKE\_CLUSTER\_NAME,SQL\_DATABASE,SQL\_INSTANCE,SQL\_PASSWORD,SQL\_PROJECT,SQL\_REGION,SQL\_USER]. | `map(any)` | `{}` | no | +| github\_app\_installation\_id | The app installation ID that was created when installing Google Cloud Build in GitHub: https://github.com/apps/google-cloud-build. | `number` | n/a | yes | +| github\_name\_prefix | A name for your GitHub connection to Cloud Build. | `string` | `"github-modules"` | no | +| github\_remote\_uri | URL of your GitHub repo. | `string` | n/a | yes | +| github\_secret\_name | Name of the GitHub secret to extract GitHub token info. | `string` | `"github-api-token"` | no | +| image\_version | The version of the Airflow running in the Cloud Composer environment. | `string` | `"composer-2.5.2-airflow-2.6.3"` | no | +| kms\_keyring | The KMS keyring that will be used when selecting the KMS key, preferably this should be on the same region as the other resources and the same environment.
This value can be obtained by running "gcloud kms keyrings list --project=KMS\_PROJECT\_ID --location=REGION." | `string` | n/a | yes | | labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | | maintenance\_window | The configuration settings for Cloud Composer maintenance window. | 
object({
    start_time = string
    end_time   = string
    recurrence = string
  })
| 
{
  "end_time": "2021-01-01T13:00:00Z",
  "recurrence": "FREQ=WEEKLY;BYDAY=SU",
  "start_time": "2021-01-01T01:00:00Z"
}
| no | -| name | name of the Composer environment | `string` | n/a | yes | -| project\_id | Optional project ID where Cloud Composer Environment is created. | `string` | `null` | no | +| name | Name of the Composer environment. | `string` | n/a | yes | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | | pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | | python\_version | The default version of Python used to run the Airflow scheduler, worker, and webserver processes. | `string` | `"3"` | no | | region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | @@ -127,3 +48,23 @@ The following table outlines which of the suggested controls for Vertex Generati | gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | + + +## Troubleshooting + +- Error: googleapi: Error 400: Composer API Service Agent service account (service-ML_PROJECT_NUMBER@cloudcomposer-accounts.iam.gserviceaccount.com) does not have required permissions set. Cloud Composer v2 API Service Agent Extension role might be missing. Please refer to https://cloud.google.com/composer/docs/composer-2/create-environments#grant-permissions and Composer Creation Troubleshooting pages to resolve this issue., failedPrecondition + +```bash +gcloud projects add-iam-policy-binding ML_PROJECT_NUMBER --member=serviceAccount:service-ML_PROJECT_NUMBER@cloudcomposer-accounts.iam.gserviceaccount.com --role=roles/composer.ServiceAgentV2Ext +``` + +- If Service Agent cannot use encryption key, grant `roles/cloudkms.cryptoKeyEncrypterDecrypter` for each of the following identities on the key: + +```txt +service-$ML_PROJECT_NUMBER@cloudcomposer-accounts.iam.gserviceaccount.com +service-$ML_PROJECT_NUMBER@gcp-sa-artifactregistry.iam.gserviceaccount.com +service-$ML_PROJECT_NUMBER@container-engine-robot.iam.gserviceaccount.com +service-$ML_PROJECT_NUMBER@gcp-sa-pubsub.iam.gserviceaccount.com +service-$ML_PROJECT_NUMBER@compute-system.iam.gserviceaccount.com +service-$ML_PROJECT_NUMBER@gs-project-accounts.iam.gserviceaccount.com +``` diff --git a/5-app-infra/source_repos/service-catalog/modules/composer/data.tf b/5-app-infra/source_repos/service-catalog/modules/composer/data.tf index 5d66e4b3..bbf89bd8 100644 --- a/5-app-infra/source_repos/service-catalog/modules/composer/data.tf +++ b/5-app-infra/source_repos/service-catalog/modules/composer/data.tf @@ -31,19 +31,9 @@ data "google_project" "project" { project_id = var.project_id } -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - data "google_kms_crypto_key" "key" { name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id + key_ring = var.kms_keyring } data "google_service_account" "composer" { diff --git a/5-app-infra/source_repos/service-catalog/modules/composer/locals.tf b/5-app-infra/source_repos/service-catalog/modules/composer/locals.tf index 795efcc1..d23e6170 100644 --- a/5-app-infra/source_repos/service-catalog/modules/composer/locals.tf +++ b/5-app-infra/source_repos/service-catalog/modules/composer/locals.tf @@ -34,8 +34,6 @@ locals { private_service_connect_ip = "10.116.46.2" - keyring_name = "sample-keyring" - sa_name = format("%s-%s", data.google_project.project.labels.env_code, var.name) labels = merge( diff --git a/5-app-infra/source_repos/service-catalog/modules/composer/variables.tf b/5-app-infra/source_repos/service-catalog/modules/composer/variables.tf index ea77a199..b4497a15 100644 --- a/5-app-infra/source_repos/service-catalog/modules/composer/variables.tf +++ b/5-app-infra/source_repos/service-catalog/modules/composer/variables.tf @@ -16,7 +16,7 @@ variable "name" { type = string - description = "name of the Composer environment" + description = "Name of the Composer environment." } variable "region" { @@ -41,14 +41,7 @@ variable "maintenance_window" { end_time = string recurrence = string }) - description = "The configuration settings for Cloud Composer maintenance window." - - # Set Start time, Timezone, Days, and Length, so that combined time for the - # specified schedule is at least 12 hours in a 7-day rolling window. For example, - # a period of 4 hours every Monday, Wednesday, and Friday provides the required amount of time. - - # 12-hour maintenance window between 01:00 and 13:00 (UTC) on Sundays default = { start_time = "2021-01-01T01:00:00Z" end_time = "2021-01-01T13:00:00Z" @@ -59,6 +52,7 @@ variable "maintenance_window" { ################################################ # software_config # ################################################ + variable "airflow_config_overrides" { type = map(string) description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"." @@ -67,13 +61,13 @@ variable "airflow_config_overrides" { variable "env_variables" { type = map(any) - description = "Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z_][a-zA-Z0-9_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW__[A-Z0-9_]+__[A-Z0-9_]+), and they cannot match any of the following reserved names: [AIRFLOW_HOME,C_FORCE_ROOT,CONTAINER_NAME,DAGS_FOLDER,GCP_PROJECT,GCS_BUCKET,GKE_CLUSTER_NAME,SQL_DATABASE,SQL_INSTANCE,SQL_PASSWORD,SQL_PROJECT,SQL_REGION,SQL_USER]" + description = "Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z_][a-zA-Z0-9_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW__[A-Z0-9_]+__[A-Z0-9_]+), and they cannot match any of the following reserved names: [AIRFLOW_HOME,C_FORCE_ROOT,CONTAINER_NAME,DAGS_FOLDER,GCP_PROJECT,GCS_BUCKET,GKE_CLUSTER_NAME,SQL_DATABASE,SQL_INSTANCE,SQL_PASSWORD,SQL_PROJECT,SQL_REGION,SQL_USER]." default = {} } variable "image_version" { type = string - description = "The version of the aiflow running in the cloud composer environment." + description = "The version of the Airflow running in the Cloud Composer environment." default = "composer-2.5.2-airflow-2.6.3" validation { condition = can(regex("^composer-([2-9]|[1-9][0-9]+)\\..*$", var.image_version)) @@ -83,56 +77,62 @@ variable "image_version" { variable "pypi_packages" { type = map(string) - description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." + description = "Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." default = {} } variable "python_version" { - description = "The default version of Python used to run the Airflow scheduler, worker, and webserver processes." type = string + description = "The default version of Python used to run the Airflow scheduler, worker, and webserver processes." default = "3" } variable "web_server_allowed_ip_ranges" { - description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied." - default = null type = list(object({ value = string description = string })) + description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied." + default = null } variable "github_remote_uri" { - description = "Url of your github repo" type = string + description = "URL of your GitHub repo." } variable "github_name_prefix" { - description = "A name for your github connection to cloubuild" type = string + description = "A name for your GitHub connection to Cloud Build." default = "github-modules" } variable "github_app_installation_id" { - description = "The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build" type = number - + description = "The app installation ID that was created when installing Google Cloud Build in GitHub: https://github.com/apps/google-cloud-build." } variable "service_account_prefix" { - description = "Name prefix to use for service accounts." type = string + description = "Name prefix to use for service accounts." default = "sa" } variable "project_id" { - description = "Optional project ID where Cloud Composer Environment is created." type = string - default = null + description = "Project ID where Cloud Composer Environment is created." } variable "github_secret_name" { - description = "Name of the github secret to extract github token info" type = string + description = "Name of the GitHub secret to extract GitHub token info." default = "github-api-token" } + +variable "kms_keyring" { + type = string + description = <