From d3526ca6949620effdddb0dc52f0cbd9c4e8b9ba Mon Sep 17 00:00:00 2001 From: Caetano Colin <164910343+caetano-colin@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:53:06 -0300 Subject: [PATCH] fix: remove data source on service catalog and artifact publish (#51) * fix: remove data sources * fix: add output on projects * adjust lint build * add modules instruction * docker generate docs * update workflows file * fix: remove data source on artifact publish * add files to lint.yaml --- .github/workflows/lint.yaml | 4 ++-- 4-projects/business_unit_3/shared/README.md | 1 + 4-projects/business_unit_3/shared/outputs.tf | 5 +++++ 5-app-infra/README.md | 2 ++ 5-app-infra/modules/publish_artifacts/data.tf | 18 +-------------- 5-app-infra/modules/publish_artifacts/main.tf | 6 ++--- .../modules/publish_artifacts/variables.tf | 5 +++++ 5-app-infra/modules/service_catalog/data.tf | 22 +------------------ 5-app-infra/modules/service_catalog/main.tf | 9 ++++---- .../modules/service_catalog/variables.tf | 12 +++++++++- .../shared/publish_artifacts.tf | 12 +++++++++- .../business_unit_3/shared/remote.tf | 1 + .../business_unit_3/shared/remote.tf | 3 ++- .../business_unit_3/shared/service_catalog.tf | 15 +++++++++++-- Makefile | 2 +- 15 files changed, 63 insertions(+), 54 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2d576017..ca36fb98 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -48,10 +48,10 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/business_unit_3/shared/README.md index b65cb8a9..e67eff55 100644 --- a/4-projects/business_unit_3/shared/README.md +++ b/4-projects/business_unit_3/shared/README.md @@ -34,6 +34,7 @@ | service\_catalog\_project\_id | Service Catalog Project ID. | | service\_catalog\_repo\_id | ID of the Service Catalog repository | | service\_catalog\_repo\_name | The name of the Service Catalog repository | +| shared\_level\_keyrings | Keyrings used on shared level project creation | | state\_buckets | GCS Buckets to store TF state | | terraform\_service\_accounts | APP Infra Pipeline Terraform Accounts. | diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/business_unit_3/shared/outputs.tf index 88c5d34c..ccd45012 100644 --- a/4-projects/business_unit_3/shared/outputs.tf +++ b/4-projects/business_unit_3/shared/outputs.tf @@ -92,3 +92,8 @@ output "artifacts_repo_id" { description = "ID of the Artifacts repository" value = module.ml_infra_projects.artifacts_repo_id } + +output "shared_level_keyrings" { + description = "Keyrings used on shared level project creation" + value = local.shared_kms_key_ring +} diff --git a/5-app-infra/README.md b/5-app-infra/README.md index 517bc177..0f7ac549 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -159,6 +159,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh @@ -330,6 +331,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh diff --git a/5-app-infra/modules/publish_artifacts/data.tf b/5-app-infra/modules/publish_artifacts/data.tf index afd1ef09..4671af93 100644 --- a/5-app-infra/modules/publish_artifacts/data.tf +++ b/5-app-infra/modules/publish_artifacts/data.tf @@ -1,5 +1,5 @@ /** - * Copyright 2023 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,19 +17,3 @@ data "google_project" "project" { project_id = var.project_id } - -data "google_projects" "kms" { - filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE" - # filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} diff --git a/5-app-infra/modules/publish_artifacts/main.tf b/5-app-infra/modules/publish_artifacts/main.tf index c3fc48a6..141e747d 100644 --- a/5-app-infra/modules/publish_artifacts/main.tf +++ b/5-app-infra/modules/publish_artifacts/main.tf @@ -21,7 +21,7 @@ resource "google_project_service_identity" "artifact_registry_agent" { } resource "google_kms_crypto_key_iam_member" "artifact-kms-key-binding" { - crypto_key_id = data.google_kms_crypto_key.key.id + crypto_key_id = var.kms_crypto_key role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = "serviceAccount:${google_project_service_identity.artifact_registry_agent.email}" } @@ -40,7 +40,7 @@ resource "google_artifact_registry_repository" "repo" { #NIST 800-53: SC-12 SC-13 #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - kms_key_name = data.google_kms_crypto_key.key.id + kms_key_name = var.kms_crypto_key #Cleanup policy #Control ID: AR-CO-6.1 @@ -101,6 +101,7 @@ resource "google_cloudbuild_trigger" "docker_build" { repo_name = var.name } build { + timeout = "1800s" step { id = "unshallow" name = "gcr.io/cloud-builders/git" @@ -119,7 +120,6 @@ resource "google_cloudbuild_trigger" "docker_build" { <<-EOT changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r) changed_folders=$(echo "$changed_files" | awk -F/ '{print $2}' | sort | uniq ) - for folder in $changed_folders; do echo "Found docker folder: $folder" echo $folder >> /workspace/docker_build diff --git a/5-app-infra/modules/publish_artifacts/variables.tf b/5-app-infra/modules/publish_artifacts/variables.tf index 28e68618..18e8a528 100644 --- a/5-app-infra/modules/publish_artifacts/variables.tf +++ b/5-app-infra/modules/publish_artifacts/variables.tf @@ -69,3 +69,8 @@ variable "environment" { variable "project_id" { description = "Project ID" } + +variable "kms_crypto_key" { + description = "KMS Key to be used" + type = string +} diff --git a/5-app-infra/modules/service_catalog/data.tf b/5-app-infra/modules/service_catalog/data.tf index 12e6f689..4671af93 100644 --- a/5-app-infra/modules/service_catalog/data.tf +++ b/5-app-infra/modules/service_catalog/data.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,3 @@ data "google_project" "project" { project_id = var.project_id } - -data "google_projects" "kms" { - filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE" - # filter = "labels.application_name:org-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_projects" "log" { - filter = "labels.application_name:org-logging labels.environment:production lifecycleState:ACTIVE" -} diff --git a/5-app-infra/modules/service_catalog/main.tf b/5-app-infra/modules/service_catalog/main.tf index 7f9aed12..875aa47e 100644 --- a/5-app-infra/modules/service_catalog/main.tf +++ b/5-app-infra/modules/service_catalog/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ # service = "storage.googleapis.com" # } # resource "google_kms_crypto_key_iam_member" "storage-kms-key-binding" { -# crypto_key_id = data.google_kms_crypto_key.key.id +# crypto_key_id = var.kms_crypto_key # role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" # member = "serviceAccount:${google_project_service_identity.storage_agent.email}" # } @@ -41,13 +41,13 @@ resource "google_storage_bucket" "bucket" { uniform_bucket_level_access = true encryption { - default_kms_key_name = data.google_kms_crypto_key.key.id + default_kms_key_name = var.kms_crypto_key } versioning { enabled = true } logging { - log_bucket = join("-", [local.log_bucket_prefix, data.google_projects.log.projects.0.project_id]) + log_bucket = var.log_bucket } } @@ -142,4 +142,3 @@ resource "google_cloudbuild_trigger" "zip_files" { } } } - diff --git a/5-app-infra/modules/service_catalog/variables.tf b/5-app-infra/modules/service_catalog/variables.tf index df0cf433..0b1bbb60 100644 --- a/5-app-infra/modules/service_catalog/variables.tf +++ b/5-app-infra/modules/service_catalog/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,3 +42,13 @@ variable "machine_learning_project_number" { description = "Project Number for the Machine Learning (Vertex) Project" type = string } + +variable "kms_crypto_key" { + description = "KMS Key to be used" + type = string +} + +variable "log_bucket" { + description = "Bucket to store logs from service catalog bucket" + type = string +} diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf index 5983f92c..2d535968 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf +++ b/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf @@ -14,8 +14,16 @@ * limitations under the License. */ +locals { + region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] +} + +data "google_project" "common_artifacts" { + project_id = local.common_artifacts_project_id +} + module "artifact_publish" { - source = "../../../../modules/publish_artifacts" + source = "../../modules/publish_artifacts" environment = local.environment description = "Publish Artifacts for ML Projects" @@ -34,4 +42,6 @@ module "artifact_publish" { } ] }] + + kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_artifacts.name}" } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf index a9292473..c98f85b7 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf +++ b/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf @@ -18,6 +18,7 @@ locals { common_artifacts_project_id = data.terraform_remote_state.projects_shared.outputs.common_artifacts_project_id service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id artifacts_repo_name = data.terraform_remote_state.projects_shared.outputs.artifacts_repo_name + shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings } data "terraform_remote_state" "projects_shared" { diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf index 21072be3..366949d7 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf +++ b/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf @@ -18,7 +18,8 @@ locals { service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id service_catalog_repo_name = data.terraform_remote_state.projects_shared.outputs.service_catalog_repo_name machine_learning_project_number = data.terraform_remote_state.machine_learning_development.outputs.machine_learning_project_number - tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["bu3-service-catalog"] + tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["ml-service-catalog"] + shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings } data "terraform_remote_state" "projects_shared" { diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf index 35376c79..2a031be5 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf +++ b/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,23 @@ * limitations under the License. */ +locals { + region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] +} + +data "google_project" "common_svc_catalog" { + project_id = local.service_catalog_project_id +} + module "service_catalog" { - source = "../../../../modules/service_catalog" + source = "../../modules/service_catalog" project_id = local.service_catalog_project_id region = var.instance_region name = local.service_catalog_repo_name machine_learning_project_number = local.machine_learning_project_number tf_service_catalog_sa_email = local.tf_service_catalog_sa_email + + log_bucket = var.log_bucket + kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_svc_catalog.name}" } diff --git a/Makefile b/Makefile index 215cbcae..a06f9d1b 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker_test_lint: docker run --rm -it \ -e ENABLE_PARALLEL=0 \ -e DISABLE_TFLINT=1 \ - -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform" \ + -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared" \ -v $(CURDIR):/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh