From e4385a23157e6a0a40982dd938e94c64e572ead0 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 09:14:05 -0300 Subject: [PATCH 1/8] fix: remove data sources --- 5-app-infra/modules/service_catalog/data.tf | 22 +------------------ 5-app-infra/modules/service_catalog/main.tf | 9 ++++---- .../modules/service_catalog/variables.tf | 12 +++++++++- .../business_unit_3/shared/remote.tf | 3 ++- .../business_unit_3/shared/service_catalog.tf | 15 +++++++++++-- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/5-app-infra/modules/service_catalog/data.tf b/5-app-infra/modules/service_catalog/data.tf index 12e6f689..4671af93 100644 --- a/5-app-infra/modules/service_catalog/data.tf +++ b/5-app-infra/modules/service_catalog/data.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,3 @@ data "google_project" "project" { project_id = var.project_id } - -data "google_projects" "kms" { - filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE" - # filter = "labels.application_name:org-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_projects" "log" { - filter = "labels.application_name:org-logging labels.environment:production lifecycleState:ACTIVE" -} diff --git a/5-app-infra/modules/service_catalog/main.tf b/5-app-infra/modules/service_catalog/main.tf index 7f9aed12..875aa47e 100644 --- a/5-app-infra/modules/service_catalog/main.tf +++ b/5-app-infra/modules/service_catalog/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ # service = "storage.googleapis.com" # } # resource "google_kms_crypto_key_iam_member" "storage-kms-key-binding" { -# crypto_key_id = data.google_kms_crypto_key.key.id +# crypto_key_id = var.kms_crypto_key # role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" # member = "serviceAccount:${google_project_service_identity.storage_agent.email}" # } @@ -41,13 +41,13 @@ resource "google_storage_bucket" "bucket" { uniform_bucket_level_access = true encryption { - default_kms_key_name = data.google_kms_crypto_key.key.id + default_kms_key_name = var.kms_crypto_key } versioning { enabled = true } logging { - log_bucket = join("-", [local.log_bucket_prefix, data.google_projects.log.projects.0.project_id]) + log_bucket = var.log_bucket } } @@ -142,4 +142,3 @@ resource "google_cloudbuild_trigger" "zip_files" { } } } - diff --git a/5-app-infra/modules/service_catalog/variables.tf b/5-app-infra/modules/service_catalog/variables.tf index df0cf433..0b1bbb60 100644 --- a/5-app-infra/modules/service_catalog/variables.tf +++ b/5-app-infra/modules/service_catalog/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,3 +42,13 @@ variable "machine_learning_project_number" { description = "Project Number for the Machine Learning (Vertex) Project" type = string } + +variable "kms_crypto_key" { + description = "KMS Key to be used" + type = string +} + +variable "log_bucket" { + description = "Bucket to store logs from service catalog bucket" + type = string +} diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf index 21072be3..366949d7 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf +++ b/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf @@ -18,7 +18,8 @@ locals { service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id service_catalog_repo_name = data.terraform_remote_state.projects_shared.outputs.service_catalog_repo_name machine_learning_project_number = data.terraform_remote_state.machine_learning_development.outputs.machine_learning_project_number - tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["bu3-service-catalog"] + tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["ml-service-catalog"] + shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings } data "terraform_remote_state" "projects_shared" { diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf index 35376c79..2a031be5 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf +++ b/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,23 @@ * limitations under the License. */ +locals { + region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] +} + +data "google_project" "common_svc_catalog" { + project_id = local.service_catalog_project_id +} + module "service_catalog" { - source = "../../../../modules/service_catalog" + source = "../../modules/service_catalog" project_id = local.service_catalog_project_id region = var.instance_region name = local.service_catalog_repo_name machine_learning_project_number = local.machine_learning_project_number tf_service_catalog_sa_email = local.tf_service_catalog_sa_email + + log_bucket = var.log_bucket + kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_svc_catalog.name}" } From 46b72a2f17d2d0f3c5d3f02decb8803a5300e01c Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 09:38:43 -0300 Subject: [PATCH 2/8] fix: add output on projects --- 4-projects/business_unit_3/shared/outputs.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/business_unit_3/shared/outputs.tf index 88c5d34c..ccd45012 100644 --- a/4-projects/business_unit_3/shared/outputs.tf +++ b/4-projects/business_unit_3/shared/outputs.tf @@ -92,3 +92,8 @@ output "artifacts_repo_id" { description = "ID of the Artifacts repository" value = module.ml_infra_projects.artifacts_repo_id } + +output "shared_level_keyrings" { + description = "Keyrings used on shared level project creation" + value = local.shared_kms_key_ring +} From 3176a9fe5fca60476b599d38545c65c9288dad36 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 11:08:29 -0300 Subject: [PATCH 3/8] adjust lint build --- .github/workflows/lint.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2d576017..b801fa7b 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -48,7 +48,7 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh env: diff --git a/Makefile b/Makefile index 215cbcae..7b546ecd 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker_test_lint: docker run --rm -it \ -e ENABLE_PARALLEL=0 \ -e DISABLE_TFLINT=1 \ - -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform" \ + -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared" \ -v $(CURDIR):/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh From 2db70246fa366db88c445c4a30858867d421a152 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 11:10:04 -0300 Subject: [PATCH 4/8] add modules instruction --- 5-app-infra/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/5-app-infra/README.md b/5-app-infra/README.md index 517bc177..fcc7f5df 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -330,6 +330,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh From 0b9bb52d0a4ca132feb1b150336634354e6c6aa8 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 11:16:34 -0300 Subject: [PATCH 5/8] docker generate docs --- 4-projects/business_unit_3/shared/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/business_unit_3/shared/README.md index b65cb8a9..e67eff55 100644 --- a/4-projects/business_unit_3/shared/README.md +++ b/4-projects/business_unit_3/shared/README.md @@ -34,6 +34,7 @@ | service\_catalog\_project\_id | Service Catalog Project ID. | | service\_catalog\_repo\_id | ID of the Service Catalog repository | | service\_catalog\_repo\_name | The name of the Service Catalog repository | +| shared\_level\_keyrings | Keyrings used on shared level project creation | | state\_buckets | GCS Buckets to store TF state | | terraform\_service\_accounts | APP Infra Pipeline Terraform Accounts. | From e94437602319686d1a2d65379eeab3edd0f9c24b Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 11:26:29 -0300 Subject: [PATCH 6/8] update workflows file --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b801fa7b..43766d12 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -54,4 +54,4 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared From fb66663ceacba74ebd78257c4e8bd740448d213c Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 12:41:37 -0300 Subject: [PATCH 7/8] fix: remove data source on artifact publish --- 5-app-infra/README.md | 1 + 5-app-infra/modules/publish_artifacts/data.tf | 18 +----------------- 5-app-infra/modules/publish_artifacts/main.tf | 6 +++--- .../modules/publish_artifacts/variables.tf | 5 +++++ .../shared/publish_artifacts.tf | 12 +++++++++++- .../business_unit_3/shared/remote.tf | 1 + 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/5-app-infra/README.md b/5-app-infra/README.md index fcc7f5df..0f7ac549 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -159,6 +159,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh diff --git a/5-app-infra/modules/publish_artifacts/data.tf b/5-app-infra/modules/publish_artifacts/data.tf index afd1ef09..4671af93 100644 --- a/5-app-infra/modules/publish_artifacts/data.tf +++ b/5-app-infra/modules/publish_artifacts/data.tf @@ -1,5 +1,5 @@ /** - * Copyright 2023 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,19 +17,3 @@ data "google_project" "project" { project_id = var.project_id } - -data "google_projects" "kms" { - filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE" - # filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} diff --git a/5-app-infra/modules/publish_artifacts/main.tf b/5-app-infra/modules/publish_artifacts/main.tf index c3fc48a6..141e747d 100644 --- a/5-app-infra/modules/publish_artifacts/main.tf +++ b/5-app-infra/modules/publish_artifacts/main.tf @@ -21,7 +21,7 @@ resource "google_project_service_identity" "artifact_registry_agent" { } resource "google_kms_crypto_key_iam_member" "artifact-kms-key-binding" { - crypto_key_id = data.google_kms_crypto_key.key.id + crypto_key_id = var.kms_crypto_key role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = "serviceAccount:${google_project_service_identity.artifact_registry_agent.email}" } @@ -40,7 +40,7 @@ resource "google_artifact_registry_repository" "repo" { #NIST 800-53: SC-12 SC-13 #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - kms_key_name = data.google_kms_crypto_key.key.id + kms_key_name = var.kms_crypto_key #Cleanup policy #Control ID: AR-CO-6.1 @@ -101,6 +101,7 @@ resource "google_cloudbuild_trigger" "docker_build" { repo_name = var.name } build { + timeout = "1800s" step { id = "unshallow" name = "gcr.io/cloud-builders/git" @@ -119,7 +120,6 @@ resource "google_cloudbuild_trigger" "docker_build" { <<-EOT changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r) changed_folders=$(echo "$changed_files" | awk -F/ '{print $2}' | sort | uniq ) - for folder in $changed_folders; do echo "Found docker folder: $folder" echo $folder >> /workspace/docker_build diff --git a/5-app-infra/modules/publish_artifacts/variables.tf b/5-app-infra/modules/publish_artifacts/variables.tf index 28e68618..18e8a528 100644 --- a/5-app-infra/modules/publish_artifacts/variables.tf +++ b/5-app-infra/modules/publish_artifacts/variables.tf @@ -69,3 +69,8 @@ variable "environment" { variable "project_id" { description = "Project ID" } + +variable "kms_crypto_key" { + description = "KMS Key to be used" + type = string +} diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf index 5983f92c..2d535968 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf +++ b/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf @@ -14,8 +14,16 @@ * limitations under the License. */ +locals { + region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] +} + +data "google_project" "common_artifacts" { + project_id = local.common_artifacts_project_id +} + module "artifact_publish" { - source = "../../../../modules/publish_artifacts" + source = "../../modules/publish_artifacts" environment = local.environment description = "Publish Artifacts for ML Projects" @@ -34,4 +42,6 @@ module "artifact_publish" { } ] }] + + kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_artifacts.name}" } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf index a9292473..c98f85b7 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf +++ b/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf @@ -18,6 +18,7 @@ locals { common_artifacts_project_id = data.terraform_remote_state.projects_shared.outputs.common_artifacts_project_id service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id artifacts_repo_name = data.terraform_remote_state.projects_shared.outputs.artifacts_repo_name + shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings } data "terraform_remote_state" "projects_shared" { From 07089547f71ad98072d07f24c2f6f9eb20a11e70 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Mon, 17 Jun 2024 14:05:42 -0300 Subject: [PATCH 8/8] add files to lint.yaml --- .github/workflows/lint.yaml | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 43766d12..ca36fb98 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -48,10 +48,10 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared diff --git a/Makefile b/Makefile index 7b546ecd..a06f9d1b 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker_test_lint: docker run --rm -it \ -e ENABLE_PARALLEL=0 \ -e DISABLE_TFLINT=1 \ - -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared" \ + -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared" \ -v $(CURDIR):/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh