From 9b223471557ac7e3ba1825287faf0c8c05871680 Mon Sep 17 00:00:00 2001 From: mariammartins Date: Fri, 14 Jun 2024 14:05:00 -0300 Subject: [PATCH 1/8] first commit --- 0-bootstrap/README-GitHub.md | 6 +- 0-bootstrap/README-Jenkins.md | 4 +- 0-bootstrap/README-Terraform-Cloud.md | 15 +-- 0-bootstrap/README.md | 2 +- 0-bootstrap/terraform_cloud.tf.example | 23 +--- 1-org/README.md | 2 +- 3-networks-dual-svpc/README.md | 2 +- 4-projects/README.md | 19 +-- .../development/README.md | 0 .../development}/backend.tf | 2 +- .../development}/backend.tf.cloud.example | 2 +- .../development/common.auto.tfvars | 0 .../development/development.auto.tfvars | 0 .../development}/locals.tf | 6 +- .../development/main.tf | 0 .../development/outputs.tf | 0 .../development/remote.tf | 0 .../development/variables.tf | 0 .../development/versions.tf | 0 .../non-production/README.md | 0 .../non-production/backend.tf | 2 +- .../non-production/backend.tf.cloud.example | 2 +- .../non-production/common.auto.tfvars | 0 .../non-production}/locals.tf | 6 +- .../non-production/main.tf | 0 .../non-production/non-production.auto.tfvars | 0 .../non-production/outputs.tf | 0 .../non-production/outputs.tf.backup | 0 .../non-production/remote.tf | 0 .../non-production/variables.tf | 0 .../production/README.md | 0 .../production}/backend.tf | 2 +- .../production}/backend.tf.cloud.example | 2 +- .../production/common.auto.tfvars | 0 .../production}/locals.tf | 6 +- .../production/main.tf | 0 .../production/outputs.tf | 0 .../production/outputs.tf.backup | 0 .../production/production.auto.tfvars | 0 .../production/remote.tf | 0 .../production/variables.tf | 0 .../shared/README.md | 0 .../shared/backend.tf | 2 +- .../shared/backend.tf.cloud.example | 2 +- .../shared/common.auto.tfvars | 0 .../shared/example_infra_pipeline.tf | 8 +- .../shared/ml_infra_projects.tf | 10 +- .../shared/outputs.tf | 0 .../shared/remote.tf | 0 .../shared/remote.tf.cloud.example | 0 .../shared/shared.auto.tfvars | 0 .../shared/variables.tf | 0 .../shared/versions.tf | 0 4-projects/modules/composer_env/variables.tf | 2 +- 4-projects/modules/env_folders/variables.tf | 2 +- 4-projects/modules/ml_env/example_vertex.tf | 6 +- 4-projects/modules/ml_env/variables.tf | 4 +- 5-app-infra/README.md | 66 +++++----- .../shared/README.md | 0 .../shared/backend.tf | 2 +- .../shared/common.auto.tfvars | 0 .../shared/locals.tf | 2 +- .../shared/outputs.tf | 0 .../shared/publish_artifacts.tf | 0 .../shared/remote.tf | 0 .../shared/variables.tf | 0 .../shared/versions.tf | 0 .../shared/README.md | 0 .../shared/backend.tf | 2 +- .../shared/locals.tf | 2 +- .../shared/outputs.tf | 0 .../shared/remote.tf | 2 +- .../shared/service_catalog.tf | 0 .../shared/variables.tf | 0 .../shared/versions.tf | 0 7-composer/dag.py | 2 +- .../dags | 2 +- 7-vertexpipeline/Readme.md | 2 +- 7-vertexpipeline/census_pipeline.ipynb | 14 +- 7-vertexpipeline/runpipeline.py | 6 +- ERRATA.md | 2 +- README.md | 2 +- build/tf-wrapper.sh | 24 ++-- docs/TROUBLESHOOTING.md | 2 +- .../shared/backend.tf.cloud.example | 2 +- .../modules/env_folders/variables.tf | 2 +- .../4-projects/modules/ml_env/variables.tf | 4 +- docs/change_resource_hierarchy.md | 14 +- examples/machine-learning-pipeline/README.md | 120 +++++++++--------- .../development/README.md | 0 .../development}/backend.tf | 2 +- .../development/common.auto.tfvars | 0 .../development/locals.tf | 4 +- .../development/main.tf | 0 .../development/outputs.tf | 0 .../development/remote.tf | 0 .../development/variables.tf | 0 .../development/versions.tf | 0 .../non-production/README.md | 0 .../non-production/backend.tf | 2 +- .../non-production/common.auto.tfvars | 0 .../non-production/locals.tf | 4 +- .../non-production/main.tf | 0 .../non-production/outputs.tf | 0 .../non-production/remote.tf | 0 .../non-production/variables.tf | 0 .../non-production/versions.tf | 0 .../production/README.md | 0 .../production}/backend.tf | 2 +- .../production/common.auto.tfvars | 0 .../production/locals.tf | 4 +- .../production/main.tf | 0 .../production/outputs.tf | 0 .../production/remote.tf | 0 .../production/variables.tf | 0 .../production/versions.tf | 0 .../modules/base_env/variables.tf | 2 +- helpers/foundation-deployer/README.md | 2 +- .../foundation-deployer/global.tfvars.example | 2 +- helpers/foundation-deployer/main.go | 8 +- helpers/foundation-deployer/stages/apply.go | 6 +- helpers/foundation-deployer/stages/data.go | 8 +- helpers/foundation-deployer/stages/destroy.go | 4 +- test/disable_tf_files.sh | 31 ++--- test/integration/app-infra/app_infra_test.go | 10 +- .../projects-shared/projects_shared_test.go | 11 +- test/integration/projects/projects_test.go | 39 ++---- test/restore_tf_files.sh | 79 +++++------- 128 files changed, 280 insertions(+), 353 deletions(-) rename 4-projects/{business_unit_3 => ml_business_unit}/development/README.md (100%) rename 4-projects/{business_unit_3/production => ml_business_unit/development}/backend.tf (91%) rename 4-projects/{business_unit_3/production => ml_business_unit/development}/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/common.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/development.auto.tfvars (100%) rename 4-projects/{business_unit_3/non-production => ml_business_unit/development}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/versions.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/README.md (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/backend.tf (91%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/backend.tf.cloud.example (94%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/common.auto.tfvars (100%) rename 4-projects/{business_unit_3/production => ml_business_unit/non-production}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/non-production.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/outputs.tf.backup (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/README.md (100%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/backend.tf (91%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/common.auto.tfvars (100%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/outputs.tf.backup (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/production.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/README.md (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/backend.tf (89%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/common.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/example_infra_pipeline.tf (96%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/ml_infra_projects.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/remote.tf.cloud.example (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/shared.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/README.md (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/backend.tf (91%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/common.auto.tfvars (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/locals.tf (94%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/publish_artifacts.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/remote.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/variables.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/README.md (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/backend.tf (91%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/locals.tf (94%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/remote.tf (95%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/service_catalog.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/variables.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3/production => ml_business_unit/development}/backend.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/versions.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/backend.tf (90%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/versions.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3/development => ml_business_unit/production}/backend.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/versions.tf (100%) diff --git a/0-bootstrap/README-GitHub.md b/0-bootstrap/README-GitHub.md index f7e635e4..cdd6e4d2 100644 --- a/0-bootstrap/README-GitHub.md +++ b/0-bootstrap/README-GitHub.md @@ -843,8 +843,8 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from bootstrap output. @@ -862,7 +862,7 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th git commit -m 'Initialize projects repo' ``` -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. 1. Use `terraform output` to get the CI/CD project ID and the projects step Terraform Service Account from gcp-bootstrap output. 1. The CI/CD project ID will be used in the [validation](https://cloud.google.com/docs/terraform/policy-validation/quickstart) of the Terraform configuration diff --git a/0-bootstrap/README-Jenkins.md b/0-bootstrap/README-Jenkins.md index 77e081ef..b840094d 100644 --- a/0-bootstrap/README-Jenkins.md +++ b/0-bootstrap/README-Jenkins.md @@ -872,8 +872,8 @@ Here you will configure a VPN Network tunnel to enable connectivity between the mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from 0-bootstrap output. ```bash diff --git a/0-bootstrap/README-Terraform-Cloud.md b/0-bootstrap/README-Terraform-Cloud.md index 935f3e78..a7338b3d 100644 --- a/0-bootstrap/README-Terraform-Cloud.md +++ b/0-bootstrap/README-Terraform-Cloud.md @@ -771,16 +771,15 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. 1. In order to manually run the apply for shared workspace from your local we need to temporary unset the TFC backend by renaming `envs/shared/backend.tf` to `envs/shared/backend.tf.temporary_disabled`. ```bash - mv business_unit_1/shared/backend.tf business_unit_1/shared/backend.tf.temporary_disabled - mv business_unit_2/shared/backend.tf business_unit_2/shared/backend.tf.temporary_disabled + mv ml_business_unit/shared/backend.tf ml_business_unit/shared/backend.tf.temporary_disabled ``` 1. Use `terraform output` to get the CI/CD project ID and the projects step Terraform Service Account from gcp-bootstrap output. @@ -831,10 +830,8 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th 1. In order to set the TFC backend for shared workspace we now can rename `envs/shared/backend.tf.temporary_disabled` to `envs/shared/backend.tf` and run `terraform init`. When you're prompted, agree to copy Terraform state to Terraform Cloud. ```bash - mv business_unit_1/shared/backend.tf.temporary_disabled business_unit_1/shared/backend.tf - mv business_unit_2/shared/backend.tf.temporary_disabled business_unit_2/shared/backend.tf - terraform -chdir="business_unit_1/shared/" init - terraform -chdir="business_unit_2/shared/" init + mv ml_business_unit/shared/backend.tf.temporary_disabled ml_business_unit/shared/backend.tf + terraform -chdir="ml_business_unit/shared/" init ``` 1. Commit changes diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 7048283f..17bfbf64 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the business unit projects using the infra pipeline setup in 4-projects. +Deploy a Compute Engine instance in one of the machine learning projects using the infra pipeline setup in 4-projects. diff --git a/0-bootstrap/terraform_cloud.tf.example b/0-bootstrap/terraform_cloud.tf.example index ce45ab1a..3aa7b54e 100644 --- a/0-bootstrap/terraform_cloud.tf.example +++ b/0-bootstrap/terraform_cloud.tf.example @@ -63,14 +63,10 @@ locals { "3-shared" = { vcs_branch = "production", directory = "/envs/shared" }, }, "proj" = { - "4-bu1-production" = { vcs_branch = "production", directory = "/business_unit_1/production" }, - "4-bu1-non-production" = { vcs_branch = "non-production", directory = "/business_unit_1/non-production" }, - "4-bu1-development" = { vcs_branch = "development", directory = "/business_unit_1/development" }, - "4-bu1-shared" = { vcs_branch = "production", directory = "/business_unit_1/shared" }, - "4-bu2-production" = { vcs_branch = "production", directory = "/business_unit_2/production" }, - "4-bu2-non-production" = { vcs_branch = "non-production", directory = "/business_unit_2/non-production" }, - "4-bu2-development" = { vcs_branch = "development", directory = "/business_unit_2/development" }, - "4-bu2-shared" = { vcs_branch = "production", directory = "/business_unit_2/shared" }, + "4-ml-production" = { vcs_branch = "production", directory = "/ml_business_unit/production" }, + "4-ml-non-production" = { vcs_branch = "non-production", directory = "/ml_business_unit/non-production" }, + "4-ml-development" = { vcs_branch = "development", directory = "/ml_business_unit/development" }, + "4-ml-shared" = { vcs_branch = "production", directory = "/ml_business_unit/shared" }, }, } @@ -218,14 +214,9 @@ resource "tfe_run_trigger" "networks_shared_production" { sourceable_id = tfe_workspace.main["3-shared"].id } -resource "tfe_run_trigger" "projects_bu1_shared_production" { - workspace_id = tfe_workspace.main["4-bu1-production"].id - sourceable_id = tfe_workspace.main["4-bu1-shared"].id -} - -resource "tfe_run_trigger" "projects_bu2_shared_production" { - workspace_id = tfe_workspace.main["4-bu2-production"].id - sourceable_id = tfe_workspace.main["4-bu2-shared"].id +resource "tfe_run_trigger" "projects_ml_shared_production" { + workspace_id = tfe_workspace.main["4-ml-production"].id + sourceable_id = tfe_workspace.main["4-ml-shared"].id } module "tfc_cicd" { diff --git a/1-org/README.md b/1-org/README.md index d528b9db..0b11ea41 100644 --- a/1-org/README.md +++ b/1-org/README.md @@ -45,7 +45,7 @@ hub-and-spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy a Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. diff --git a/3-networks-dual-svpc/README.md b/3-networks-dual-svpc/README.md index cb47dcf7..8db18575 100644 --- a/3-networks-dual-svpc/README.md +++ b/3-networks-dual-svpc/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. diff --git a/4-projects/README.md b/4-projects/README.md index f5ff9c88..9051c974 100644 --- a/4-projects/README.md +++ b/4-projects/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. @@ -57,10 +57,10 @@ For an overview of the architecture and the parts, see the The purpose of this step is to set up the folder structure, projects, and infrastructure pipelines for applications that are connected as service projects to the shared VPC created in the previous stage. -For each business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. +For machine learning business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. This step follows the same [conventions](https://github.com/terraform-google-modules/terraform-google-enterprise-genai#branching-strategy) as the Foundation pipeline deployed in [0-bootstrap](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/0-bootstrap/README.md). -A custom [workspace](https://github.com/terraform-google-modules/terraform-google-bootstrap/blob/master/modules/tf_cloudbuild_workspace/README.md) (`bu1-example-app`) is created by this pipeline and necessary roles are granted to the Terraform Service Account of this workspace by enabling variable `sa_roles` as shown in this [example](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/modules/base_env/example_base_shared_vpc_project.tf). +A custom [workspace](https://github.com/terraform-google-modules/terraform-google-bootstrap/blob/master/modules/tf_cloudbuild_workspace/README.md) (`ml-example-app`) is created by this pipeline and necessary roles are granted to the Terraform Service Account of this workspace by enabling variable `sa_roles` as shown in this [example](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/modules/base_env/example_base_shared_vpc_project.tf). This pipeline is utilized to deploy resources in projects across development/non-production/production in step [5-app-infra](../5-app-infra/README.md). Other Workspaces can also be created to isolate deployments if needed. @@ -122,8 +122,9 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](./business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. + +1. See any of the shared folder [README.md](./ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from 0-bootstrap output. @@ -141,8 +142,10 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get git commit -m 'Initialize projects repo' ``` -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. + 1. To use the `validate` option of the `tf-wrapper.sh` script, please follow the [instructions](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) to install the terraform-tools component. + 1. Use `terraform output` to get the Cloud Build project ID and the projects step Terraform Service Account from 0-bootstrap output. An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set using the Terraform Service Account to enable impersonation. ```bash @@ -253,8 +256,8 @@ See `0-bootstrap` [README-GitHub.md](../0-bootstrap/README-GitHub.md#deploying-s mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. - See any of the shared folder [README.md](./business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. + See any of the shared folder [README.md](./ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. Use `terraform output` to get the remote state bucket (the backend bucket used by previous steps) value from `0-bootstrap` output. ```bash diff --git a/4-projects/business_unit_3/development/README.md b/4-projects/ml_business_unit/development/README.md similarity index 100% rename from 4-projects/business_unit_3/development/README.md rename to 4-projects/ml_business_unit/development/README.md diff --git a/4-projects/business_unit_3/production/backend.tf b/4-projects/ml_business_unit/development/backend.tf similarity index 91% rename from 4-projects/business_unit_3/production/backend.tf rename to 4-projects/ml_business_unit/development/backend.tf index c035921a..e5ccedf9 100644 --- a/4-projects/business_unit_3/production/backend.tf +++ b/4-projects/ml_business_unit/development/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/production" + prefix = "terraform/projects/ml_business_unit/development" } } diff --git a/4-projects/business_unit_3/production/backend.tf.cloud.example b/4-projects/ml_business_unit/development/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/production/backend.tf.cloud.example rename to 4-projects/ml_business_unit/development/backend.tf.cloud.example index 23cd6e71..c09ed6d6 100644 --- a/4-projects/business_unit_3/production/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/development/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-production" + name = "4-ml-development" } } } diff --git a/4-projects/business_unit_3/development/common.auto.tfvars b/4-projects/ml_business_unit/development/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/development/common.auto.tfvars rename to 4-projects/ml_business_unit/development/common.auto.tfvars diff --git a/4-projects/business_unit_3/development/development.auto.tfvars b/4-projects/ml_business_unit/development/development.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/development/development.auto.tfvars rename to 4-projects/ml_business_unit/development/development.auto.tfvars diff --git a/4-projects/business_unit_3/non-production/locals.tf b/4-projects/ml_business_unit/development/locals.tf similarity index 86% rename from 4-projects/business_unit_3/non-production/locals.tf rename to 4-projects/ml_business_unit/development/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/non-production/locals.tf +++ b/4-projects/ml_business_unit/development/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/development/main.tf b/4-projects/ml_business_unit/development/main.tf similarity index 100% rename from 4-projects/business_unit_3/development/main.tf rename to 4-projects/ml_business_unit/development/main.tf diff --git a/4-projects/business_unit_3/development/outputs.tf b/4-projects/ml_business_unit/development/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/development/outputs.tf rename to 4-projects/ml_business_unit/development/outputs.tf diff --git a/4-projects/business_unit_3/development/remote.tf b/4-projects/ml_business_unit/development/remote.tf similarity index 100% rename from 4-projects/business_unit_3/development/remote.tf rename to 4-projects/ml_business_unit/development/remote.tf diff --git a/4-projects/business_unit_3/development/variables.tf b/4-projects/ml_business_unit/development/variables.tf similarity index 100% rename from 4-projects/business_unit_3/development/variables.tf rename to 4-projects/ml_business_unit/development/variables.tf diff --git a/4-projects/business_unit_3/development/versions.tf b/4-projects/ml_business_unit/development/versions.tf similarity index 100% rename from 4-projects/business_unit_3/development/versions.tf rename to 4-projects/ml_business_unit/development/versions.tf diff --git a/4-projects/business_unit_3/non-production/README.md b/4-projects/ml_business_unit/non-production/README.md similarity index 100% rename from 4-projects/business_unit_3/non-production/README.md rename to 4-projects/ml_business_unit/non-production/README.md diff --git a/4-projects/business_unit_3/non-production/backend.tf b/4-projects/ml_business_unit/non-production/backend.tf similarity index 91% rename from 4-projects/business_unit_3/non-production/backend.tf rename to 4-projects/ml_business_unit/non-production/backend.tf index 3872830c..0adc3933 100644 --- a/4-projects/business_unit_3/non-production/backend.tf +++ b/4-projects/ml_business_unit/non-production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/non-production" + prefix = "terraform/projects/ml_business_unit/non-production" } } diff --git a/4-projects/business_unit_3/non-production/backend.tf.cloud.example b/4-projects/ml_business_unit/non-production/backend.tf.cloud.example similarity index 94% rename from 4-projects/business_unit_3/non-production/backend.tf.cloud.example rename to 4-projects/ml_business_unit/non-production/backend.tf.cloud.example index ccefb79c..c6fa4995 100644 --- a/4-projects/business_unit_3/non-production/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/non-production/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-non-production" + name = "4-ml-non-production" } } } diff --git a/4-projects/business_unit_3/non-production/common.auto.tfvars b/4-projects/ml_business_unit/non-production/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/non-production/common.auto.tfvars rename to 4-projects/ml_business_unit/non-production/common.auto.tfvars diff --git a/4-projects/business_unit_3/production/locals.tf b/4-projects/ml_business_unit/non-production/locals.tf similarity index 86% rename from 4-projects/business_unit_3/production/locals.tf rename to 4-projects/ml_business_unit/non-production/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/production/locals.tf +++ b/4-projects/ml_business_unit/non-production/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/non-production/main.tf b/4-projects/ml_business_unit/non-production/main.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/main.tf rename to 4-projects/ml_business_unit/non-production/main.tf diff --git a/4-projects/business_unit_3/non-production/non-production.auto.tfvars b/4-projects/ml_business_unit/non-production/non-production.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/non-production/non-production.auto.tfvars rename to 4-projects/ml_business_unit/non-production/non-production.auto.tfvars diff --git a/4-projects/business_unit_3/non-production/outputs.tf b/4-projects/ml_business_unit/non-production/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/outputs.tf rename to 4-projects/ml_business_unit/non-production/outputs.tf diff --git a/4-projects/business_unit_3/non-production/outputs.tf.backup b/4-projects/ml_business_unit/non-production/outputs.tf.backup similarity index 100% rename from 4-projects/business_unit_3/non-production/outputs.tf.backup rename to 4-projects/ml_business_unit/non-production/outputs.tf.backup diff --git a/4-projects/business_unit_3/non-production/remote.tf b/4-projects/ml_business_unit/non-production/remote.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/remote.tf rename to 4-projects/ml_business_unit/non-production/remote.tf diff --git a/4-projects/business_unit_3/non-production/variables.tf b/4-projects/ml_business_unit/non-production/variables.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/variables.tf rename to 4-projects/ml_business_unit/non-production/variables.tf diff --git a/4-projects/business_unit_3/production/README.md b/4-projects/ml_business_unit/production/README.md similarity index 100% rename from 4-projects/business_unit_3/production/README.md rename to 4-projects/ml_business_unit/production/README.md diff --git a/4-projects/business_unit_3/development/backend.tf b/4-projects/ml_business_unit/production/backend.tf similarity index 91% rename from 4-projects/business_unit_3/development/backend.tf rename to 4-projects/ml_business_unit/production/backend.tf index d22c4775..f900eb0c 100644 --- a/4-projects/business_unit_3/development/backend.tf +++ b/4-projects/ml_business_unit/production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/development" + prefix = "terraform/projects/ml_business_unit/production" } } diff --git a/4-projects/business_unit_3/development/backend.tf.cloud.example b/4-projects/ml_business_unit/production/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/development/backend.tf.cloud.example rename to 4-projects/ml_business_unit/production/backend.tf.cloud.example index e20a0d82..704da6c2 100644 --- a/4-projects/business_unit_3/development/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/production/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-development" + name = "4-ml-production" } } } diff --git a/4-projects/business_unit_3/production/common.auto.tfvars b/4-projects/ml_business_unit/production/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/production/common.auto.tfvars rename to 4-projects/ml_business_unit/production/common.auto.tfvars diff --git a/4-projects/business_unit_3/development/locals.tf b/4-projects/ml_business_unit/production/locals.tf similarity index 86% rename from 4-projects/business_unit_3/development/locals.tf rename to 4-projects/ml_business_unit/production/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/development/locals.tf +++ b/4-projects/ml_business_unit/production/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/production/main.tf b/4-projects/ml_business_unit/production/main.tf similarity index 100% rename from 4-projects/business_unit_3/production/main.tf rename to 4-projects/ml_business_unit/production/main.tf diff --git a/4-projects/business_unit_3/production/outputs.tf b/4-projects/ml_business_unit/production/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/production/outputs.tf rename to 4-projects/ml_business_unit/production/outputs.tf diff --git a/4-projects/business_unit_3/production/outputs.tf.backup b/4-projects/ml_business_unit/production/outputs.tf.backup similarity index 100% rename from 4-projects/business_unit_3/production/outputs.tf.backup rename to 4-projects/ml_business_unit/production/outputs.tf.backup diff --git a/4-projects/business_unit_3/production/production.auto.tfvars b/4-projects/ml_business_unit/production/production.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/production/production.auto.tfvars rename to 4-projects/ml_business_unit/production/production.auto.tfvars diff --git a/4-projects/business_unit_3/production/remote.tf b/4-projects/ml_business_unit/production/remote.tf similarity index 100% rename from 4-projects/business_unit_3/production/remote.tf rename to 4-projects/ml_business_unit/production/remote.tf diff --git a/4-projects/business_unit_3/production/variables.tf b/4-projects/ml_business_unit/production/variables.tf similarity index 100% rename from 4-projects/business_unit_3/production/variables.tf rename to 4-projects/ml_business_unit/production/variables.tf diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/ml_business_unit/shared/README.md similarity index 100% rename from 4-projects/business_unit_3/shared/README.md rename to 4-projects/ml_business_unit/shared/README.md diff --git a/4-projects/business_unit_3/shared/backend.tf b/4-projects/ml_business_unit/shared/backend.tf similarity index 89% rename from 4-projects/business_unit_3/shared/backend.tf rename to 4-projects/ml_business_unit/shared/backend.tf index 997695e5..130bb23c 100644 --- a/4-projects/business_unit_3/shared/backend.tf +++ b/4-projects/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/shared" + prefix = "terraform/projects/ml_business_unit/shared" } } diff --git a/4-projects/business_unit_3/shared/backend.tf.cloud.example b/4-projects/ml_business_unit/shared/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/shared/backend.tf.cloud.example rename to 4-projects/ml_business_unit/shared/backend.tf.cloud.example index 5dfb05be..2f8429e2 100644 --- a/4-projects/business_unit_3/shared/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/shared/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu2-shared" + name = "4-ml-shared" } } } diff --git a/4-projects/business_unit_3/shared/common.auto.tfvars b/4-projects/ml_business_unit/shared/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/shared/common.auto.tfvars rename to 4-projects/ml_business_unit/shared/common.auto.tfvars diff --git a/4-projects/business_unit_3/shared/example_infra_pipeline.tf b/4-projects/ml_business_unit/shared/example_infra_pipeline.tf similarity index 96% rename from 4-projects/business_unit_3/shared/example_infra_pipeline.tf rename to 4-projects/ml_business_unit/shared/example_infra_pipeline.tf index 0c1a33dd..476c1191 100644 --- a/4-projects/business_unit_3/shared/example_infra_pipeline.tf +++ b/4-projects/ml_business_unit/shared/example_infra_pipeline.tf @@ -16,9 +16,9 @@ locals { repo_names = [ - "bu3-artifact-publish", - "bu3-service-catalog", - "bu3-machine-learning", + "ml-artifact-publish", + "ml-service-catalog", + "ml-machine-learning", ] } @@ -50,7 +50,7 @@ module "app_infra_cloudbuild_project" { billing_code = "1234" primary_contact = "example@example.com" secondary_contact = "example2@example.com" - business_code = "bu3" + business_code = "ml" } module "infra_pipelines" { diff --git a/4-projects/business_unit_3/shared/ml_infra_projects.tf b/4-projects/ml_business_unit/shared/ml_infra_projects.tf similarity index 86% rename from 4-projects/business_unit_3/shared/ml_infra_projects.tf rename to 4-projects/ml_business_unit/shared/ml_infra_projects.tf index 431b0c2a..6ec434b0 100644 --- a/4-projects/business_unit_3/shared/ml_infra_projects.tf +++ b/4-projects/ml_business_unit/shared/ml_infra_projects.tf @@ -14,23 +14,23 @@ * limitations under the License. */ -module "ml_infra_projects" { - source = "../../modules/ml_infra_projects" +module "_infra_projects" { + source = "../../modules/_infra_projects" org_id = local.org_id folder_id = local.common_folder_name billing_account = local.billing_account environment = "common" key_rings = local.shared_kms_key_ring - business_code = "bu3" + business_code = "ml" billing_code = "1234" primary_contact = "example@example.com" secondary_contact = "example2@example.com" cloud_source_artifacts_repo_name = var.cloud_source_artifacts_repo_name cloud_source_service_catalog_repo_name = var.cloud_source_service_catalog_repo_name remote_state_bucket = var.remote_state_bucket - artifacts_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["bu3-artifact-publish"] - service_catalog_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"] + artifacts_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["ml-artifact-publish"] + service_catalog_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["ml-service-catalog"] environment_kms_project_id = "" prevent_destroy = var.prevent_destroy } diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/ml_business_unit/shared/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/shared/outputs.tf rename to 4-projects/ml_business_unit/shared/outputs.tf diff --git a/4-projects/business_unit_3/shared/remote.tf b/4-projects/ml_business_unit/shared/remote.tf similarity index 100% rename from 4-projects/business_unit_3/shared/remote.tf rename to 4-projects/ml_business_unit/shared/remote.tf diff --git a/4-projects/business_unit_3/shared/remote.tf.cloud.example b/4-projects/ml_business_unit/shared/remote.tf.cloud.example similarity index 100% rename from 4-projects/business_unit_3/shared/remote.tf.cloud.example rename to 4-projects/ml_business_unit/shared/remote.tf.cloud.example diff --git a/4-projects/business_unit_3/shared/shared.auto.tfvars b/4-projects/ml_business_unit/shared/shared.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/shared/shared.auto.tfvars rename to 4-projects/ml_business_unit/shared/shared.auto.tfvars diff --git a/4-projects/business_unit_3/shared/variables.tf b/4-projects/ml_business_unit/shared/variables.tf similarity index 100% rename from 4-projects/business_unit_3/shared/variables.tf rename to 4-projects/ml_business_unit/shared/variables.tf diff --git a/4-projects/business_unit_3/shared/versions.tf b/4-projects/ml_business_unit/shared/versions.tf similarity index 100% rename from 4-projects/business_unit_3/shared/versions.tf rename to 4-projects/ml_business_unit/shared/versions.tf diff --git a/4-projects/modules/composer_env/variables.tf b/4-projects/modules/composer_env/variables.tf index 2a20f9c3..867bb69c 100644 --- a/4-projects/modules/composer_env/variables.tf +++ b/4-projects/modules/composer_env/variables.tf @@ -105,6 +105,6 @@ variable "shared_kms_key_ring" { } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/4-projects/modules/env_folders/variables.tf b/4-projects/modules/env_folders/variables.tf index 2297d829..3661fa96 100644 --- a/4-projects/modules/env_folders/variables.tf +++ b/4-projects/modules/env_folders/variables.tf @@ -15,7 +15,7 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } diff --git a/4-projects/modules/ml_env/example_vertex.tf b/4-projects/modules/ml_env/example_vertex.tf index 547bc932..0c1bd6e9 100644 --- a/4-projects/modules/ml_env/example_vertex.tf +++ b/4-projects/modules/ml_env/example_vertex.tf @@ -59,7 +59,7 @@ module "machine_learning_project" { // Map for the roles where the key is the repository name ("${var.business_code}-example-app") // and the value is the list of roles that this SA need to deploy step 5-app-infra sa_roles = { - "bu3-machine-learning" = [ + "ml-machine-learning" = [ "roles/aiplatform.admin", "roles/artifactregistry.admin", "roles/bigquery.admin", @@ -162,7 +162,7 @@ resource "google_kms_crypto_key_iam_member" "kms_admin" { for_each = module.machine_learning_project.kms_keys crypto_key_id = each.value.id role = "roles/cloudkms.admin" - member = "serviceAccount:${local.app_infra_pipeline_service_accounts["bu3-machine-learning"]}" + member = "serviceAccount:${local.app_infra_pipeline_service_accounts["ml-machine-learning"]}" } // Add crypto key viewer role to kms environment project @@ -212,7 +212,7 @@ resource "google_sourcerepo_repository_iam_member" "read" { project = local.service_catalog_project_id repository = local.service_catalog_repo_name role = "roles/viewer" - member = "serviceAccount:${local.app_infra_pipeline_service_accounts["bu3-machine-learning"]}" + member = "serviceAccount:${local.app_infra_pipeline_service_accounts["ml-machine-learning"]}" } // Add Browser Role to CloudBuild at Env Folder diff --git a/4-projects/modules/ml_env/variables.tf b/4-projects/modules/ml_env/variables.tf index b30f7c90..ad1509e2 100644 --- a/4-projects/modules/ml_env/variables.tf +++ b/4-projects/modules/ml_env/variables.tf @@ -15,12 +15,12 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/5-app-infra/README.md b/5-app-infra/README.md index 517bc177..cf4b2d5b 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -84,7 +84,7 @@ Clone the repo at the same level of the `terraform-google-enterprise-genai` fold Run `terraform output cloudbuild_project_id` in the `4-projects` folder to get the Cloud Build Project ID. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} gcloud source repos clone gcp-policies gcp-policies-app-infra --project=${INFRA_PIPELINE_PROJECT_ID} @@ -141,21 +141,21 @@ each folder under `images` has the full name and tag of the image that must be b Once pushed, the pipeline build logs can be accessed by navigating to the artifacts project name created in step-4: ```bash - terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw common_artifacts_project_id + terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw common_artifacts_project_id ``` -1. Clone the `bu3-artifact-publish` repo. +1. Clone the `ml-artifact-publish` repo. ```bash - gcloud source repos clone bu3-artifact-publish --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-artifact-publish --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of genAI to new repo. - All subsequent steps assume you are running them from the bu3-artifact-publish directory. + All subsequent steps assume you are running them from the ml-artifact-publish directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-artifact-publish/ + cd ml-artifact-publish/ git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ . @@ -170,7 +170,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa mv common.auto.example.tfvars common.auto.tfvars ``` -1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. +1. Update the file with values from your environment and 0-bootstrap. See machine learning business unit env folder [README.md](./ml_business_unit/production/README.md) file for additional information on the values in the `common.auto.tfvars` file. ```bash export remote_state_bucket=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw projects_gcs_bucket_tfstate) @@ -181,7 +181,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-artifact-publish"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-artifact-publish"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -210,7 +210,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa git push origin production ``` -1. `cd` out of the `bu3-artifacts-publish` repository. +1. `cd` out of the `ml-artifacts-publish` repository. ```bash cd .. @@ -223,7 +223,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa 1. Grab the Artifact Project ID ```bash - export ARTIFACT_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared" output -raw common_artifacts_project_id) + export ARTIFACT_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared" output -raw common_artifacts_project_id) echo ${ARTIFACT_PROJECT_ID} ``` @@ -272,7 +272,7 @@ The resoning behind utilizing one repository with two deployment methodologies i The repository has the structure (truncated for brevity): ``` - business_unit_3 + ml_business_unit ├── development ├── non-production ├── production @@ -315,18 +315,18 @@ This pipeline is listening to the `main` branch of this repository for changes i The pipeline also listens for changes made to `plan`, `development`, `non-production` & `production` branches, this is used for deploying infrastructure to each project. -1. Clone the `bu3-service-catalog` repo. +1. Clone the `ml-service-catalog` repo. ```bash - gcloud source repos clone bu3-service-catalog --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-service-catalog --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of foundation to new repo. - All subsequent steps assume you are running them from the bu3-service-catalog directory. + All subsequent steps assume you are running them from the ml-service-catalog directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-service-catalog + cd ml-service-catalog git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ . @@ -341,7 +341,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc mv common.auto.example.tfvars common.auto.tfvars ``` -1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. +1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. ```bash export remote_state_bucket=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw projects_gcs_bucket_tfstate) @@ -352,7 +352,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-service-catalog"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-service-catalog"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -381,7 +381,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc git push origin production ``` -1. `cd` out of the `bu3-service-catalog` repository. +1. `cd` out of the `ml-service-catalog` repository. ```bash cd .. @@ -392,7 +392,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Grab the Service Catalogs ID ```bash - export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared" output -raw service_catalog_project_id) + export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared" output -raw service_catalog_project_id) echo ${SERVICE_CATALOG_PROJECT_ID} ``` @@ -456,7 +456,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -464,10 +464,10 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-artifact-publish"' --raw-output) + terraform_sa=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-artifact-publish"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -476,7 +476,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-artifact-publish"' --raw-output) + export backend_bucket=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-artifact-publish"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -490,10 +490,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-artifact-publish"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-artifact-publish"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -539,7 +539,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT 1. Grab the Artifact Project ID ```bash - export ARTIFACT_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/business_unit_3/shared" output -raw common_artifacts_project_id) + export ARTIFACT_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/ml_business_unit/shared" output -raw common_artifacts_project_id) echo ${ARTIFACT_PROJECT_ID} ``` @@ -600,7 +600,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -608,10 +608,10 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-service-catalog"' --raw-output) + terraform_sa=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-service-catalog"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -620,7 +620,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-service-catalog"' --raw-output) + export backend_bucket=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-service-catalog"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -634,10 +634,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-service-catalog"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-service-catalog"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -675,7 +675,7 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Grab the Service Catalogs ID ```bash - export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/business_unit_3/shared" output -raw service_catalog_project_id) + export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/ml_business_unit/shared" output -raw service_catalog_project_id) echo ${SERVICE_CATALOG_PROJECT_ID} ``` diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/README.md b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/README.md similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/README.md rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/README.md diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf similarity index 91% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf index 023a3c07..0b671313 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf +++ b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/shared" + prefix = "terraform/app-infra/ml_business_unit/shared" } } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/common.auto.tfvars b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/common.auto.tfvars similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/common.auto.tfvars rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/common.auto.tfvars diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf similarity index 94% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf index d708d36e..d049424c 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf +++ b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf @@ -15,6 +15,6 @@ */ locals { - business_unit = "business_unit_3" + business_unit = "ml_business_unit" environment = "common" } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/outputs.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/outputs.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/outputs.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/outputs.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/publish_artifacts.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/publish_artifacts.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/remote.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/remote.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/variables.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/variables.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/variables.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/variables.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/versions.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/versions.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/versions.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/versions.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/README.md b/5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/README.md rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf similarity index 91% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf index 023a3c07..0b671313 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/shared" + prefix = "terraform/app-infra/ml_business_unit/shared" } } diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf similarity index 94% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf index e4a9236a..09352bd6 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf @@ -15,7 +15,7 @@ */ locals { - business_unit = "business_unit_3" + business_unit = "ml_business_unit" environment = "common" } diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/outputs.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/outputs.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/outputs.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/outputs.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/remote.tf similarity index 95% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/remote.tf index 21072be3..23bbe8e6 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/remote.tf @@ -18,7 +18,7 @@ locals { service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id service_catalog_repo_name = data.terraform_remote_state.projects_shared.outputs.service_catalog_repo_name machine_learning_project_number = data.terraform_remote_state.machine_learning_development.outputs.machine_learning_project_number - tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["bu3-service-catalog"] + tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["ml-service-catalog"] } data "terraform_remote_state" "projects_shared" { diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/service_catalog.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/service_catalog.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/variables.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/variables.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/variables.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/variables.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/versions.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/versions.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/versions.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/versions.tf diff --git a/7-composer/dag.py b/7-composer/dag.py index 949fb8dc..40260767 100644 --- a/7-composer/dag.py +++ b/7-composer/dag.py @@ -36,7 +36,7 @@ RUNNER = "DataflowRunner" REGION = "us-central1" JOB_NAME = "census-ingest-composer" -default_kms_key_name = "projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning" +default_kms_key_name = "projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning" deployment_image = "us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-8:latest" service_account = "728034955955-compute@developer.gserviceaccount.com" prod_service_account = "728034955955-compute@developer.gserviceaccount.com" diff --git a/7-composer/us-central1-test-census-034e6abc-bucket/dags b/7-composer/us-central1-test-census-034e6abc-bucket/dags index 80d289fb..ba51813b 100644 --- a/7-composer/us-central1-test-census-034e6abc-bucket/dags +++ b/7-composer/us-central1-test-census-034e6abc-bucket/dags @@ -12,7 +12,7 @@ EVAL_TABLE_ID = 'census_eval_table_composer' RUNNER = "DataflowRunner" REGION = "us-central1" JOB_NAME = "census-ingest-composer" -default_kms_key_name="projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning" +default_kms_key_name="projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning" default_args = { 'owner' : 'airflow', diff --git a/7-vertexpipeline/Readme.md b/7-vertexpipeline/Readme.md index cc157d92..ab5fd41c 100644 --- a/7-vertexpipeline/Readme.md +++ b/7-vertexpipeline/Readme.md @@ -127,7 +127,7 @@ The following method runs the pipeline. Note that a kms encryption key is suppli display_name=f"census_income_{timestamp}", template_path='./common/vertex-ai-pipeline/pipeline_package.yaml', pipeline_root=pipelineroot, - encryption_spec_key_name='projects/prj-d-kms-ui2h/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning', + encryption_spec_key_name='projects/prj-d-kms-ui2h/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning', parameter_values={ "create_bq_dataset_query": create_bq_dataset_query, "bq_dataset": data_config['bq_dataset'], diff --git a/7-vertexpipeline/census_pipeline.ipynb b/7-vertexpipeline/census_pipeline.ipynb index 9412fc12..1e10de64 100644 --- a/7-vertexpipeline/census_pipeline.ipynb +++ b/7-vertexpipeline/census_pipeline.ipynb @@ -27,7 +27,7 @@ }, "outputs": [], "source": [ - "PROJECT_ID = \"prj-d-bu3machine-learning-gxcv\"\n", + "PROJECT_ID = \"prj-d-ml-machine-learning-gxcv\"\n", "REGION = \"us-central1\"\n", "BUCKET_URI = \"gs://bkt-d-vertexpipe-test-dev\"" ] @@ -72,7 +72,7 @@ "metadata": {}, "source": [ "This is the image we will use to run pipeline components. Replace the name of the artifact project with that of yours, e.g.:\n", - "##### \"us-central1-docker.pkg.dev/{prj-c-bu3artifacts-####}/c-publish-artifacts/vertexpipeline:v2\"\n", + "##### \"us-central1-docker.pkg.dev/{prj-c-ml-artifacts-####}/c-publish-artifacts/vertexpipeline:v2\"\n", "As part of the project inflation pipelines, the image from the Dockerfile in this repository is built and pushed to project" ] }, @@ -85,7 +85,7 @@ }, "outputs": [], "source": [ - "Image = \"us-central1-docker.pkg.dev/prj-c-bu3artifacts-5wdo/c-publish-artifacts/vertexpipeline:v2\"" + "Image = \"us-central1-docker.pkg.dev/prj-c-ml-artifacts-5wdo/c-publish-artifacts/vertexpipeline:v2\"" ] }, { @@ -1555,8 +1555,8 @@ " 'max_nodes': 4,\n", " 'deployment_project': PROJECT_ID,\n", " # important to replace the envryption key here with the key in your own dev environment.\n", - " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning\n", - " \"encryption\": 'projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning',\n", + " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning\n", + " \"encryption\": 'projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning',\n", " \"service_account\": service_account,\n", " \"prod_service_account\": prod_service_account,\n", "}\n", @@ -1573,8 +1573,8 @@ " template_path='./common/vertex-ai-pipeline/pipeline_package.yaml',\n", " pipeline_root=pipelineroot,\n", " # important to replace the envryption key here with the key in your own dev environment.\n", - " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning\n", - " encryption_spec_key_name='projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning',\n", + " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning\n", + " encryption_spec_key_name='projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning',\n", " parameter_values={\n", " \"create_bq_dataset_query\": create_bq_dataset_query,\n", " \"bq_dataset\": data_config['bq_dataset'],\n", diff --git a/7-vertexpipeline/runpipeline.py b/7-vertexpipeline/runpipeline.py index 59c09d06..b737ebf5 100644 --- a/7-vertexpipeline/runpipeline.py +++ b/7-vertexpipeline/runpipeline.py @@ -117,7 +117,7 @@ def __init__(self, 'max_nodes': 4, 'deployment_project': self.PROD_PROJECT_ID, # Raplace encryption with the name of the kms key in the kms project of the prod folder - "encryption": 'projects/prj-p-kms-lkuy/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-p-bu3machine-learning', + "encryption": 'projects/prj-p-kms-lkuy/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-p-ml-machine-learning', "service_account": self.SERVICE_ACCOUNT, "prod_service_account": self.PROD_SERVICE_ACCOUNT } @@ -135,7 +135,7 @@ def execute(self): template_path=self.yaml_file_path, pipeline_root=self.pipelineroot, # Raplace encryption with the name of the kms key in the kms project of the non-prod folder - encryption_spec_key_name='projects/prj-n-kms-gi2r/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-n-bu3machine-learning', + encryption_spec_key_name='projects/prj-n-kms-gi2r/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-n-ml-machine-learning', parameter_values={ "create_bq_dataset_query": self.create_bq_dataset_query, "bq_dataset": self.data_config['bq_dataset'], @@ -179,7 +179,7 @@ def execute(self): pipeline = vertex_ai_pipeline( # Replace with your non-prod project Id PROJECT_ID="prj-n-bu3machine-learning-brk1", \ - PROD_PROJECT_ID='prj-p-bu3machine-learning-skc4', \ # Replace with your prod project Id + PROD_PROJECT_ID='prj-p-ml-machine-learning-skc4', \ # Replace with your prod project Id REGION="us-central1", \ BUCKET_URI="gs://bkt-n-ml-storage-akdv", \ # Replace with your bucket in non-prod DATA_PATH="data", \ diff --git a/ERRATA.md b/ERRATA.md index a00986a7..2c8bec34 100644 --- a/ERRATA.md +++ b/ERRATA.md @@ -24,7 +24,7 @@ This will be addressed in the next version of the blueprint guide. #### Naming - The Service Account naming is not aligned to the blueprint guide. Naming will be modified accordingly in a future release. -- The infrastructure pipeline project naming (`prj-buN-c-infra-pipeline`) is not aligned to the blueprint guide(`prj-buN-c-sample-infra-pipeline`). Naming will be modified accordingly in a future release. +- The infrastructure pipeline project naming (`prj-ml-buN-c-infra-pipeline`) is not aligned to the blueprint guide(`prj-ml-buN-c-sample-infra-pipeline`). Naming will be modified accordingly in a future release. #### Networking - The “allow-windows-activation” rule that exists in the code is not explicitly called out in the guide. diff --git a/README.md b/README.md index 8f063b27..84bc361c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Alternatively, the user can follow steps on [`docs/deploy_on_foundation_v4.0.0.m * [3. networks-dual-svpc](./3-networks-dual-svpc/) * On this repository, it will also configure a private DNS zone for workbench instances to use either `private.googleapis.com` or `restricted.googleapis.com`. * [4. projects](./4-projects/) - * Instead of creating `business_unit_1` and `business_unit_2`, this repository exclusively creates `business_unit_3`. + * Instead of creating `business_unit_1` and `business_unit_2`, this repository exclusively creates `ml_business_unit`. * Additionally, it will establish a Service Catalog project capable of hosting terraform solutions and an artifacts project. * Will create a Machine Learning project for each environment. * [5. app-infra](./5-app-infra/) diff --git a/build/tf-wrapper.sh b/build/tf-wrapper.sh index 1b39b6e8..fe7d36e1 100755 --- a/build/tf-wrapper.sh +++ b/build/tf-wrapper.sh @@ -41,15 +41,15 @@ min_depth=1 # Must be configured based in your directory design # additional special value "shared" # # When using environments as root nodes the regex contains the name of the -# folder that contain the Terraform configuration e.g: business_unit_1 -# and business_unit_2 +# folder that contain the Terraform configuration e.g: ml_business_unit_1 +# and ml_business_unit_2 #==============================================================================# # Environments as leaf nodes in source code case leaf_regex_plan="^(development|non-production|production|shared)$" # Environments as root nodes in source code case -# leaf_regex_plan="^(business_unit_1|business_unit_2)$" +# leaf_regex_plan="^(ml_business_unit_1|ml_business_unit_2)$" #====================================================================# # Function used for the criteria for running terraform int/plan/show @@ -69,11 +69,11 @@ do_plan() { # Environments as leaf nodes in source code case (Default) # Example: # git-repo -# └── business_unit_1 +# └── ml_business_unit_1 # ├── development # ├── non-production # └── production -# └── business_unit_2 +# └── ml_business_unit_2 # ├── development # ├── non-production # └── production @@ -96,19 +96,19 @@ do_action() { # Example: # git-repo # └── development -# ├── business_unit_1 -# └── business_unit_2 +# ├── ml_business_unit_1 +# └── ml_business_unit_2 # └── non-production -# ├── business_unit_1 -# └── business_unit_2 +# ├── ml_business_unit_1 +# └── ml_business_unit_2 # └── production -# ├── business_unit_1 -# └── business_unit_2 +# ├── ml_business_unit_1 +# └── ml_business_unit_2 #=============================================================# ##### Start of alternative source organization - uncomment to use Environments as root nodes ##### -# leaf_regex_action="^(business_unit_1|business_unit_2)$" # edit this list +# leaf_regex_action="^(ml_business_unit_1|ml_business_unit_2)$" # edit this list # do_action() { # local env_path="$1" # local tf_env="${env_path#$base_dir/}" diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 8f4637a4..aef67481 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -269,7 +269,7 @@ You will need to mark some Terraform resources as **tainted** in order to trigge 1. In a terminal, navigate to the path where the error is being reported. - For example, if the unknown project ID is `prj-bu1-p-sample-base-abcd`, you should go to ./gcp-projects/business_unit_1/production (`business_unit_1` due to `bu1` and `production` due to `p`, see the Security Foundations [naming conventions](https://cloud.google.com/architecture/security-foundations/using-example-terraform#naming_conventions) for more information on the projects naming guideline). + For example, if the unknown project ID is `prj-ml-p-sample-base-abcd`, you should go to ./gcp-projects/ml_business_unit/production (`ml_business_unit` due to `ml` and `production` due to `p`, see the Security Foundations [naming conventions](https://cloud.google.com/architecture/security-foundations/using-example-terraform#naming_conventions) for more information on the projects naming guideline). ```bash cd ./gcp-projects// diff --git a/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example b/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example index 5dfb05be..2f8429e2 100644 --- a/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example +++ b/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu2-shared" + name = "4-ml-shared" } } } diff --git a/docs/assets/terraform/4-projects/modules/env_folders/variables.tf b/docs/assets/terraform/4-projects/modules/env_folders/variables.tf index 2297d829..3661fa96 100644 --- a/docs/assets/terraform/4-projects/modules/env_folders/variables.tf +++ b/docs/assets/terraform/4-projects/modules/env_folders/variables.tf @@ -15,7 +15,7 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } diff --git a/docs/assets/terraform/4-projects/modules/ml_env/variables.tf b/docs/assets/terraform/4-projects/modules/ml_env/variables.tf index 592d10fc..dc374ee0 100644 --- a/docs/assets/terraform/4-projects/modules/ml_env/variables.tf +++ b/docs/assets/terraform/4-projects/modules/ml_env/variables.tf @@ -15,12 +15,12 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/docs/change_resource_hierarchy.md b/docs/change_resource_hierarchy.md index 34e0fae1..d13fb1eb 100644 --- a/docs/change_resource_hierarchy.md +++ b/docs/change_resource_hierarchy.md @@ -180,24 +180,16 @@ example-organization/ 1. **(Optional)** To simplify the below changes renaming business_units here is helper script. **Remember to review the changes**. The below script assumes you are in `gcp-projects` folder: ```bash - for i in `find "./business_unit_1" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu1//" $i; done + for i in `find "./ml_business_unit" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/ml//" $i; done - for i in `find "./business_unit_1" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_1//" $i; done - - for i in `find "./business_unit_2" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu2//" $i; done - - for i in `find "./business_unit_2" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_2//" $i; done + for i in `find "./ml_business_unit" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/ml_business_unit//" $i; done for i in `find "./business_unit_" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu//" $i; done for i in `find "./business_unit_" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_//" $i; done ``` -1. For this example, just rename folders business_unit_1 and business_unit_2 to your Business Units names, i.e: finance and retail, to match the example folder hierarchy. - - - - +1. For this example, just rename folder ml_business_unit to your Business Units name, i.e: finance and retail, to match the example folder hierarchy. 1. Change backend gcs prefix for each business unit shared resources. Example: diff --git a/examples/machine-learning-pipeline/README.md b/examples/machine-learning-pipeline/README.md index 7ce16ee4..0161c9ed 100644 --- a/examples/machine-learning-pipeline/README.md +++ b/examples/machine-learning-pipeline/README.md @@ -76,15 +76,15 @@ Below, you can find the values that will need to be applied to `common.auto.tfva In `common.auto.tfvars` update your `perimeter_additional_members` to include: ``` - "serviceAccount:sa-tf-cb-bu3-machine-learning@[prj_c_bu3infra_pipeline_project_id].iam.gserviceaccount.com" + "serviceAccount:sa-tf-cb-ml-machine-learning@[prj_c_ml_infra_pipeline_project_id].iam.gserviceaccount.com" "serviceAccount:sa-terraform-env@[prj_b_seed_project_id].iam.gserviceaccount.com" "serviceAccount:service-[prj_d_logging_project_number]@gs-project-accounts.iam.gserviceaccount.com" "serviceAccount:[prj_d_machine_learning_project_number]@cloudbuild.gserviceaccount.com" ``` ```bash - export prj_c_bu3infra_pipeline_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) - echo "prj_c_bu3infra_pipeline_project_id = ${prj_c_bu3infra_pipeline_project_id}" + export prj_c_ml-infra_pipeline_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) + echo "prj_c_ml-infra_pipeline_project_id = ${prj_c_ml_infra_pipeline_project_id}" export prj_b_seed_project_id=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw seed_project_id) echo "prj_b_seed_project_id = ${prj_b_seed_project_id}" @@ -104,7 +104,7 @@ In `common.auto.tfvars` update your `perimeter_additional_members` to include: export project_d_logging_project_number=$(gsutil cat gs://$backend_bucket/terraform/environments/development/default.tfstate | jq -r '.outputs.env_log_project_number.value') echo "project_d_logging_project_number = ${project_d_logging_project_number}" - prj_d_machine_learning_project_number=$(gsutil cat gs://$backend_bucket_projects/terraform/projects/business_unit_3/development/default.tfstate | jq -r '.outputs.machine_learning_project_number.value') + prj_d_machine_learning_project_number=$(gsutil cat gs://$backend_bucket_projects/terraform/projects/ml_business_unit/development/default.tfstate | jq -r '.outputs.machine_learning_project_number.value') echo "project_d_machine_learning_number = ${prj_d_machine_learning_project_number}" ``` @@ -131,7 +131,7 @@ Once there, select the perimeter that is associated with the environment (eg. `d "resources" = [ "projects/[your-environment-shared-restricted-project-number]", "projects/[your-environment-kms-project-number]", - "projects/[your-environment-bu3machine-learning-number]", + "projects/[your-environment-mlmachine-learning-number]", ] "operations" = { "compute.googleapis.com" = { @@ -190,8 +190,8 @@ For your DEVELOPMENT.AUTO.TFVARS file, also include this as an egress policy: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@compute-system.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@compute-system.iam.gserviceaccount.com", ] }, "to" = { @@ -232,7 +232,7 @@ Add in your dags in the `dags` folder. Any changes to this folder will trigger Have a github token for access to your repository ready, along with an [Application Installation Id](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github#connecting_a_github_host_programmatically) and the remote uri to your repository. -These environmental project inflations are closely tied to the `service-catalog` project that have already deployed. By now, the `bu3-service-catalog` should have been inflated. `service-catalog` contains modules that are being deployed in an interactive (development) environment. Since they already exist; they can be used as terraform modules for operational (non-production, production) environments. This was done in order to avoid code redundancy. One area for all `machine-learning` deployments. +These environmental project inflations are closely tied to the `service-catalog` project that have already deployed. By now, the `ml-service-catalog` should have been inflated. `service-catalog` contains modules that are being deployed in an interactive (development) environment. Since they already exist; they can be used as terraform modules for operational (non-production, production) environments. This was done in order to avoid code redundancy. One area for all `machine-learning` deployments. Under `modules/base_env/main.tf` you will notice all module calls are using `git` links as sources. These links refer to the `service-catalog` cloud source repository we have already set up. @@ -245,7 +245,7 @@ Clone the repo at the same level of the `terraform-google-enterprise-genai` fold Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get the Cloud Build Project ID. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} gcloud source repos clone gcp-policies gcp-policies-app-infra --project=${INFRA_PIPELINE_PROJECT_ID} @@ -279,18 +279,18 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get cd .. ``` -1. Clone the `bu3-machine-learning` repo. +1. Clone the `ml-machine-learning` repo. ```bash - gcloud source repos clone bu3-machine-learning --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-machine-learning --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of foundation to new repo. - All subsequent steps assume you are running them from the bu3-machine-learning directory. + All subsequent steps assume you are running them from the ml-machine-learning directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-machine-learning + cd ml-machine-learning git checkout -b plan cp -RT ../terraform-google-enterprise-genai/examples/machine-learning-pipeline . @@ -326,7 +326,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Use `terraform output` to retrieve the Service Catalog project-id from the projects step and update values in `module/base_env`. ```bash - export service_catalog_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) echo "service_catalog_project_id = ${service_catalog_project_id}" ## Linux @@ -336,7 +336,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-machine-learning"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-machine-learning"' --raw-output) echo "backend_bucket = ${backend_bucket}" ## Linux @@ -349,7 +349,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `modules/base_env/main.tf` with the name of service catalog project id to complete the git fqdn for module sources: ```bash - export service_catalog_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) ##LINUX sed -i "s/SERVICE-CATALOG-PROJECT-ID/${service_catalog_project_id}/" ./modules/base_env/main.tf @@ -373,7 +373,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get github_token="YOUR-GITHUB-TOKEN" for env in "${envs[@]}"; do - output=$(terraform -chdir="../gcp-projects/business_unit_3/${env}" output -raw machine_learning_project_id) + output=$(terraform -chdir="../gcp-projects/ml_business_unit/${env}" output -raw machine_learning_project_id) project_ids+=("$output") done @@ -446,7 +446,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -454,10 +454,10 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-machine-learning"' --raw-output) + terraform_sa=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-machine-learning"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -466,7 +466,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-machine-learning"' --raw-output) + export backend_bucket=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-machine-learning"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -475,7 +475,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `modules/base_env/main.tf` with Service Catalog Project Id. ```bash - export service_catalog_project_id=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) echo "service_catalog_project_id = ${service_catalog_project_id}" ## Linux @@ -490,10 +490,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-machine-learning"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-machine-learning"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -568,9 +568,9 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir In order to avoid having to specify a kms key for every query against a bigquery resource, we set the default project encryption key to the corresponding environment key in advance ```bash - ml_project_dev=$(terraform -chdir="gcp-projects/business_unit_3/development" output -json) - ml_project_nonprd=$(terraform -chdir="gcp-projects/business_unit_3/non-production" output -json) - ml_project_prd=$(terraform -chdir="gcp-projects/business_unit_3/production" output -json) + ml_project_dev=$(terraform -chdir="gcp-projects/ml_business_unit/development" output -json) + ml_project_nonprd=$(terraform -chdir="gcp-projects/ml_business_unit/non-production" output -json) + ml_project_prd=$(terraform -chdir="gcp-projects/ml_business_unit/production" output -json) projects=( "$ml_project_dev" "$ml_project_nonprd" "$ml_project_prd" ) @@ -586,18 +586,18 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Now that machine learning's projects have all been inflated, please _return to gcp-projects_ and update COMMON.AUTO.TFVARS with this __additional__ information under `perimeter_additional_members`: ``` - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", - "serviceAccount:[prj-n-bu3machine-learning-number]@cloudbuild.gserviceaccount.com", - "serviceAccount:[prj-n-bu3machine-learning-number]-compute@developer.gserviceaccount.com", - "serviceAccount:[prj-p-bu3machine-learning-number]@cloudbuild.gserviceaccount.com", - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]@cloudbuild.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]-compute@developer.gserviceaccount.com", + "serviceAccount:[prj-p-ml-machine-learning-number]@cloudbuild.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", ``` 2. optional - run the below command to generate a list of the above changes needed to COMMON.AUTO.TFVARS ```bash - ml_n=$(terraform -chdir="gcp-projects/business_unit_3/non-production" output -raw machine_learning_project_number) - ml_p=$(terraform -chdir="gcp-projects/business_unit_3/production" output -raw machine_learning_project_number) + ml_n=$(terraform -chdir="gcp-projects/ml_business_unit/non-production" output -raw machine_learning_project_number) + ml_p=$(terraform -chdir="gcp-projects/ml_business_unit/production" output -raw machine_learning_project_number) echo "serviceAccount:service-${ml_n}@dataflow-service-producer-prod.iam.gserviceaccount.com", echo "serviceAccount:${ml_n}@cloudbuild.gserviceaccount.com", @@ -611,7 +611,7 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Add in more agents to the DEVELOPMENT.AUTO.TFVARS file under `egress_policies`. Notably: - * "serviceAccount:bq-[prj-d-bu3machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" + * "serviceAccount:bq-[prj-d-ml-machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" This should be added under identities. It should look like this:: @@ -622,9 +622,9 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:bq-[prj-d-bu3machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" << New Addition - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@compute-system.iam.gserviceaccount.com", + "serviceAccount:bq-[prj-d-ml-machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" << New Addition + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@compute-system.iam.gserviceaccount.com", ] }, "to" = { @@ -650,11 +650,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-d-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-c-bu3artifacts-number]"] + "resources" = ["projects/[prj-c-ml-artifacts-number]"] "operations" = { "artifactregistry.googleapis.com" = { "methods" = ["*"] @@ -667,11 +667,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-n-bu3machine-learning-number]"] + "resources" = ["projects/[prj-n-ml-machine-learning-number]"] "operations" = { "compute.googleapis.com" = { "methods" = ["*"] @@ -691,7 +691,7 @@ Notably: }, "to" = { "resources" = [ - "projects/[prj-c-bu3artifacts-number]" + "projects/[prj-c-ml-artifacts-number]" ] "operations" = { "artifactregistry.googleapis.com" = { @@ -705,11 +705,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-c-bu3artifacts-number]"] + "resources" = ["projects/[prj-c-ml-artifacts-number]"] "operations" = { "artifactregistry.googleapis.com" = { "methods" = ["*"] @@ -722,7 +722,7 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", ] }, "to" = { @@ -738,12 +738,12 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:[prj-n-bu3machine-learning-number]-compute@developer.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]-compute@developer.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-p-bu3machine-learning-number]"] + "resources" = ["projects/[prj-p-ml-machine-learning-number]"] "operations" = { "aiplatform.googleapis.com" = { "methods" = ["*"] @@ -766,15 +766,15 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", "serviceAccount:cloud-cicd-artifact-registry-copier@system.gserviceaccount.com", ] }, "to" = { "resources" = [ - "projects/[prj-n-bu3machine-learning-number]", - "projects/[prj-c-bu3artifacts-number]", + "projects/[prj-n-ml-machine-learning-number]", + "projects/[prj-c-ml-artifacts-number]", ] "operations" = { "artifactregistry.googleapis.com" = { @@ -795,7 +795,7 @@ Notably: Once you have set up service catalog and attempt to deploy out terraform code, there is a high chance you will encounter this error: `Permission denied; please check you have the correct IAM permissions and APIs enabled.` -This is due to a VPC Service control error that until now, is impossible to add into the egress policy. Go to `prj-d-bu3machine-learning` project and view the logs, filtering for ERRORS. There will be a VPC Service Controls entry that has an `egressViolation`. It should look something like the following: +This is due to a VPC Service control error that until now, is impossible to add into the egress policy. Go to `prj-d-ml-machine-learning` project and view the logs, filtering for ERRORS. There will be a VPC Service Controls entry that has an `egressViolation`. It should look something like the following: ``` egressViolations: [ 0: { @@ -926,14 +926,14 @@ Also make sure to have a gcs bucket ready to store the artifacts for the tutoria |variable|definition|example value|How to obtain| |--------|----------|-------------|-------------| - |PROJECT_ID|The id of the non-prod project|`{none-prod-project-id}`|From the project's menu in console navigate to the `fldr-non-production/fldr-non-production-bu3` folder; here you can find the machine learning project in non-prod (`prj-n-bu3machine-learning`) and obtain its' ID| - |BUCKET_URI|URI of the non-prod bucket|`gs://non-prod-bucket`|From the project menu in console navigate to the non-prod ML project `fldr-non-production/fldr-non-production-bu3/prj-n-bu3machine-learning` project, navigate to cloud storage and copy the name of the bucket available there| + |PROJECT_ID|The id of the non-prod project|`{none-prod-project-id}`|From the project's menu in console navigate to the `fldr-non-production/fldr-non-production-ml` folder; here you can find the machine learning project in non-prod (`prj-n-ml-machine-learning`) and obtain its' ID| + |BUCKET_URI|URI of the non-prod bucket|`gs://non-prod-bucket`|From the project menu in console navigate to the non-prod ML project `fldr-non-production/fldr-non-production-ml/prj-n-ml-machine-learning` project, navigate to cloud storage and copy the name of the bucket available there| |REGION|The region for pipeline jobs|Can be left as default `us-central1`| - |PROD_PROJECT_ID|ID of the prod project|`prod-project-id`|In console's project menu, navigate to the `fldr-production/fldr-production-bu3` folder; here you can find the machine learning project in prod (`prj-p-bu3machine-learning`) and obtain its' ID| - |Image|The image artifact used to run the pipeline components. The image is already built and pushed to the artifact repository in your artifact project under the common folder|`f"us-central1-docker.pkg.dev/{{artifact-project}}/{{artifact-repository}}/vertexpipeline:v2"`|Navigate to `fldr-common/prj-c-bu3artifacts` project. Navigate to the artifact registry repositories in the project to find the full name of the image artifact.| + |PROD_PROJECT_ID|ID of the prod project|`prod-project-id`|In console's project menu, navigate to the `fldr-production/fldr-production-ml` folder; here you can find the machine learning project in prod (`prj-p-ml-machine-learning`) and obtain its' ID| + |Image|The image artifact used to run the pipeline components. The image is already built and pushed to the artifact repository in your artifact project under the common folder|`f"us-central1-docker.pkg.dev/{{artifact-project}}/{{artifact-repository}}/vertexpipeline:v2"`|Navigate to `fldr-common/prj-c-ml-artifacts` project. Navigate to the artifact registry repositories in the project to find the full name of the image artifact.| |DATAFLOW_SUBNET|The shared subnet in non-prod env required to run the dataflow job|`https://www.googleapis.com/compute/v1/projects/{non-prod-network-project}/regions/us-central1/subnetworks/{subnetwork-name}`|Navigate to the `fldr-network/prj-n-shared-restricted` project. Navigate to the VPC networks and under the subnets tab, find the name of the network associated with your region (us-central1)| - |SERVICE_ACCOUNT|The service account used to run the pipeline and it's components such as the model monitoring job. This is the compute default service account of non-prod if you don't plan on using another costume service account|`{non-prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the non-prod project `fldr-non-production/fldr-non-production-bu3/prj-n-bu3machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| - |PROD_SERICE_ACCOUNT|The service account used to create endpoint, upload the model, and deploy the model in the prod project. This is the compute default service account of prod if you don't plan on using another costume service account|`{prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the prod project `fldr-production/fldr-production-bu3/prj-p-bu3machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| + |SERVICE_ACCOUNT|The service account used to run the pipeline and it's components such as the model monitoring job. This is the compute default service account of non-prod if you don't plan on using another costume service account|`{non-prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the non-prod project `fldr-non-production/fldr-non-production-ml/prj-n-ml-machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| + |PROD_SERICE_ACCOUNT|The service account used to create endpoint, upload the model, and deploy the model in the prod project. This is the compute default service account of prod if you don't plan on using another costume service account|`{prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the prod project `fldr-production/fldr-production-ml/prj-p-ml-machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| |deployment_config['encryption']|The kms key for the prod env. This key is used to encrypt the vertex model, endpoint, model deployment, and model monitoring.|`projects/{prod-kms-project}/locations/us-central1/keyRings/{keyring-name}/cryptoKeys/{key-name}`|Navigate to `fldr-production/prj-n-kms`, navigate to the Security/Key management in that project to find the key in `sample-keyring` keyring of your target region `us-central1`| |encryption_spec_key_name|The name of the encryption key for the non-prod env. This key is used to create the vertex pipeline job and it's associated metadata store|`projects/{non-prod-kms-project}/locations/us-central1/keyRings/{keyring-name}/cryptoKeys/{key-name}`|Navigate to `fldr-non-production/prj-n-kms`, navigate to the Security/Key management in that project to find the key in `sample-keyring` keyring of your target region `us-central1`| |monitoring_config['email']|The email that Vertex AI monitoring will email alerts to|`your email`|your email associated with your gcp account| diff --git a/examples/machine-learning-pipeline/business_unit_3/development/README.md b/examples/machine-learning-pipeline/ml_business_unit/development/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/README.md rename to examples/machine-learning-pipeline/ml_business_unit/development/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/production/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/development/backend.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/production/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/backend.tf index 9a280f76..ebcfd862 100644 --- a/examples/machine-learning-pipeline/business_unit_3/production/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/development/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/production" + prefix = "terraform/app-infra/ml_business_unit/development" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/development/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/development/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/development/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/development/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/development/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/development/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/locals.tf index 39b3471f..593d0e47 100644 --- a/examples/machine-learning-pipeline/business_unit_3/development/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/development/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "development" environment_code = "d" } diff --git a/examples/machine-learning-pipeline/business_unit_3/development/main.tf b/examples/machine-learning-pipeline/ml_business_unit/development/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/development/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/development/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/development/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/development/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/versions.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/README.md b/examples/machine-learning-pipeline/ml_business_unit/non-production/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/README.md rename to examples/machine-learning-pipeline/ml_business_unit/non-production/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf similarity index 90% rename from examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf index a8c71ae7..d737ec7d 100644 --- a/examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/non-production" + prefix = "terraform/app-infra/ml_business_unit/non-production" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/non-production/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/non-production/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf index 6600d7b7..e59d1a15 100644 --- a/examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "non-production" environment_code = "n" } diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/main.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/versions.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/README.md b/examples/machine-learning-pipeline/ml_business_unit/production/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/README.md rename to examples/machine-learning-pipeline/ml_business_unit/production/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/development/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/production/backend.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/development/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/backend.tf index 60060517..f8d1bace 100644 --- a/examples/machine-learning-pipeline/business_unit_3/development/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/development" + prefix = "terraform/app-infra/ml_business_unit/production" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/production/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/production/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/production/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/production/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/production/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/production/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/locals.tf index 5a1cd7c8..daf31502 100644 --- a/examples/machine-learning-pipeline/business_unit_3/production/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/production/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "production" environment_code = "p" } diff --git a/examples/machine-learning-pipeline/business_unit_3/production/main.tf b/examples/machine-learning-pipeline/ml_business_unit/production/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/production/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/production/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/production/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/production/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/versions.tf diff --git a/examples/machine-learning-pipeline/modules/base_env/variables.tf b/examples/machine-learning-pipeline/modules/base_env/variables.tf index 55ff0fee..848a9ad4 100644 --- a/examples/machine-learning-pipeline/modules/base_env/variables.tf +++ b/examples/machine-learning-pipeline/modules/base_env/variables.tf @@ -29,7 +29,7 @@ variable "environment_code" { } variable "business_code" { - description = "Business unit code (ie. bu3)" + description = "Business unit code (ie. ml)" type = string } diff --git a/helpers/foundation-deployer/README.md b/helpers/foundation-deployer/README.md index 22c06132..4217512a 100644 --- a/helpers/foundation-deployer/README.md +++ b/helpers/foundation-deployer/README.md @@ -123,7 +123,7 @@ Im addition to the variables declared in the file `global.tfvars` for configurin ```text deploy-directory/ - └── bu1-example-app + └── ml-example-app └── gcp-bootstrap └── gcp-environments └── gcp-networks diff --git a/helpers/foundation-deployer/global.tfvars.example b/helpers/foundation-deployer/global.tfvars.example index 5da7ab8b..f161423a 100644 --- a/helpers/foundation-deployer/global.tfvars.example +++ b/helpers/foundation-deployer/global.tfvars.example @@ -149,7 +149,7 @@ target_name_server_addresses = [ // 4-projects inputs -// https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/business_unit_1/production/README.md#inputs +// https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/ml_business_unit/production/README.md#inputs projects_gcs_location = "US" projects_kms_location = "us" diff --git a/helpers/foundation-deployer/main.go b/helpers/foundation-deployer/main.go index 1475a645..01bd3c8a 100644 --- a/helpers/foundation-deployer/main.go +++ b/helpers/foundation-deployer/main.go @@ -163,8 +163,8 @@ func main() { // Note: destroy is only terraform destroy, local directories are not deleted. // 5-app-infra msg.PrintStageMsg("Destroying 5-app-infra stage") - err = s.RunDestroyStep("bu1-example-app", func() error { - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") + err = s.RunDestroyStep("ml-example-app", func() error { + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") return stages.DestroyExampleAppStage(t, s, io, conf) }) if err != nil { @@ -302,12 +302,12 @@ func main() { // 5-app-infra msg.PrintStageMsg("Deploying 5-app-infra stage") - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") io.RemoteStateBucket = bo.RemoteStateBucketProjects msg.PrintBuildMsg(io.InfraPipeProj, io.DefaultRegion, conf.DisablePrompt) - err = s.RunStep("bu1-example-app", func() error { + err = s.RunStep("ml-example-app", func() error { return stages.DeployExampleAppStage(t, s, globalTFVars, io, conf) }) if err != nil { diff --git a/helpers/foundation-deployer/stages/apply.go b/helpers/foundation-deployer/stages/apply.go index f78486e7..6d92ea83 100644 --- a/helpers/foundation-deployer/stages/apply.go +++ b/helpers/foundation-deployer/stages/apply.go @@ -375,7 +375,7 @@ func DeployProjectsStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, outpu Repo: ProjectsRepo, GitConf: conf, HasManualStep: true, - GroupingUnits: []string{"business_unit_1", "business_unit_2"}, + GroupingUnits: []string{"ml_business_unit"}, Envs: []string{"production", "non-production", "development"}, } @@ -395,7 +395,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out } // update backend bucket for _, e := range []string{"production", "non-production", "development"} { - err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "business_unit_1", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) + err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "ml_business_unit", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) if err != nil { return err } @@ -404,7 +404,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out gcpPoliciesPath := filepath.Join(c.CheckoutPath, "gcp-policies-app-infra") policiesConf := utils.CloneCSR(t, PoliciesRepo, gcpPoliciesPath, outputs.InfraPipeProj, c.Logger) policiesBranch := "main" - err = s.RunStep("bu1-example-app.gcp-policies-app-infra", func() error { + err = s.RunStep("ml-example-app.gcp-policies-app-infra", func() error { return preparePoliciesRepo(policiesConf, policiesBranch, c.FoundationPath, gcpPoliciesPath) }) if err != nil { diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index 59f90df2..e62bb54f 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -34,7 +34,7 @@ const ( EnvironmentsRepo = "gcp-environments" NetworksRepo = "gcp-networks" ProjectsRepo = "gcp-projects" - AppInfraRepo = "bu1-example-app" + AppInfraRepo = "ml-example-app" BootstrapStep = "0-bootstrap" OrgStep = "1-org" EnvironmentsStep = "2-environments" @@ -286,15 +286,15 @@ func GetBootstrapStepOutputs(t testing.TB, foundationPath string) BootstrapOutpu func GetInfraPipelineOutputs(t testing.TB, checkoutPath, workspace string) InfraPipelineOutputs { options := &terraform.Options{ - TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "business_unit_1", "shared"), + TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "ml_business_unit", "shared"), Logger: logger.Discard, NoColor: true, } return InfraPipelineOutputs{ InfraPipeProj: terraform.Output(t, options, "cloudbuild_project_id"), DefaultRegion: terraform.Output(t, options, "default_region"), - TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["bu1-example-app"], - StateBucket: terraform.OutputMap(t, options, "state_buckets")["bu1-example-app"], + TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["ml-example-app"], + StateBucket: terraform.OutputMap(t, options, "state_buckets")["ml-example-app"], } } diff --git a/helpers/foundation-deployer/stages/destroy.go b/helpers/foundation-deployer/stages/destroy.go index 250dc512..f75e7560 100644 --- a/helpers/foundation-deployer/stages/destroy.go +++ b/helpers/foundation-deployer/stages/destroy.go @@ -135,7 +135,7 @@ func DestroyProjectsStage(t testing.TB, s steps.Steps, outputs BootstrapOutputs, Step: ProjectsStep, Repo: ProjectsRepo, HasManualStep: true, - GroupingUnits: []string{"business_unit_1", "business_unit_2"}, + GroupingUnits: []string{"ml_business_unit"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) @@ -148,7 +148,7 @@ func DestroyExampleAppStage(t testing.TB, s steps.Steps, outputs InfraPipelineOu CICDProject: outputs.InfraPipeProj, Step: AppInfraStep, Repo: AppInfraRepo, - GroupingUnits: []string{"business_unit_1"}, + GroupingUnits: []string{"ml_business_unit"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) diff --git a/test/disable_tf_files.sh b/test/disable_tf_files.sh index 9e8f8a73..4d3c612d 100755 --- a/test/disable_tf_files.sh +++ b/test/disable_tf_files.sh @@ -56,40 +56,31 @@ function shared(){ function projectsshared(){ # disable shared.auto.tfvars - mv 4-projects/business_unit_1/shared/shared.auto.tfvars 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled - mv 4-projects/business_unit_2/shared/shared.auto.tfvars 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled + mv 4-projects/ml_business_unit/shared/shared.auto.tfvars 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled # disable common.auto.tfvars - mv 4-projects/business_unit_1/shared/common.auto.tfvars 4-projects/business_unit_1/shared/common.auto.tfvars.disabled - mv 4-projects/business_unit_2/shared/common.auto.tfvars 4-projects/business_unit_2/shared/common.auto.tfvars.disabled + mv 4-projects/ml_business_unit/shared/common.auto.tfvars 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled } function projects(){ # disable ENVS.auto.tfvars in main module - mv 4-projects/business_unit_1/development/development.auto.tfvars 4-projects/business_unit_1/development/development.auto.tfvars.disabled - mv 4-projects/business_unit_2/development/development.auto.tfvars 4-projects/business_unit_2/development/development.auto.tfvars.disabled - mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled - mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled - mv 4-projects/business_unit_1/production/production.auto.tfvars 4-projects/business_unit_1/production/production.auto.tfvars.disabled - mv 4-projects/business_unit_2/production/production.auto.tfvars 4-projects/business_unit_2/production/production.auto.tfvars.disabled + mv 4-projects/ml_business_unit/development/development.auto.tfvars 4-projects/ml_business_unit/development/development.auto.tfvars.disabled + mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled + mv 4-projects/ml_business_unit/production/production.auto.tfvars 4-projects/ml_business_unit/production/production.auto.tfvars.disabled # disable common.auto.tfvars in main module - mv 4-projects/business_unit_1/development/common.auto.tfvars 4-projects/business_unit_1/development/common.auto.tfvars.disabled - mv 4-projects/business_unit_1/non-production/common.auto.tfvars 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled - mv 4-projects/business_unit_1/production/common.auto.tfvars 4-projects/business_unit_1/production/common.auto.tfvars.disabled - mv 4-projects/business_unit_2/development/common.auto.tfvars 4-projects/business_unit_2/development/common.auto.tfvars.disabled - mv 4-projects/business_unit_2/non-production/common.auto.tfvars 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled - mv 4-projects/business_unit_2/production/common.auto.tfvars 4-projects/business_unit_2/production/common.auto.tfvars.disabled + mv 4-projects/ml_business_unit/development/common.auto.tfvars 4-projects/ml_business_unit/development/common.auto.tfvars.disabled + mv 4-projects/ml_business_unit/non-production/common.auto.tfvars 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled + mv 4-projects/ml_business_unit/production/common.auto.tfvars 4-projects/ml_business_unit/production/common.auto.tfvars.disabled } function appinfra(){ # disable common.auto.tfvars in main module - mv 5-app-infra/business_unit_1/development/common.auto.tfvars 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled - mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled - mv 5-app-infra/business_unit_1/production/common.auto.tfvars 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled + mv 5-app-infra/ml_business_unit/development/common.auto.tfvars 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled + mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled + mv 5-app-infra/ml_business_unit/production/common.auto.tfvars 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled } - # parse args for arg in "$@" do diff --git a/test/integration/app-infra/app_infra_test.go b/test/integration/app-infra/app_infra_test.go index b07cf1b2..ea57c78f 100644 --- a/test/integration/app-infra/app_infra_test.go +++ b/test/integration/app-infra/app_infra_test.go @@ -36,12 +36,12 @@ func TestAppInfra(t *testing.T) { } shared := tft.NewTFBlueprintTest(t, - tft.WithTFDir("../../../4-projects/business_unit_1/shared"), + tft.WithTFDir("../../../4-projects/ml_business_unit/shared"), ) // Configure impersonation for test execution - terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["bu1-example-app"] - backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["bu1-example-app"] + terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["ml-example-app"] + backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["ml-example-app"] utils.SetEnv(t, "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT", terraformSA) backendConfig := map[string]interface{}{ "bucket": backend_bucket, @@ -55,11 +55,11 @@ func TestAppInfra(t *testing.T) { t.Run(envName, func(t *testing.T) { projects := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../4-projects/business_unit_1/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../4-projects/ml_business_unit/%s", envName)), ) appInfra := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/business_unit_1/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/ml_business_unit/%s", envName)), tft.WithBackendConfig(backendConfig), tft.WithPolicyLibraryPath("/workspace/policy-library", projects.GetStringOutput("base_shared_vpc_project")), tft.WithVars(vars), diff --git a/test/integration/projects-shared/projects_shared_test.go b/test/integration/projects-shared/projects_shared_test.go index f25f47c1..db3d6df8 100644 --- a/test/integration/projects-shared/projects_shared_test.go +++ b/test/integration/projects-shared/projects_shared_test.go @@ -56,14 +56,9 @@ func TestProjectsShared(t *testing.T) { tfDir string }{ { - name: "bu1", - repo: "bu1-example-app", - tfDir: "../../../4-projects/business_unit_1/shared", - }, - { - name: "bu2", - repo: "bu2-example-app", - tfDir: "../../../4-projects/business_unit_2/shared", + name: "ml", + repo: "ml-example-app", + tfDir: "../../../4-projects/ml_business_unit/shared", }, } { t.Run(tts.name, func(t *testing.T) { diff --git a/test/integration/projects/projects_test.go b/test/integration/projects/projects_test.go index e7ecd2db..6c9d65d5 100644 --- a/test/integration/projects/projects_test.go +++ b/test/integration/projects/projects_test.go @@ -78,44 +78,23 @@ func TestProjects(t *testing.T) { restrictedNetwork string }{ { - name: "bu1_development", - repo: "bu1-example-app", - baseDir: "../../../4-projects/business_unit_1/%s", + name: "ml_development", + repo: "ml-example-app", + baseDir: "../../../4-projects/ml_business_unit/%s", baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), }, { - name: "bu1_non-production", - repo: "bu1-example-app", - baseDir: "../../../4-projects/business_unit_1/%s", + name: "ml_non-production", + repo: "ml-example-app", + baseDir: "../../../4-projects/ml_business_unit/%s", baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), }, { - name: "bu1_production", - repo: "bu1-example-app", - baseDir: "../../../4-projects/business_unit_1/%s", - baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), - restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), - }, - { - name: "bu2_development", - repo: "bu2-example-app", - baseDir: "../../../4-projects/business_unit_2/%s", - baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), - restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), - }, - { - name: "bu2_non-production", - repo: "bu2-example-app", - baseDir: "../../../4-projects/business_unit_2/%s", - baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), - restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), - }, - { - name: "bu2_production", - repo: "bu2-example-app", - baseDir: "../../../4-projects/business_unit_2/%s", + name: "ml_production", + repo: "ml-example-app", + baseDir: "../../../4-projects/ml_business_unit/%s", baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), }, diff --git a/test/restore_tf_files.sh b/test/restore_tf_files.sh index d7022b09..62b85559 100644 --- a/test/restore_tf_files.sh +++ b/test/restore_tf_files.sh @@ -76,70 +76,49 @@ function shared(){ function projects(){ # restore backend configs in main module - mv 4-projects/business_unit_1/development/backend.tf.disabled 4-projects/business_unit_1/development/backend.tf - mv 4-projects/business_unit_1/non-production/backend.tf.disabled 4-projects/business_unit_1/non-production/backend.tf - mv 4-projects/business_unit_1/production/backend.tf.disabled 4-projects/business_unit_1/production/backend.tf - mv 4-projects/business_unit_1/shared/backend.tf.disabled 4-projects/business_unit_1/shared/backend.tf - mv 4-projects/business_unit_2/development/backend.tf.disabled 4-projects/business_unit_2/development/backend.tf - mv 4-projects/business_unit_2/non-production/backend.tf.disabled 4-projects/business_unit_2/non-production/backend.tf - mv 4-projects/business_unit_2/production/backend.tf.disabled 4-projects/business_unit_2/production/backend.tf - mv 4-projects/business_unit_2/shared/backend.tf.disabled 4-projects/business_unit_2/shared/backend.tf + mv 4-projects/ml_business_unit/development/backend.tf.disabled 4-projects/ml_business_unit/development/backend.tf + mv 4-projects/ml_business_unit/non-production/backend.tf.disabled 4-projects/ml_business_unit/non-production/backend.tf + mv 4-projects/ml_business_unit/production/backend.tf.disabled 4-projects/ml_business_unit/production/backend.tf + mv 4-projects/ml_business_unit/shared/backend.tf.disabled 4-projects/ml_business_unit/shared/backend.tf # restore access_context.auto.tfvars in main module - mv 4-projects/business_unit_1/development/access_context.auto.tfvars.disabled 4-projects/business_unit_1/development/access_context.auto.tfvars - mv 4-projects/business_unit_1/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/non-production/access_context.auto.tfvars - mv 4-projects/business_unit_1/production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/production/access_context.auto.tfvars - mv 4-projects/business_unit_2/development/access_context.auto.tfvars.disabled 4-projects/business_unit_2/development/access_context.auto.tfvars - mv 4-projects/business_unit_2/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/non-production/access_context.auto.tfvars - mv 4-projects/business_unit_2/production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/production/access_context.auto.tfvars - - # restore business_unit_1.auto.tfvars in main module - mv 4-projects/business_unit_1/development/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/development/business_unit_1.auto.tfvars - mv 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars - mv 4-projects/business_unit_1/production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/production/business_unit_1.auto.tfvars - - # restore business_unit_2.auto.tfvars in main module - mv 4-projects/business_unit_2/development/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/development/business_unit_2.auto.tfvars - mv 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars - mv 4-projects/business_unit_2/production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/production/business_unit_2.auto.tfvars + mv 4-projects/ml_business_unit/development/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/development/access_context.auto.tfvars + mv 4-projects/ml_business_unit/non-production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/access_context.auto.tfvars + mv 4-projects/ml_business_unit/production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/production/access_context.auto.tfvars + + # restore ml_business_unit.auto.tfvars in main module + mv 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars + mv 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars + mv 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars # restore ENVS.auto.tfvars in main module - mv 4-projects/business_unit_1/development/development.auto.tfvars.disabled 4-projects/business_unit_1/development/development.auto.tfvars - mv 4-projects/business_unit_2/development/development.auto.tfvars.disabled 4-projects/business_unit_2/development/development.auto.tfvars - mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_1/non-production/non-production.auto.tfvars - mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_2/non-production/non-production.auto.tfvars - mv 4-projects/business_unit_1/production/production.auto.tfvars.disabled 4-projects/business_unit_1/production/production.auto.tfvars - mv 4-projects/business_unit_2/production/production.auto.tfvars.disabled 4-projects/business_unit_2/production/production.auto.tfvars - mv 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled 4-projects/business_unit_1/shared/shared.auto.tfvars - mv 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled 4-projects/business_unit_2/shared/shared.auto.tfvars + mv 4-projects/ml_business_unit/development/development.auto.tfvars.disabled 4-projects/ml_business_unit/development/development.auto.tfvars + mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/non-production.auto.tfvars + mv 4-projects/ml_business_unit/production/production.auto.tfvars.disabled 4-projects/ml_business_unit/production/production.auto.tfvars + mv 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled 4-projects/ml_business_unit/shared/shared.auto.tfvars # restore common.auto.tfvars in main module - mv 4-projects/business_unit_1/development/common.auto.tfvars.disabled 4-projects/business_unit_1/development/common.auto.tfvars - mv 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled 4-projects/business_unit_1/non-production/common.auto.tfvars - mv 4-projects/business_unit_1/production/common.auto.tfvars.disabled 4-projects/business_unit_1/production/common.auto.tfvars - mv 4-projects/business_unit_1/shared/common.auto.tfvars.disabled 4-projects/business_unit_1/shared/common.auto.tfvars - mv 4-projects/business_unit_2/development/common.auto.tfvars.disabled 4-projects/business_unit_2/development/common.auto.tfvars - mv 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled 4-projects/business_unit_2/non-production/common.auto.tfvars - mv 4-projects/business_unit_2/production/common.auto.tfvars.disabled 4-projects/business_unit_2/production/common.auto.tfvars - mv 4-projects/business_unit_2/shared/common.auto.tfvars.disabled 4-projects/business_unit_2/shared/common.auto.tfvars - + mv 4-projects/ml_business_unit/development/common.auto.tfvars.disabled 4-projects/ml_business_unit/development/common.auto.tfvars + mv 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/common.auto.tfvars + mv 4-projects/ml_business_unit/production/common.auto.tfvars.disabled 4-projects/ml_business_unit/production/common.auto.tfvars + mv 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled 4-projects/ml_business_unit/shared/common.auto.tfvars } function appinfra(){ # restore backend configs in main module - mv 5-app-infra/business_unit_1/development/backend.tf.disabled 5-app-infra/business_unit_1/development/backend.tf - mv 5-app-infra/business_unit_1/non-production/backend.tf.disabled 5-app-infra/business_unit_1/non-production/backend.tf - mv 5-app-infra/business_unit_1/production/backend.tf.disabled 5-app-infra/business_unit_1/production/backend.tf + mv 5-app-infra/ml_business_unit/development/backend.tf.disabled 5-app-infra/ml_business_unit/development/backend.tf + mv 5-app-infra/ml_business_unit/non-production/backend.tf.disabled 5-app-infra/ml_business_unit/non-production/backend.tf + mv 5-app-infra/ml_business_unit/production/backend.tf.disabled 5-app-infra/ml_business_unit/production/backend.tf # restore ENVS.auto.tfvars in main module - mv 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars.disabled 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars - mv 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars - mv 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars.disabled 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars + mv 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars + mv 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars + mv 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars # restore common.auto.tfvars in main module - mv 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled 5-app-infra/business_unit_1/development/common.auto.tfvars - mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/common.auto.tfvars - mv 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/production/common.auto.tfvars + mv 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/common.auto.tfvars + mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/common.auto.tfvars + mv 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/common.auto.tfvars } From 7cfb47c24e4548f187a1c79414e704c103c0ca12 Mon Sep 17 00:00:00 2001 From: mariammartins Date: Fri, 14 Jun 2024 14:13:24 -0300 Subject: [PATCH 2/8] fix ml_infra_projects name --- 4-projects/ml_business_unit/shared/ml_infra_projects.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/4-projects/ml_business_unit/shared/ml_infra_projects.tf b/4-projects/ml_business_unit/shared/ml_infra_projects.tf index 6ec434b0..7640d30e 100644 --- a/4-projects/ml_business_unit/shared/ml_infra_projects.tf +++ b/4-projects/ml_business_unit/shared/ml_infra_projects.tf @@ -14,8 +14,8 @@ * limitations under the License. */ -module "_infra_projects" { - source = "../../modules/_infra_projects" +module "ml_infra_projects" { + source = "../../modules/ml_infra_projects" org_id = local.org_id folder_id = local.common_folder_name From 62a28c507dfbb8546cd0d9f83e565d11cd97255d Mon Sep 17 00:00:00 2001 From: mariammartins Date: Mon, 17 Jun 2024 16:33:07 -0300 Subject: [PATCH 3/8] restore helper/ and test/ original files --- build/tf-wrapper.sh | 24 +++--- helpers/foundation-deployer/README.md | 2 +- .../foundation-deployer/global.tfvars.example | 2 +- helpers/foundation-deployer/main.go | 10 +-- helpers/foundation-deployer/stages/apply.go | 20 +---- helpers/foundation-deployer/stages/data.go | 24 ++---- helpers/foundation-deployer/stages/destroy.go | 4 +- test/disable_tf_files.sh | 31 +++++--- test/integration/app-infra/app_infra_test.go | 10 +-- .../projects-shared/projects_shared_test.go | 11 ++- test/integration/projects/projects_test.go | 39 ++++++--- test/restore_tf_files.sh | 79 ++++++++++++------- 12 files changed, 142 insertions(+), 114 deletions(-) diff --git a/build/tf-wrapper.sh b/build/tf-wrapper.sh index fe7d36e1..1b39b6e8 100755 --- a/build/tf-wrapper.sh +++ b/build/tf-wrapper.sh @@ -41,15 +41,15 @@ min_depth=1 # Must be configured based in your directory design # additional special value "shared" # # When using environments as root nodes the regex contains the name of the -# folder that contain the Terraform configuration e.g: ml_business_unit_1 -# and ml_business_unit_2 +# folder that contain the Terraform configuration e.g: business_unit_1 +# and business_unit_2 #==============================================================================# # Environments as leaf nodes in source code case leaf_regex_plan="^(development|non-production|production|shared)$" # Environments as root nodes in source code case -# leaf_regex_plan="^(ml_business_unit_1|ml_business_unit_2)$" +# leaf_regex_plan="^(business_unit_1|business_unit_2)$" #====================================================================# # Function used for the criteria for running terraform int/plan/show @@ -69,11 +69,11 @@ do_plan() { # Environments as leaf nodes in source code case (Default) # Example: # git-repo -# └── ml_business_unit_1 +# └── business_unit_1 # ├── development # ├── non-production # └── production -# └── ml_business_unit_2 +# └── business_unit_2 # ├── development # ├── non-production # └── production @@ -96,19 +96,19 @@ do_action() { # Example: # git-repo # └── development -# ├── ml_business_unit_1 -# └── ml_business_unit_2 +# ├── business_unit_1 +# └── business_unit_2 # └── non-production -# ├── ml_business_unit_1 -# └── ml_business_unit_2 +# ├── business_unit_1 +# └── business_unit_2 # └── production -# ├── ml_business_unit_1 -# └── ml_business_unit_2 +# ├── business_unit_1 +# └── business_unit_2 #=============================================================# ##### Start of alternative source organization - uncomment to use Environments as root nodes ##### -# leaf_regex_action="^(ml_business_unit_1|ml_business_unit_2)$" # edit this list +# leaf_regex_action="^(business_unit_1|business_unit_2)$" # edit this list # do_action() { # local env_path="$1" # local tf_env="${env_path#$base_dir/}" diff --git a/helpers/foundation-deployer/README.md b/helpers/foundation-deployer/README.md index 4217512a..22c06132 100644 --- a/helpers/foundation-deployer/README.md +++ b/helpers/foundation-deployer/README.md @@ -123,7 +123,7 @@ Im addition to the variables declared in the file `global.tfvars` for configurin ```text deploy-directory/ - └── ml-example-app + └── bu1-example-app └── gcp-bootstrap └── gcp-environments └── gcp-networks diff --git a/helpers/foundation-deployer/global.tfvars.example b/helpers/foundation-deployer/global.tfvars.example index f161423a..5da7ab8b 100644 --- a/helpers/foundation-deployer/global.tfvars.example +++ b/helpers/foundation-deployer/global.tfvars.example @@ -149,7 +149,7 @@ target_name_server_addresses = [ // 4-projects inputs -// https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/ml_business_unit/production/README.md#inputs +// https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/business_unit_1/production/README.md#inputs projects_gcs_location = "US" projects_kms_location = "us" diff --git a/helpers/foundation-deployer/main.go b/helpers/foundation-deployer/main.go index 01bd3c8a..5b6665c7 100644 --- a/helpers/foundation-deployer/main.go +++ b/helpers/foundation-deployer/main.go @@ -163,8 +163,8 @@ func main() { // Note: destroy is only terraform destroy, local directories are not deleted. // 5-app-infra msg.PrintStageMsg("Destroying 5-app-infra stage") - err = s.RunDestroyStep("ml-example-app", func() error { - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") + err = s.RunDestroyStep("bu1-example-app", func() error { + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") return stages.DestroyExampleAppStage(t, s, io, conf) }) if err != nil { @@ -302,16 +302,16 @@ func main() { // 5-app-infra msg.PrintStageMsg("Deploying 5-app-infra stage") - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") io.RemoteStateBucket = bo.RemoteStateBucketProjects msg.PrintBuildMsg(io.InfraPipeProj, io.DefaultRegion, conf.DisablePrompt) - err = s.RunStep("ml-example-app", func() error { + err = s.RunStep("bu1-example-app", func() error { return stages.DeployExampleAppStage(t, s, globalTFVars, io, conf) }) if err != nil { fmt.Printf("# Example app step failed. Error: %s\n", err.Error()) os.Exit(3) } -} +} \ No newline at end of file diff --git a/helpers/foundation-deployer/stages/apply.go b/helpers/foundation-deployer/stages/apply.go index 6d92ea83..09684806 100644 --- a/helpers/foundation-deployer/stages/apply.go +++ b/helpers/foundation-deployer/stages/apply.go @@ -1,17 +1,3 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package stages import ( @@ -375,7 +361,7 @@ func DeployProjectsStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, outpu Repo: ProjectsRepo, GitConf: conf, HasManualStep: true, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1", "business_unit_2"}, Envs: []string{"production", "non-production", "development"}, } @@ -395,7 +381,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out } // update backend bucket for _, e := range []string{"production", "non-production", "development"} { - err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "ml_business_unit", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) + err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "business_unit_1", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) if err != nil { return err } @@ -404,7 +390,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out gcpPoliciesPath := filepath.Join(c.CheckoutPath, "gcp-policies-app-infra") policiesConf := utils.CloneCSR(t, PoliciesRepo, gcpPoliciesPath, outputs.InfraPipeProj, c.Logger) policiesBranch := "main" - err = s.RunStep("ml-example-app.gcp-policies-app-infra", func() error { + err = s.RunStep("bu1-example-app.gcp-policies-app-infra", func() error { return preparePoliciesRepo(policiesConf, policiesBranch, c.FoundationPath, gcpPoliciesPath) }) if err != nil { diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index e62bb54f..776d2b75 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -1,17 +1,3 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package stages import ( @@ -34,7 +20,7 @@ const ( EnvironmentsRepo = "gcp-environments" NetworksRepo = "gcp-networks" ProjectsRepo = "gcp-projects" - AppInfraRepo = "ml-example-app" + AppInfraRepo = "bu1-example-app" BootstrapStep = "0-bootstrap" OrgStep = "1-org" EnvironmentsStep = "2-environments" @@ -286,15 +272,15 @@ func GetBootstrapStepOutputs(t testing.TB, foundationPath string) BootstrapOutpu func GetInfraPipelineOutputs(t testing.TB, checkoutPath, workspace string) InfraPipelineOutputs { options := &terraform.Options{ - TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "ml_business_unit", "shared"), + TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "business_unit_1", "shared"), Logger: logger.Discard, NoColor: true, } return InfraPipelineOutputs{ InfraPipeProj: terraform.Output(t, options, "cloudbuild_project_id"), DefaultRegion: terraform.Output(t, options, "default_region"), - TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["ml-example-app"], - StateBucket: terraform.OutputMap(t, options, "state_buckets")["ml-example-app"], + TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["bu1-example-app"], + StateBucket: terraform.OutputMap(t, options, "state_buckets")["bu1-example-app"], } } @@ -320,4 +306,4 @@ func GetNetworkStep(enableHubAndSpoke bool) string { return HubAndSpokeStep } return DualSvpcStep -} +} \ No newline at end of file diff --git a/helpers/foundation-deployer/stages/destroy.go b/helpers/foundation-deployer/stages/destroy.go index f75e7560..250dc512 100644 --- a/helpers/foundation-deployer/stages/destroy.go +++ b/helpers/foundation-deployer/stages/destroy.go @@ -135,7 +135,7 @@ func DestroyProjectsStage(t testing.TB, s steps.Steps, outputs BootstrapOutputs, Step: ProjectsStep, Repo: ProjectsRepo, HasManualStep: true, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1", "business_unit_2"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) @@ -148,7 +148,7 @@ func DestroyExampleAppStage(t testing.TB, s steps.Steps, outputs InfraPipelineOu CICDProject: outputs.InfraPipeProj, Step: AppInfraStep, Repo: AppInfraRepo, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) diff --git a/test/disable_tf_files.sh b/test/disable_tf_files.sh index 4d3c612d..9e8f8a73 100755 --- a/test/disable_tf_files.sh +++ b/test/disable_tf_files.sh @@ -56,31 +56,40 @@ function shared(){ function projectsshared(){ # disable shared.auto.tfvars - mv 4-projects/ml_business_unit/shared/shared.auto.tfvars 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled + mv 4-projects/business_unit_1/shared/shared.auto.tfvars 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled + mv 4-projects/business_unit_2/shared/shared.auto.tfvars 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled # disable common.auto.tfvars - mv 4-projects/ml_business_unit/shared/common.auto.tfvars 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/shared/common.auto.tfvars 4-projects/business_unit_1/shared/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/shared/common.auto.tfvars 4-projects/business_unit_2/shared/common.auto.tfvars.disabled } function projects(){ # disable ENVS.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/development.auto.tfvars 4-projects/ml_business_unit/development/development.auto.tfvars.disabled - mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled - mv 4-projects/ml_business_unit/production/production.auto.tfvars 4-projects/ml_business_unit/production/production.auto.tfvars.disabled + mv 4-projects/business_unit_1/development/development.auto.tfvars 4-projects/business_unit_1/development/development.auto.tfvars.disabled + mv 4-projects/business_unit_2/development/development.auto.tfvars 4-projects/business_unit_2/development/development.auto.tfvars.disabled + mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled + mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled + mv 4-projects/business_unit_1/production/production.auto.tfvars 4-projects/business_unit_1/production/production.auto.tfvars.disabled + mv 4-projects/business_unit_2/production/production.auto.tfvars 4-projects/business_unit_2/production/production.auto.tfvars.disabled # disable common.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/common.auto.tfvars 4-projects/ml_business_unit/development/common.auto.tfvars.disabled - mv 4-projects/ml_business_unit/non-production/common.auto.tfvars 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled - mv 4-projects/ml_business_unit/production/common.auto.tfvars 4-projects/ml_business_unit/production/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/development/common.auto.tfvars 4-projects/business_unit_1/development/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/non-production/common.auto.tfvars 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/production/common.auto.tfvars 4-projects/business_unit_1/production/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/development/common.auto.tfvars 4-projects/business_unit_2/development/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/non-production/common.auto.tfvars 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/production/common.auto.tfvars 4-projects/business_unit_2/production/common.auto.tfvars.disabled } function appinfra(){ # disable common.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/common.auto.tfvars 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled - mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled - mv 5-app-infra/ml_business_unit/production/common.auto.tfvars 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/development/common.auto.tfvars 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/production/common.auto.tfvars 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled } + # parse args for arg in "$@" do diff --git a/test/integration/app-infra/app_infra_test.go b/test/integration/app-infra/app_infra_test.go index ea57c78f..b07cf1b2 100644 --- a/test/integration/app-infra/app_infra_test.go +++ b/test/integration/app-infra/app_infra_test.go @@ -36,12 +36,12 @@ func TestAppInfra(t *testing.T) { } shared := tft.NewTFBlueprintTest(t, - tft.WithTFDir("../../../4-projects/ml_business_unit/shared"), + tft.WithTFDir("../../../4-projects/business_unit_1/shared"), ) // Configure impersonation for test execution - terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["ml-example-app"] - backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["ml-example-app"] + terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["bu1-example-app"] + backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["bu1-example-app"] utils.SetEnv(t, "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT", terraformSA) backendConfig := map[string]interface{}{ "bucket": backend_bucket, @@ -55,11 +55,11 @@ func TestAppInfra(t *testing.T) { t.Run(envName, func(t *testing.T) { projects := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../4-projects/ml_business_unit/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../4-projects/business_unit_1/%s", envName)), ) appInfra := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/ml_business_unit/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/business_unit_1/%s", envName)), tft.WithBackendConfig(backendConfig), tft.WithPolicyLibraryPath("/workspace/policy-library", projects.GetStringOutput("base_shared_vpc_project")), tft.WithVars(vars), diff --git a/test/integration/projects-shared/projects_shared_test.go b/test/integration/projects-shared/projects_shared_test.go index db3d6df8..f25f47c1 100644 --- a/test/integration/projects-shared/projects_shared_test.go +++ b/test/integration/projects-shared/projects_shared_test.go @@ -56,9 +56,14 @@ func TestProjectsShared(t *testing.T) { tfDir string }{ { - name: "ml", - repo: "ml-example-app", - tfDir: "../../../4-projects/ml_business_unit/shared", + name: "bu1", + repo: "bu1-example-app", + tfDir: "../../../4-projects/business_unit_1/shared", + }, + { + name: "bu2", + repo: "bu2-example-app", + tfDir: "../../../4-projects/business_unit_2/shared", }, } { t.Run(tts.name, func(t *testing.T) { diff --git a/test/integration/projects/projects_test.go b/test/integration/projects/projects_test.go index 6c9d65d5..e7ecd2db 100644 --- a/test/integration/projects/projects_test.go +++ b/test/integration/projects/projects_test.go @@ -78,23 +78,44 @@ func TestProjects(t *testing.T) { restrictedNetwork string }{ { - name: "ml_development", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_development", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), }, { - name: "ml_non-production", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_non-production", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), }, { - name: "ml_production", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_production", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", + baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), + }, + { + name: "bu2_development", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", + baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), + }, + { + name: "bu2_non-production", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", + baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), + }, + { + name: "bu2_production", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), }, diff --git a/test/restore_tf_files.sh b/test/restore_tf_files.sh index 62b85559..d7022b09 100644 --- a/test/restore_tf_files.sh +++ b/test/restore_tf_files.sh @@ -76,49 +76,70 @@ function shared(){ function projects(){ # restore backend configs in main module - mv 4-projects/ml_business_unit/development/backend.tf.disabled 4-projects/ml_business_unit/development/backend.tf - mv 4-projects/ml_business_unit/non-production/backend.tf.disabled 4-projects/ml_business_unit/non-production/backend.tf - mv 4-projects/ml_business_unit/production/backend.tf.disabled 4-projects/ml_business_unit/production/backend.tf - mv 4-projects/ml_business_unit/shared/backend.tf.disabled 4-projects/ml_business_unit/shared/backend.tf + mv 4-projects/business_unit_1/development/backend.tf.disabled 4-projects/business_unit_1/development/backend.tf + mv 4-projects/business_unit_1/non-production/backend.tf.disabled 4-projects/business_unit_1/non-production/backend.tf + mv 4-projects/business_unit_1/production/backend.tf.disabled 4-projects/business_unit_1/production/backend.tf + mv 4-projects/business_unit_1/shared/backend.tf.disabled 4-projects/business_unit_1/shared/backend.tf + mv 4-projects/business_unit_2/development/backend.tf.disabled 4-projects/business_unit_2/development/backend.tf + mv 4-projects/business_unit_2/non-production/backend.tf.disabled 4-projects/business_unit_2/non-production/backend.tf + mv 4-projects/business_unit_2/production/backend.tf.disabled 4-projects/business_unit_2/production/backend.tf + mv 4-projects/business_unit_2/shared/backend.tf.disabled 4-projects/business_unit_2/shared/backend.tf # restore access_context.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/development/access_context.auto.tfvars - mv 4-projects/ml_business_unit/non-production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/access_context.auto.tfvars - mv 4-projects/ml_business_unit/production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/production/access_context.auto.tfvars - - # restore ml_business_unit.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars - mv 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars - mv 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars + mv 4-projects/business_unit_1/development/access_context.auto.tfvars.disabled 4-projects/business_unit_1/development/access_context.auto.tfvars + mv 4-projects/business_unit_1/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/non-production/access_context.auto.tfvars + mv 4-projects/business_unit_1/production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/production/access_context.auto.tfvars + mv 4-projects/business_unit_2/development/access_context.auto.tfvars.disabled 4-projects/business_unit_2/development/access_context.auto.tfvars + mv 4-projects/business_unit_2/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/non-production/access_context.auto.tfvars + mv 4-projects/business_unit_2/production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/production/access_context.auto.tfvars + + # restore business_unit_1.auto.tfvars in main module + mv 4-projects/business_unit_1/development/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/development/business_unit_1.auto.tfvars + mv 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars + mv 4-projects/business_unit_1/production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/production/business_unit_1.auto.tfvars + + # restore business_unit_2.auto.tfvars in main module + mv 4-projects/business_unit_2/development/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/development/business_unit_2.auto.tfvars + mv 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars + mv 4-projects/business_unit_2/production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/production/business_unit_2.auto.tfvars # restore ENVS.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/development.auto.tfvars.disabled 4-projects/ml_business_unit/development/development.auto.tfvars - mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/non-production.auto.tfvars - mv 4-projects/ml_business_unit/production/production.auto.tfvars.disabled 4-projects/ml_business_unit/production/production.auto.tfvars - mv 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled 4-projects/ml_business_unit/shared/shared.auto.tfvars + mv 4-projects/business_unit_1/development/development.auto.tfvars.disabled 4-projects/business_unit_1/development/development.auto.tfvars + mv 4-projects/business_unit_2/development/development.auto.tfvars.disabled 4-projects/business_unit_2/development/development.auto.tfvars + mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_1/non-production/non-production.auto.tfvars + mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_2/non-production/non-production.auto.tfvars + mv 4-projects/business_unit_1/production/production.auto.tfvars.disabled 4-projects/business_unit_1/production/production.auto.tfvars + mv 4-projects/business_unit_2/production/production.auto.tfvars.disabled 4-projects/business_unit_2/production/production.auto.tfvars + mv 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled 4-projects/business_unit_1/shared/shared.auto.tfvars + mv 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled 4-projects/business_unit_2/shared/shared.auto.tfvars # restore common.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/common.auto.tfvars.disabled 4-projects/ml_business_unit/development/common.auto.tfvars - mv 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/common.auto.tfvars - mv 4-projects/ml_business_unit/production/common.auto.tfvars.disabled 4-projects/ml_business_unit/production/common.auto.tfvars - mv 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled 4-projects/ml_business_unit/shared/common.auto.tfvars + mv 4-projects/business_unit_1/development/common.auto.tfvars.disabled 4-projects/business_unit_1/development/common.auto.tfvars + mv 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled 4-projects/business_unit_1/non-production/common.auto.tfvars + mv 4-projects/business_unit_1/production/common.auto.tfvars.disabled 4-projects/business_unit_1/production/common.auto.tfvars + mv 4-projects/business_unit_1/shared/common.auto.tfvars.disabled 4-projects/business_unit_1/shared/common.auto.tfvars + mv 4-projects/business_unit_2/development/common.auto.tfvars.disabled 4-projects/business_unit_2/development/common.auto.tfvars + mv 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled 4-projects/business_unit_2/non-production/common.auto.tfvars + mv 4-projects/business_unit_2/production/common.auto.tfvars.disabled 4-projects/business_unit_2/production/common.auto.tfvars + mv 4-projects/business_unit_2/shared/common.auto.tfvars.disabled 4-projects/business_unit_2/shared/common.auto.tfvars + } function appinfra(){ # restore backend configs in main module - mv 5-app-infra/ml_business_unit/development/backend.tf.disabled 5-app-infra/ml_business_unit/development/backend.tf - mv 5-app-infra/ml_business_unit/non-production/backend.tf.disabled 5-app-infra/ml_business_unit/non-production/backend.tf - mv 5-app-infra/ml_business_unit/production/backend.tf.disabled 5-app-infra/ml_business_unit/production/backend.tf + mv 5-app-infra/business_unit_1/development/backend.tf.disabled 5-app-infra/business_unit_1/development/backend.tf + mv 5-app-infra/business_unit_1/non-production/backend.tf.disabled 5-app-infra/business_unit_1/non-production/backend.tf + mv 5-app-infra/business_unit_1/production/backend.tf.disabled 5-app-infra/business_unit_1/production/backend.tf # restore ENVS.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars - mv 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars - mv 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars + mv 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars.disabled 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars + mv 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars + mv 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars.disabled 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars # restore common.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/common.auto.tfvars - mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/common.auto.tfvars - mv 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/common.auto.tfvars + mv 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled 5-app-infra/business_unit_1/development/common.auto.tfvars + mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/common.auto.tfvars + mv 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/production/common.auto.tfvars } From 6a3f42b825c2f37af6df4b4def390164a23c8bb4 Mon Sep 17 00:00:00 2001 From: mariammartins Date: Mon, 17 Jun 2024 16:57:18 -0300 Subject: [PATCH 4/8] fix 5-app-infra step description --- 0-bootstrap/README.md | 2 +- 1-org/README.md | 2 +- 2-environments/README.md | 2 +- 3-networks-dual-svpc/README.md | 2 +- 4-projects/README.md | 6 +----- 7-vertexpipeline/Readme.md | 1 - helpers/foundation-deployer/main.go | 2 +- helpers/foundation-deployer/stages/data.go | 2 +- 8 files changed, 7 insertions(+), 12 deletions(-) diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 17bfbf64..d6d3e6c1 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the machine learning projects using the infra pipeline setup in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/1-org/README.md b/1-org/README.md index 0b11ea41..c774aabe 100644 --- a/1-org/README.md +++ b/1-org/README.md @@ -45,7 +45,7 @@ hub-and-spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/2-environments/README.md b/2-environments/README.md index ccabb5fd..7218cec0 100644 --- a/2-environments/README.md +++ b/2-environments/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/3-networks-dual-svpc/README.md b/3-networks-dual-svpc/README.md index 8db18575..213c95ad 100644 --- a/3-networks-dual-svpc/README.md +++ b/3-networks-dual-svpc/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/4-projects/README.md b/4-projects/README.md index 9051c974..981c7a1f 100644 --- a/4-projects/README.md +++ b/4-projects/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy a service catalog and artifacts pipeline. @@ -60,10 +60,6 @@ The purpose of this step is to set up the folder structure, projects, and infras For machine learning business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. This step follows the same [conventions](https://github.com/terraform-google-modules/terraform-google-enterprise-genai#branching-strategy) as the Foundation pipeline deployed in [0-bootstrap](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/0-bootstrap/README.md). -A custom [workspace](https://github.com/terraform-google-modules/terraform-google-bootstrap/blob/master/modules/tf_cloudbuild_workspace/README.md) (`ml-example-app`) is created by this pipeline and necessary roles are granted to the Terraform Service Account of this workspace by enabling variable `sa_roles` as shown in this [example](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/modules/base_env/example_base_shared_vpc_project.tf). - -This pipeline is utilized to deploy resources in projects across development/non-production/production in step [5-app-infra](../5-app-infra/README.md). -Other Workspaces can also be created to isolate deployments if needed. ## Prerequisites diff --git a/7-vertexpipeline/Readme.md b/7-vertexpipeline/Readme.md index ab5fd41c..3c775865 100644 --- a/7-vertexpipeline/Readme.md +++ b/7-vertexpipeline/Readme.md @@ -1,6 +1,5 @@ Machine learning pipeline from development to production - # Use case This example illustrates the promotion of a a machine learning pipeline from an interactive tenant to a production tenant. The example specifically trains a model on a [UCI census dataset](%28https://archive.ics.uci.edu/dataset/20/census+income%29) for binary classification. diff --git a/helpers/foundation-deployer/main.go b/helpers/foundation-deployer/main.go index 5b6665c7..1475a645 100644 --- a/helpers/foundation-deployer/main.go +++ b/helpers/foundation-deployer/main.go @@ -314,4 +314,4 @@ func main() { fmt.Printf("# Example app step failed. Error: %s\n", err.Error()) os.Exit(3) } -} \ No newline at end of file +} diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index 776d2b75..61c8292f 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -306,4 +306,4 @@ func GetNetworkStep(enableHubAndSpoke bool) string { return HubAndSpokeStep } return DualSvpcStep -} \ No newline at end of file +} From dda3832328b4226b6f6418f55c45cf7ae2176264 Mon Sep 17 00:00:00 2001 From: mariammartins Date: Mon, 17 Jun 2024 20:44:24 -0300 Subject: [PATCH 5/8] fix headers --- 0-bootstrap/README.md | 2 +- 1-org/README.md | 2 +- 2-environments/README.md | 2 +- 3-networks-dual-svpc/README.md | 2 +- 4-projects/README.md | 3 +- 7-vertexpipeline/Readme.md | 4 - helpers/foundation-deployer/README.md | 2 +- helpers/foundation-deployer/main.go | 8 +- helpers/foundation-deployer/stages/apply.go | 6 +- helpers/foundation-deployer/stages/data.go | 8 +- helpers/foundation-deployer/stages/destroy.go | 4 +- test/disable_tf_files.sh | 30 ++++--- test/integration/app-infra/app_infra_test.go | 10 +-- .../projects-shared/projects_shared_test.go | 11 ++- test/integration/projects/projects_test.go | 39 ++++++--- test/restore_tf_files.sh | 79 ++++++++++++------- 16 files changed, 130 insertions(+), 82 deletions(-) diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 17bfbf64..7cc6b99d 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the machine learning projects using the infra pipeline setup in 4-projects. +Deploy service catalog and artifacts pipeline. diff --git a/1-org/README.md b/1-org/README.md index 0b11ea41..a97da789 100644 --- a/1-org/README.md +++ b/1-org/README.md @@ -45,7 +45,7 @@ hub-and-spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. diff --git a/2-environments/README.md b/2-environments/README.md index ccabb5fd..f48cb589 100644 --- a/2-environments/README.md +++ b/2-environments/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. diff --git a/3-networks-dual-svpc/README.md b/3-networks-dual-svpc/README.md index 8db18575..b9c6a9b6 100644 --- a/3-networks-dual-svpc/README.md +++ b/3-networks-dual-svpc/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. diff --git a/4-projects/README.md b/4-projects/README.md index 9051c974..45b203ca 100644 --- a/4-projects/README.md +++ b/4-projects/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the machine learning business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. @@ -60,7 +60,6 @@ The purpose of this step is to set up the folder structure, projects, and infras For machine learning business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. This step follows the same [conventions](https://github.com/terraform-google-modules/terraform-google-enterprise-genai#branching-strategy) as the Foundation pipeline deployed in [0-bootstrap](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/0-bootstrap/README.md). -A custom [workspace](https://github.com/terraform-google-modules/terraform-google-bootstrap/blob/master/modules/tf_cloudbuild_workspace/README.md) (`ml-example-app`) is created by this pipeline and necessary roles are granted to the Terraform Service Account of this workspace by enabling variable `sa_roles` as shown in this [example](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/modules/base_env/example_base_shared_vpc_project.tf). This pipeline is utilized to deploy resources in projects across development/non-production/production in step [5-app-infra](../5-app-infra/README.md). Other Workspaces can also be created to isolate deployments if needed. diff --git a/7-vertexpipeline/Readme.md b/7-vertexpipeline/Readme.md index ab5fd41c..389b0399 100644 --- a/7-vertexpipeline/Readme.md +++ b/7-vertexpipeline/Readme.md @@ -19,7 +19,6 @@ In the first step, a bigquery dataset is created using a bigquery operator offer location=region, ) - Note that the default encryption key for bigquery is set after the projecet inflation so you don't have to pass the key in every query. ## Dataflow for data ingestion @@ -188,6 +187,3 @@ Note that the is triggered by cloud build (for the first time) and cloud compose - The bigquery service agent on the non-prod project will need EncryptDecrypt permission on the kms key so that it can create the dataset using the CMEK key. - First, a non-prod service account to take care of components that run in non-prod (dataset creation, dataflow, training, and evaluation). This could simply be the default compute engine service account for the non-prod tenant. This service account needs write permission to upload the trained model from the non-prod bucket to the Vertex environment of prod. - Another service account that has permissions on the prod tenant in order to deploy the model and the model monitoring job. This could simply be the default service account for the prod tenant. This service account will also need read permission on bigquery of non-prod where the data exists so that the monitoring job deployed by this service account in prod - - - diff --git a/helpers/foundation-deployer/README.md b/helpers/foundation-deployer/README.md index 4217512a..22c06132 100644 --- a/helpers/foundation-deployer/README.md +++ b/helpers/foundation-deployer/README.md @@ -123,7 +123,7 @@ Im addition to the variables declared in the file `global.tfvars` for configurin ```text deploy-directory/ - └── ml-example-app + └── bu1-example-app └── gcp-bootstrap └── gcp-environments └── gcp-networks diff --git a/helpers/foundation-deployer/main.go b/helpers/foundation-deployer/main.go index 01bd3c8a..1475a645 100644 --- a/helpers/foundation-deployer/main.go +++ b/helpers/foundation-deployer/main.go @@ -163,8 +163,8 @@ func main() { // Note: destroy is only terraform destroy, local directories are not deleted. // 5-app-infra msg.PrintStageMsg("Destroying 5-app-infra stage") - err = s.RunDestroyStep("ml-example-app", func() error { - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") + err = s.RunDestroyStep("bu1-example-app", func() error { + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") return stages.DestroyExampleAppStage(t, s, io, conf) }) if err != nil { @@ -302,12 +302,12 @@ func main() { // 5-app-infra msg.PrintStageMsg("Deploying 5-app-infra stage") - io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "ml-example-app") + io := stages.GetInfraPipelineOutputs(t, conf.CheckoutPath, "bu1-example-app") io.RemoteStateBucket = bo.RemoteStateBucketProjects msg.PrintBuildMsg(io.InfraPipeProj, io.DefaultRegion, conf.DisablePrompt) - err = s.RunStep("ml-example-app", func() error { + err = s.RunStep("bu1-example-app", func() error { return stages.DeployExampleAppStage(t, s, globalTFVars, io, conf) }) if err != nil { diff --git a/helpers/foundation-deployer/stages/apply.go b/helpers/foundation-deployer/stages/apply.go index 6d92ea83..f78486e7 100644 --- a/helpers/foundation-deployer/stages/apply.go +++ b/helpers/foundation-deployer/stages/apply.go @@ -375,7 +375,7 @@ func DeployProjectsStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, outpu Repo: ProjectsRepo, GitConf: conf, HasManualStep: true, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1", "business_unit_2"}, Envs: []string{"production", "non-production", "development"}, } @@ -395,7 +395,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out } // update backend bucket for _, e := range []string{"production", "non-production", "development"} { - err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "ml_business_unit", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) + err = utils.ReplaceStringInFile(filepath.Join(c.FoundationPath, AppInfraStep, "business_unit_1", e, "backend.tf"), "UPDATE_APP_INFRA_BUCKET", outputs.StateBucket) if err != nil { return err } @@ -404,7 +404,7 @@ func DeployExampleAppStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, out gcpPoliciesPath := filepath.Join(c.CheckoutPath, "gcp-policies-app-infra") policiesConf := utils.CloneCSR(t, PoliciesRepo, gcpPoliciesPath, outputs.InfraPipeProj, c.Logger) policiesBranch := "main" - err = s.RunStep("ml-example-app.gcp-policies-app-infra", func() error { + err = s.RunStep("bu1-example-app.gcp-policies-app-infra", func() error { return preparePoliciesRepo(policiesConf, policiesBranch, c.FoundationPath, gcpPoliciesPath) }) if err != nil { diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index e62bb54f..59f90df2 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -34,7 +34,7 @@ const ( EnvironmentsRepo = "gcp-environments" NetworksRepo = "gcp-networks" ProjectsRepo = "gcp-projects" - AppInfraRepo = "ml-example-app" + AppInfraRepo = "bu1-example-app" BootstrapStep = "0-bootstrap" OrgStep = "1-org" EnvironmentsStep = "2-environments" @@ -286,15 +286,15 @@ func GetBootstrapStepOutputs(t testing.TB, foundationPath string) BootstrapOutpu func GetInfraPipelineOutputs(t testing.TB, checkoutPath, workspace string) InfraPipelineOutputs { options := &terraform.Options{ - TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "ml_business_unit", "shared"), + TerraformDir: filepath.Join(checkoutPath, "gcp-projects", "business_unit_1", "shared"), Logger: logger.Discard, NoColor: true, } return InfraPipelineOutputs{ InfraPipeProj: terraform.Output(t, options, "cloudbuild_project_id"), DefaultRegion: terraform.Output(t, options, "default_region"), - TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["ml-example-app"], - StateBucket: terraform.OutputMap(t, options, "state_buckets")["ml-example-app"], + TerraformSA: terraform.OutputMap(t, options, "terraform_service_accounts")["bu1-example-app"], + StateBucket: terraform.OutputMap(t, options, "state_buckets")["bu1-example-app"], } } diff --git a/helpers/foundation-deployer/stages/destroy.go b/helpers/foundation-deployer/stages/destroy.go index f75e7560..ecdacd5b 100644 --- a/helpers/foundation-deployer/stages/destroy.go +++ b/helpers/foundation-deployer/stages/destroy.go @@ -135,7 +135,7 @@ func DestroyProjectsStage(t testing.TB, s steps.Steps, outputs BootstrapOutputs, Step: ProjectsStep, Repo: ProjectsRepo, HasManualStep: true, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1", "business_unit_2"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) @@ -148,7 +148,7 @@ func DestroyExampleAppStage(t testing.TB, s steps.Steps, outputs InfraPipelineOu CICDProject: outputs.InfraPipeProj, Step: AppInfraStep, Repo: AppInfraRepo, - GroupingUnits: []string{"ml_business_unit"}, + GroupingUnits: []string{"business_unit_1", "business_unit_2"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) diff --git a/test/disable_tf_files.sh b/test/disable_tf_files.sh index 4d3c612d..0102ff58 100755 --- a/test/disable_tf_files.sh +++ b/test/disable_tf_files.sh @@ -56,29 +56,37 @@ function shared(){ function projectsshared(){ # disable shared.auto.tfvars - mv 4-projects/ml_business_unit/shared/shared.auto.tfvars 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled + mv 4-projects/business_unit_1/shared/shared.auto.tfvars 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled + mv 4-projects/business_unit_2/shared/shared.auto.tfvars 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled # disable common.auto.tfvars - mv 4-projects/ml_business_unit/shared/common.auto.tfvars 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/shared/common.auto.tfvars 4-projects/business_unit_1/shared/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/shared/common.auto.tfvars 4-projects/business_unit_2/shared/common.auto.tfvars.disabled } function projects(){ # disable ENVS.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/development.auto.tfvars 4-projects/ml_business_unit/development/development.auto.tfvars.disabled - mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled - mv 4-projects/ml_business_unit/production/production.auto.tfvars 4-projects/ml_business_unit/production/production.auto.tfvars.disabled + mv 4-projects/business_unit_1/development/development.auto.tfvars 4-projects/business_unit_1/development/development.auto.tfvars.disabled + mv 4-projects/business_unit_2/development/development.auto.tfvars 4-projects/business_unit_2/development/development.auto.tfvars.disabled + mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled + mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled + mv 4-projects/business_unit_1/production/production.auto.tfvars 4-projects/business_unit_1/production/production.auto.tfvars.disabled + mv 4-projects/business_unit_2/production/production.auto.tfvars 4-projects/business_unit_2/production/production.auto.tfvars.disabled # disable common.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/common.auto.tfvars 4-projects/ml_business_unit/development/common.auto.tfvars.disabled - mv 4-projects/ml_business_unit/non-production/common.auto.tfvars 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled - mv 4-projects/ml_business_unit/production/common.auto.tfvars 4-projects/ml_business_unit/production/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/development/common.auto.tfvars 4-projects/business_unit_1/development/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/non-production/common.auto.tfvars 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled + mv 4-projects/business_unit_1/production/common.auto.tfvars 4-projects/business_unit_1/production/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/development/common.auto.tfvars 4-projects/business_unit_2/development/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/non-production/common.auto.tfvars 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled + mv 4-projects/business_unit_2/production/common.auto.tfvars 4-projects/business_unit_2/production/common.auto.tfvars.disabled } function appinfra(){ # disable common.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/common.auto.tfvars 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled - mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled - mv 5-app-infra/ml_business_unit/production/common.auto.tfvars 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/development/common.auto.tfvars 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled + mv 5-app-infra/business_unit_1/production/common.auto.tfvars 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled } # parse args diff --git a/test/integration/app-infra/app_infra_test.go b/test/integration/app-infra/app_infra_test.go index ea57c78f..b07cf1b2 100644 --- a/test/integration/app-infra/app_infra_test.go +++ b/test/integration/app-infra/app_infra_test.go @@ -36,12 +36,12 @@ func TestAppInfra(t *testing.T) { } shared := tft.NewTFBlueprintTest(t, - tft.WithTFDir("../../../4-projects/ml_business_unit/shared"), + tft.WithTFDir("../../../4-projects/business_unit_1/shared"), ) // Configure impersonation for test execution - terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["ml-example-app"] - backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["ml-example-app"] + terraformSA := terraform.OutputMap(t, shared.GetTFOptions(), "terraform_service_accounts")["bu1-example-app"] + backend_bucket := terraform.OutputMap(t, shared.GetTFOptions(), "state_buckets")["bu1-example-app"] utils.SetEnv(t, "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT", terraformSA) backendConfig := map[string]interface{}{ "bucket": backend_bucket, @@ -55,11 +55,11 @@ func TestAppInfra(t *testing.T) { t.Run(envName, func(t *testing.T) { projects := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../4-projects/ml_business_unit/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../4-projects/business_unit_1/%s", envName)), ) appInfra := tft.NewTFBlueprintTest(t, - tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/ml_business_unit/%s", envName)), + tft.WithTFDir(fmt.Sprintf("../../../5-app-infra/business_unit_1/%s", envName)), tft.WithBackendConfig(backendConfig), tft.WithPolicyLibraryPath("/workspace/policy-library", projects.GetStringOutput("base_shared_vpc_project")), tft.WithVars(vars), diff --git a/test/integration/projects-shared/projects_shared_test.go b/test/integration/projects-shared/projects_shared_test.go index db3d6df8..f25f47c1 100644 --- a/test/integration/projects-shared/projects_shared_test.go +++ b/test/integration/projects-shared/projects_shared_test.go @@ -56,9 +56,14 @@ func TestProjectsShared(t *testing.T) { tfDir string }{ { - name: "ml", - repo: "ml-example-app", - tfDir: "../../../4-projects/ml_business_unit/shared", + name: "bu1", + repo: "bu1-example-app", + tfDir: "../../../4-projects/business_unit_1/shared", + }, + { + name: "bu2", + repo: "bu2-example-app", + tfDir: "../../../4-projects/business_unit_2/shared", }, } { t.Run(tts.name, func(t *testing.T) { diff --git a/test/integration/projects/projects_test.go b/test/integration/projects/projects_test.go index 6c9d65d5..e7ecd2db 100644 --- a/test/integration/projects/projects_test.go +++ b/test/integration/projects/projects_test.go @@ -78,23 +78,44 @@ func TestProjects(t *testing.T) { restrictedNetwork string }{ { - name: "ml_development", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_development", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), }, { - name: "ml_non-production", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_non-production", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), }, { - name: "ml_production", - repo: "ml-example-app", - baseDir: "../../../4-projects/ml_business_unit/%s", + name: "bu1_production", + repo: "bu1-example-app", + baseDir: "../../../4-projects/business_unit_1/%s", + baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), + }, + { + name: "bu2_development", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", + baseNetwork: fmt.Sprintf("vpc-d-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-d-shared-restricted%s", networkMode), + }, + { + name: "bu2_non-production", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", + baseNetwork: fmt.Sprintf("vpc-n-shared-base%s", networkMode), + restrictedNetwork: fmt.Sprintf("vpc-n-shared-restricted%s", networkMode), + }, + { + name: "bu2_production", + repo: "bu2-example-app", + baseDir: "../../../4-projects/business_unit_2/%s", baseNetwork: fmt.Sprintf("vpc-p-shared-base%s", networkMode), restrictedNetwork: fmt.Sprintf("vpc-p-shared-restricted%s", networkMode), }, diff --git a/test/restore_tf_files.sh b/test/restore_tf_files.sh index 62b85559..3749ced9 100644 --- a/test/restore_tf_files.sh +++ b/test/restore_tf_files.sh @@ -76,52 +76,71 @@ function shared(){ function projects(){ # restore backend configs in main module - mv 4-projects/ml_business_unit/development/backend.tf.disabled 4-projects/ml_business_unit/development/backend.tf - mv 4-projects/ml_business_unit/non-production/backend.tf.disabled 4-projects/ml_business_unit/non-production/backend.tf - mv 4-projects/ml_business_unit/production/backend.tf.disabled 4-projects/ml_business_unit/production/backend.tf - mv 4-projects/ml_business_unit/shared/backend.tf.disabled 4-projects/ml_business_unit/shared/backend.tf + mv 4-projects/business_unit_1/development/backend.tf.disabled 4-projects/business_unit_1/development/backend.tf + mv 4-projects/business_unit_1/non-production/backend.tf.disabled 4-projects/business_unit_1/non-production/backend.tf + mv 4-projects/business_unit_1/production/backend.tf.disabled 4-projects/business_unit_1/production/backend.tf + mv 4-projects/business_unit_1/shared/backend.tf.disabled 4-projects/business_unit_1/shared/backend.tf + mv 4-projects/business_unit_2/development/backend.tf.disabled 4-projects/business_unit_2/development/backend.tf + mv 4-projects/business_unit_2/non-production/backend.tf.disabled 4-projects/business_unit_2/non-production/backend.tf + mv 4-projects/business_unit_2/production/backend.tf.disabled 4-projects/business_unit_2/production/backend.tf + mv 4-projects/business_unit_2/shared/backend.tf.disabled 4-projects/business_unit_2/shared/backend.tf # restore access_context.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/development/access_context.auto.tfvars - mv 4-projects/ml_business_unit/non-production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/access_context.auto.tfvars - mv 4-projects/ml_business_unit/production/access_context.auto.tfvars.disabled 4-projects/ml_business_unit/production/access_context.auto.tfvars - - # restore ml_business_unit.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/development/ml_business_unit.auto.tfvars - mv 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/ml_business_unit.auto.tfvars - mv 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars.disabled 4-projects/ml_business_unit/production/ml_business_unit.auto.tfvars + mv 4-projects/business_unit_1/development/access_context.auto.tfvars.disabled 4-projects/business_unit_1/development/access_context.auto.tfvars + mv 4-projects/business_unit_1/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/non-production/access_context.auto.tfvars + mv 4-projects/business_unit_1/production/access_context.auto.tfvars.disabled 4-projects/business_unit_1/production/access_context.auto.tfvars + mv 4-projects/business_unit_2/development/access_context.auto.tfvars.disabled 4-projects/business_unit_2/development/access_context.auto.tfvars + mv 4-projects/business_unit_2/non-production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/non-production/access_context.auto.tfvars + mv 4-projects/business_unit_2/production/access_context.auto.tfvars.disabled 4-projects/business_unit_2/production/access_context.auto.tfvars + + # restore business_unit_1.auto.tfvars in main module + mv 4-projects/business_unit_1/development/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/development/business_unit_1.auto.tfvars + mv 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/non-production/business_unit_1.auto.tfvars + mv 4-projects/business_unit_1/production/business_unit_1.auto.tfvars.disabled 4-projects/business_unit_1/production/business_unit_1.auto.tfvars + + # restore business_unit_2.auto.tfvars in main module + mv 4-projects/business_unit_2/development/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/development/business_unit_2.auto.tfvars + mv 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/non-production/business_unit_2.auto.tfvars + mv 4-projects/business_unit_2/production/business_unit_2.auto.tfvars.disabled 4-projects/business_unit_2/production/business_unit_2.auto.tfvars # restore ENVS.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/development.auto.tfvars.disabled 4-projects/ml_business_unit/development/development.auto.tfvars - mv 4-projects/ml_business_unit/non-production/non-production.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/non-production.auto.tfvars - mv 4-projects/ml_business_unit/production/production.auto.tfvars.disabled 4-projects/ml_business_unit/production/production.auto.tfvars - mv 4-projects/ml_business_unit/shared/shared.auto.tfvars.disabled 4-projects/ml_business_unit/shared/shared.auto.tfvars + mv 4-projects/business_unit_1/development/development.auto.tfvars.disabled 4-projects/business_unit_1/development/development.auto.tfvars + mv 4-projects/business_unit_2/development/development.auto.tfvars.disabled 4-projects/business_unit_2/development/development.auto.tfvars + mv 4-projects/business_unit_1/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_1/non-production/non-production.auto.tfvars + mv 4-projects/business_unit_2/non-production/non-production.auto.tfvars.disabled 4-projects/business_unit_2/non-production/non-production.auto.tfvars + mv 4-projects/business_unit_1/production/production.auto.tfvars.disabled 4-projects/business_unit_1/production/production.auto.tfvars + mv 4-projects/business_unit_2/production/production.auto.tfvars.disabled 4-projects/business_unit_2/production/production.auto.tfvars + mv 4-projects/business_unit_1/shared/shared.auto.tfvars.disabled 4-projects/business_unit_1/shared/shared.auto.tfvars + mv 4-projects/business_unit_2/shared/shared.auto.tfvars.disabled 4-projects/business_unit_2/shared/shared.auto.tfvars # restore common.auto.tfvars in main module - mv 4-projects/ml_business_unit/development/common.auto.tfvars.disabled 4-projects/ml_business_unit/development/common.auto.tfvars - mv 4-projects/ml_business_unit/non-production/common.auto.tfvars.disabled 4-projects/ml_business_unit/non-production/common.auto.tfvars - mv 4-projects/ml_business_unit/production/common.auto.tfvars.disabled 4-projects/ml_business_unit/production/common.auto.tfvars - mv 4-projects/ml_business_unit/shared/common.auto.tfvars.disabled 4-projects/ml_business_unit/shared/common.auto.tfvars + mv 4-projects/business_unit_1/development/common.auto.tfvars.disabled 4-projects/business_unit_1/development/common.auto.tfvars + mv 4-projects/business_unit_1/non-production/common.auto.tfvars.disabled 4-projects/business_unit_1/non-production/common.auto.tfvars + mv 4-projects/business_unit_1/production/common.auto.tfvars.disabled 4-projects/business_unit_1/production/common.auto.tfvars + mv 4-projects/business_unit_1/shared/common.auto.tfvars.disabled 4-projects/business_unit_1/shared/common.auto.tfvars + mv 4-projects/business_unit_2/development/common.auto.tfvars.disabled 4-projects/business_unit_2/development/common.auto.tfvars + mv 4-projects/business_unit_2/non-production/common.auto.tfvars.disabled 4-projects/business_unit_2/non-production/common.auto.tfvars + mv 4-projects/business_unit_2/production/common.auto.tfvars.disabled 4-projects/business_unit_2/production/common.auto.tfvars + mv 4-projects/business_unit_2/shared/common.auto.tfvars.disabled 4-projects/business_unit_2/shared/common.auto.tfvars } function appinfra(){ # restore backend configs in main module - mv 5-app-infra/ml_business_unit/development/backend.tf.disabled 5-app-infra/ml_business_unit/development/backend.tf - mv 5-app-infra/ml_business_unit/non-production/backend.tf.disabled 5-app-infra/ml_business_unit/non-production/backend.tf - mv 5-app-infra/ml_business_unit/production/backend.tf.disabled 5-app-infra/ml_business_unit/production/backend.tf + mv 5-app-infra/business_unit_1/development/backend.tf.disabled 5-app-infra/business_unit_1/development/backend.tf + mv 5-app-infra/business_unit_1/non-production/backend.tf.disabled 5-app-infra/business_unit_1/non-production/backend.tf + mv 5-app-infra/business_unit_1/production/backend.tf.disabled 5-app-infra/business_unit_1/production/backend.tf # restore ENVS.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/ml-development.auto.tfvars - mv 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/ml-non-production.auto.tfvars - mv 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/ml-production.auto.tfvars + mv 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars.disabled 5-app-infra/business_unit_1/development/bu1-development.auto.tfvars + mv 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/bu1-non-production.auto.tfvars + mv 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars.disabled 5-app-infra/business_unit_1/production/bu1-production.auto.tfvars # restore common.auto.tfvars in main module - mv 5-app-infra/ml_business_unit/development/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/development/common.auto.tfvars - mv 5-app-infra/ml_business_unit/non-production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/non-production/common.auto.tfvars - mv 5-app-infra/ml_business_unit/production/common.auto.tfvars.disabled 5-app-infra/ml_business_unit/production/common.auto.tfvars + mv 5-app-infra/business_unit_1/development/common.auto.tfvars.disabled 5-app-infra/business_unit_1/development/common.auto.tfvars + mv 5-app-infra/business_unit_1/non-production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/non-production/common.auto.tfvars + mv 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/production/common.auto.tfvars } - # parse args for arg in "$@" do From 9ff2fbffe528c529fe10ed3e0ff6ae4d5a22972e Mon Sep 17 00:00:00 2001 From: mariammartins Date: Tue, 18 Jun 2024 09:38:57 -0300 Subject: [PATCH 6/8] fix headers --- helpers/foundation-deployer/stages/apply.go | 14 ++++++++++++++ helpers/foundation-deployer/stages/data.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/helpers/foundation-deployer/stages/apply.go b/helpers/foundation-deployer/stages/apply.go index 09684806..f78486e7 100644 --- a/helpers/foundation-deployer/stages/apply.go +++ b/helpers/foundation-deployer/stages/apply.go @@ -1,3 +1,17 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package stages import ( diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index 61c8292f..59f90df2 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -1,3 +1,17 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package stages import ( From bba88e726fe6bb9150130a82f190361dfcdc7b6d Mon Sep 17 00:00:00 2001 From: mariammartins Date: Thu, 20 Jun 2024 09:45:50 -0300 Subject: [PATCH 7/8] fix lint --- .github/workflows/lint.yaml | 4 ++-- Makefile | 2 +- .../terraform/4-projects/ml_business_unit/shared/README.md | 1 + .../ml_business_unit/shared/README.md | 1 + helpers/foundation-deployer/stages/destroy.go | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index ca36fb98..b6d9f3b8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -48,10 +48,10 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared diff --git a/Makefile b/Makefile index a06f9d1b..52ba3fa8 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker_test_lint: docker run --rm -it \ -e ENABLE_PARALLEL=0 \ -e DISABLE_TFLINT=1 \ - -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared" \ + -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared" \ -v $(CURDIR):/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh diff --git a/docs/assets/terraform/4-projects/ml_business_unit/shared/README.md b/docs/assets/terraform/4-projects/ml_business_unit/shared/README.md index b65cb8a9..e67eff55 100644 --- a/docs/assets/terraform/4-projects/ml_business_unit/shared/README.md +++ b/docs/assets/terraform/4-projects/ml_business_unit/shared/README.md @@ -34,6 +34,7 @@ | service\_catalog\_project\_id | Service Catalog Project ID. | | service\_catalog\_repo\_id | ID of the Service Catalog repository | | service\_catalog\_repo\_name | The name of the Service Catalog repository | +| shared\_level\_keyrings | Keyrings used on shared level project creation | | state\_buckets | GCS Buckets to store TF state | | terraform\_service\_accounts | APP Infra Pipeline Terraform Accounts. | diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md index 849fc76c..55c6e6bb 100644 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md +++ b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md @@ -4,6 +4,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | instance\_region | The region where compute instance will be created. A subnetwork must exists in the instance region. | `string` | n/a | yes | +| log\_bucket | Log bucket to be used by Service Catalog Bucket | `string` | n/a | yes | | remote\_state\_bucket | Backend bucket to load remote state information from previous steps. | `string` | n/a | yes | ## Outputs diff --git a/helpers/foundation-deployer/stages/destroy.go b/helpers/foundation-deployer/stages/destroy.go index ecdacd5b..250dc512 100644 --- a/helpers/foundation-deployer/stages/destroy.go +++ b/helpers/foundation-deployer/stages/destroy.go @@ -148,7 +148,7 @@ func DestroyExampleAppStage(t testing.TB, s steps.Steps, outputs InfraPipelineOu CICDProject: outputs.InfraPipeProj, Step: AppInfraStep, Repo: AppInfraRepo, - GroupingUnits: []string{"business_unit_1", "business_unit_2"}, + GroupingUnits: []string{"business_unit_1"}, Envs: []string{"development", "non-production", "production"}, } return destroyStage(t, stageConf, s, c) From 77df2f224e175ed4b10f6cfc6538beba810cf6e0 Mon Sep 17 00:00:00 2001 From: mariammartins Date: Fri, 21 Jun 2024 14:50:38 -0300 Subject: [PATCH 8/8] fix hardcoded --- .../service-catalog/common.auto.tfvars | 26 -- .../projects/service-catalog/tf-wrapper.sh | 341 ------------------ 2 files changed, 367 deletions(-) delete mode 100644 5-app-infra/projects/service-catalog/common.auto.tfvars delete mode 100755 5-app-infra/projects/service-catalog/tf-wrapper.sh diff --git a/5-app-infra/projects/service-catalog/common.auto.tfvars b/5-app-infra/projects/service-catalog/common.auto.tfvars deleted file mode 100644 index 46ade0d3..00000000 --- a/5-app-infra/projects/service-catalog/common.auto.tfvars +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -instance_region = "us-central1" // should be one of the regions used to create network on step 3-networks - -remote_state_bucket = "bkt-prj-b-seed-7c57-gcp-projects-tfstate" - -# github_ api_ token = "PUT IN TOKEN" - -# github_app_installation_id = "18685983" - -# github_remote_uri = "https://github.com/badal-io/ml-foundations-tf-modules.git" - diff --git a/5-app-infra/projects/service-catalog/tf-wrapper.sh b/5-app-infra/projects/service-catalog/tf-wrapper.sh deleted file mode 100755 index 1b39b6e8..00000000 --- a/5-app-infra/projects/service-catalog/tf-wrapper.sh +++ /dev/null @@ -1,341 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -action=$1 -branch=$2 -policy_source=$3 -project_id=$4 -policy_type=$5 # FILESYSTEM | CLOUDSOURCE -runner_env=$6 # GITHUB | CLOUDBUILD | JENKINS | LOCAL -base_dir=$(pwd) -tmp_plan="${base_dir}/tmp_plan" #if you change this, update build triggers - -#==============================================================================# -# Configuration for the search depth of folders in the source code that -# contains the terraform configurations. -#==============================================================================# -max_depth=1 # Must be configured based in your directory design -min_depth=1 # Must be configured based in your directory design - - -#==============================================================================# -# The regex to find folders that contains the Terraform configurations to apply. -# -# When using environments as leaf nodes (default) the regex contains the there -# branches/environments development, non-production, and "production" and the -# additional special value "shared" -# -# When using environments as root nodes the regex contains the name of the -# folder that contain the Terraform configuration e.g: business_unit_1 -# and business_unit_2 -#==============================================================================# - -# Environments as leaf nodes in source code case -leaf_regex_plan="^(development|non-production|production|shared)$" - -# Environments as root nodes in source code case -# leaf_regex_plan="^(business_unit_1|business_unit_2)$" - -#====================================================================# -# Function used for the criteria for running terraform int/plan/show -# and gcloud beta terraform vet for all the Terraform configurations. -#====================================================================# -do_plan() { - local leaf - leaf="$(basename "$1")" - if [[ "$leaf" =~ $leaf_regex_plan ]] ; then - echo "true" - else - echo "false" - fi -} - -#=========================================================# -# Environments as leaf nodes in source code case (Default) -# Example: -# git-repo -# └── business_unit_1 -# ├── development -# ├── non-production -# └── production -# └── business_unit_2 -# ├── development -# ├── non-production -# └── production -#=========================================================# - -##### Start of default source organization - comment to use Environments as root nodes ##### -do_action() { - local leaf - leaf="$(basename "$1")" - if [[ "$leaf" == "$branch" ]] || [[ "$leaf" == "shared" && "$branch" == "production" ]]; then - echo "true" - else - echo "false" - fi -} -##### End of default source organization ##### - -#=============================================================# -# Environments as root nodes in source code Case (alternative) -# Example: -# git-repo -# └── development -# ├── business_unit_1 -# └── business_unit_2 -# └── non-production -# ├── business_unit_1 -# └── business_unit_2 -# └── production -# ├── business_unit_1 -# └── business_unit_2 -#=============================================================# - -##### Start of alternative source organization - uncomment to use Environments as root nodes ##### - -# leaf_regex_action="^(business_unit_1|business_unit_2)$" # edit this list -# do_action() { -# local env_path="$1" -# local tf_env="${env_path#$base_dir/}" -# local tf_leaf -# local tf_root -# tf_leaf="$(basename "$env_path")" -# tf_root="$(echo "$tf_env" | cut -d/ -f1)" -# if [[ "$tf_leaf" =~ $leaf_regex_action ]] && [[ "$tf_root" == "$branch" ]] ; then -# echo "true" -# else -# if [[ "$tf_leaf" =~ $leaf_regex_action && "$tf_root" == "shared" && "$branch" == "production" ]]; then -# echo "true" -# else -# echo "false" -# fi -# fi -# } - -##### End of alternative source organization ##### - -#====================================================================# -# Function to replace '/' with '-' to convert a path to a file name -#====================================================================# -convert_path() { - echo "$1" | sed -r 's/\//-/g' -} - -## Terraform apply for single environment. -tf_apply() { - local path=$1 - local tf_env="${path#$base_dir/}" - local tf_file - tf_file="$(convert_path "$tf_env")" - echo "*************** TERRAFORM APPLY *******************" - echo " At environment: ${tf_env} " - echo "***************************************************" - if [ -d "$path" ]; then - cd "$path" || exit - terraform apply -no-color -input=false -auto-approve "${tmp_plan}/${tf_file}.tfplan" || exit 1 - cd "$base_dir" || exit - else - echo "ERROR: ${path} does not exist" - fi -} - -## terraform init for single environment. -tf_init() { - local path=$1 - local tf_env="${path#$base_dir/}" - echo "*************** TERRAFORM INIT *******************" - echo " At environment: ${tf_env} " - echo "**************************************************" - if [ -d "$path" ]; then - cd "$path" || exit - terraform init -no-color || exit 11 - cd "$base_dir" || exit - else - echo "ERROR: ${path} does not exist" - fi -} - -## terraform plan for single environment. -tf_plan() { - local path=$1 - local tf_env="${path#$base_dir/}" - local tf_file - tf_file="$(convert_path "$tf_env")" - echo "*************** TERRAFORM PLAN *******************" - echo " At environment: ${tf_env} " - echo "**************************************************" - if [ ! -d "${tmp_plan}" ]; then - mkdir "${tmp_plan}" || exit - fi - if [ -d "$path" ]; then - cd "$path" || exit - terraform plan -no-color -input=false -out "${tmp_plan}/${tf_file}.tfplan" || exit 21 - cd "$base_dir" || exit - else - echo "ERROR: ${path} does not exist" - fi -} - -#============================================================================# -# terraform init/plan/validate for all valid environments matching condition. -#============================================================================# -tf_plan_validate_all() { - local leaf - find "$base_dir" -mindepth 1 -maxdepth 1 -type d \ - -not -path "$base_dir/modules" \ - -not -path "$base_dir/.git" \ - -not -path "$base_dir/.terraform" | while read -r component_path ; do - find "$component_path" -mindepth "$min_depth" -maxdepth "$max_depth" -type d | while read -r env_path ; do - if [[ "$(do_plan "$env_path")" == "true" ]] ; then - tf_init "$env_path" - tf_plan "$env_path" - tf_validate "$env_path" "$policy_source" - else - echo "${env_path#$base_dir/} doesn't match $leaf_regex_plan; skipping" - fi - done - done -} - -## terraform show for single environment. -tf_show() { - local path=$1 - local tf_env="${path#$base_dir/}" - local tf_file - tf_file="$(convert_path "$tf_env")" - echo "*************** TERRAFORM SHOW *******************" - echo " At environment: ${tf_env} " - echo "**************************************************" - if [ -d "$path" ]; then - cd "$path" || exit - terraform show -no-color "${tmp_plan}/${tf_file}.tfplan" || exit 41 - cd "$base_dir" || exit - else - echo "ERROR: ${path} does not exist" - fi -} - -## terraform validate for single environment. -tf_validate() { - local path=$1 - local policy_file_path=$2 - local tf_env="${path#$base_dir/}" - local tf_file - tf_file="$(convert_path "$tf_env")" - echo "*************** TERRAFORM VALIDATE ******************" - echo " At environment: ${tf_env} " - echo " Using policy from: ${policy_file_path} " - echo "*****************************************************" - if [ -z "$policy_file_path" ]; then - echo "no policy repo found! Check the argument provided for policy_source to this script." - echo "https://github.com/GoogleCloudPlatform/policy-library/blob/main/docs/user_guide.md#how-to-set-up-constraints-with-policy-library" - else - if [ -d "$path" ]; then - cd "$path" || exit - # In GitHub actions environment 'terraform' is not the terraform binary but a wrapper around it - # that prints the command 'terraform show' itself in the redirection to the json file, making - # the json file to have an invalid format. 'terraform-bin' is the actual terraform binary. - if [[ "$runner_env" == "GITHUB" ]]; then - terraform-bin show -no-color -json "${tmp_plan}/${tf_file}.tfplan" > "${tf_file}.json" || exit 32 - else - terraform show -no-color -json "${tmp_plan}/${tf_file}.tfplan" > "${tf_file}.json" || exit 32 - fi - if [[ "$policy_type" == "CLOUDSOURCE" ]]; then - # Check if $policy_file_path is empty so we clone the policies repo only once - if [ -z "$(ls -A "${policy_file_path}" 2> /dev/null)" ]; then - gcloud source repos clone gcp-policies "${policy_file_path}" --project="${project_id}" || exit 34 - pushd . - cd "${policy_file_path}" - # Commented command below works only on Git 2.22.0+ - # current_branch=$(git branch --show-current) - # As Cloud Build is based on step 4-projects docker image having - # git version 2.20.1 installed the command below keeps compatibility - current_branch=$(git symbolic-ref --short HEAD) - echo "current gcp-policies branch $current_branch" - if [[ "$current_branch" != "main" ]]; then - git checkout main || exit 35 - fi - popd - fi - fi - gcloud beta terraform vet "${tf_file}.json" --policy-library="${policy_file_path}" --project="${project_id}" || exit 33 - cd "$base_dir" || exit - else - echo "ERROR: ${path} does not exist" - fi - fi -} - -#=================================================================# -# Runs single action for each instance of env in folder hierarchy. -#=================================================================# -single_action_runner() { - local leaf - # filter folders that does not contain Terraform configurations - find "$base_dir" -mindepth 1 -maxdepth 1 -type d \ - -not -path "$base_dir/modules" \ - -not -path "$base_dir/.git" \ - -not -path "$base_dir/.terraform" | while read -r component_path ; do - # sort -r is added to ensure shared is first if it exists. - find "$component_path" -mindepth "$min_depth" -maxdepth "$max_depth" -type d | sort -r | while read -r env_path ; do - if [[ "$(do_action "$env_path")" == "true" ]]; then - case "$action" in - apply ) - tf_apply "$env_path" - ;; - - init ) - tf_init "$env_path" - ;; - - plan ) - tf_plan "$env_path" - ;; - - show ) - tf_show "$env_path" - ;; - - validate ) - tf_validate "$env_path" "$policy_source" - ;; - * ) - echo "unknown option: ${action}" - ;; - esac - else - echo "${env_path#$base_dir/} doesn't match ${branch}; skipping" - fi - done - done -} - -case "$action" in - init|plan|apply|show|validate ) - single_action_runner - ;; - - plan_validate_all ) - tf_plan_validate_all - ;; - - * ) - echo "unknown option: ${1}" - exit 99 - ;; -esac