From 468d69c5f456d8e40bc2ca76a06e72bb11785031 Mon Sep 17 00:00:00 2001 From: Duda Martins <109083465+mariammartins@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:27:10 -0300 Subject: [PATCH] Rename bu3 to ml (#63) * first commit * fix ml_infra_projects name * restore helper/ and test/ original files * fix 5-app-infra step description * fix headers * fix headers * fix lint * fix hardcoded --- .github/workflows/lint.yaml | 4 +- 0-bootstrap/README-GitHub.md | 6 +- 0-bootstrap/README-Jenkins.md | 4 +- 0-bootstrap/README-Terraform-Cloud.md | 15 +-- 0-bootstrap/README.md | 2 +- 0-bootstrap/terraform_cloud.tf.example | 23 +--- 1-org/README.md | 2 +- 2-environments/README.md | 3 +- 3-networks-dual-svpc/README.md | 3 +- 4-projects/README.md | 18 +-- .../development/README.md | 0 .../development}/backend.tf | 2 +- .../development}/backend.tf.cloud.example | 2 +- .../development/common.auto.tfvars | 0 .../development/development.auto.tfvars | 0 .../development}/locals.tf | 6 +- .../development/main.tf | 0 .../development/outputs.tf | 0 .../development/remote.tf | 0 .../development/variables.tf | 0 .../development/versions.tf | 0 .../non-production/README.md | 0 .../non-production/backend.tf | 2 +- .../non-production/backend.tf.cloud.example | 2 +- .../non-production/common.auto.tfvars | 0 .../non-production}/locals.tf | 6 +- .../non-production/main.tf | 0 .../non-production/non-production.auto.tfvars | 0 .../non-production/outputs.tf | 0 .../non-production/outputs.tf.backup | 0 .../non-production/remote.tf | 0 .../non-production/variables.tf | 0 .../production/README.md | 0 .../production}/backend.tf | 2 +- .../production}/backend.tf.cloud.example | 2 +- .../production/common.auto.tfvars | 0 .../production}/locals.tf | 6 +- .../production/main.tf | 0 .../production/outputs.tf | 0 .../production/outputs.tf.backup | 0 .../production/production.auto.tfvars | 0 .../production/remote.tf | 0 .../production/variables.tf | 0 .../shared/README.md | 0 .../shared/backend.tf | 2 +- .../shared/backend.tf.cloud.example | 2 +- .../shared/common.auto.tfvars | 0 .../shared/example_infra_pipeline.tf | 8 +- .../shared/ml_infra_projects.tf | 6 +- .../shared/outputs.tf | 0 .../shared/remote.tf | 0 .../shared/remote.tf.cloud.example | 0 .../shared/shared.auto.tfvars | 0 .../shared/variables.tf | 0 .../shared/versions.tf | 0 4-projects/modules/composer_env/variables.tf | 2 +- 4-projects/modules/env_folders/variables.tf | 2 +- 4-projects/modules/ml_env/example_vertex.tf | 6 +- 4-projects/modules/ml_env/variables.tf | 4 +- 5-app-infra/README.md | 66 +++++----- .../shared/README.md | 1 + .../shared/backend.tf | 2 +- .../shared/common.auto.tfvars | 0 .../shared/locals.tf | 2 +- .../shared/outputs.tf | 0 .../shared/publish_artifacts.tf | 0 .../shared/remote.tf | 0 .../ml_business_unit}/shared/variables.tf | 0 .../shared/versions.tf | 0 .../ml_business_unit/shared/README.md | 16 +++ .../shared/backend.tf | 2 +- .../shared/common.auto.tfvars | 0 .../shared/locals.tf | 2 +- .../shared/outputs.tf | 0 .../shared/remote.tf | 0 .../shared/service_catalog.tf | 0 .../ml_business_unit}/shared/variables.tf | 0 .../shared/versions.tf | 0 .../service-catalog/modules/pubsub/locals.tf | 1 - 7-composer/dag.py | 2 +- .../dags | 2 +- 7-vertexpipeline/Readme.md | 7 +- 7-vertexpipeline/census_pipeline.ipynb | 14 +- 7-vertexpipeline/runpipeline.py | 6 +- ERRATA.md | 2 +- Makefile | 2 +- docs/TROUBLESHOOTING.md | 2 +- .../shared/backend.tf.cloud.example | 2 +- .../modules/env_folders/variables.tf | 2 +- .../4-projects/modules/ml_env/variables.tf | 4 +- .../ml_business_unit}/shared/README.md | 0 docs/change_resource_hierarchy.md | 14 +- examples/machine-learning-pipeline/README.md | 120 +++++++++--------- .../development/README.md | 0 .../development}/backend.tf | 2 +- .../development/common.auto.tfvars | 0 .../development/locals.tf | 4 +- .../development/main.tf | 0 .../development/outputs.tf | 0 .../development/remote.tf | 0 .../development/variables.tf | 0 .../development/versions.tf | 0 .../non-production/README.md | 0 .../non-production/backend.tf | 2 +- .../non-production/common.auto.tfvars | 0 .../non-production/locals.tf | 4 +- .../non-production/main.tf | 0 .../non-production/outputs.tf | 0 .../non-production/remote.tf | 0 .../non-production/variables.tf | 0 .../non-production/versions.tf | 0 .../production/README.md | 0 .../production}/backend.tf | 2 +- .../production/common.auto.tfvars | 0 .../production/locals.tf | 4 +- .../production/main.tf | 0 .../production/outputs.tf | 0 .../production/remote.tf | 0 .../production/variables.tf | 0 .../production/versions.tf | 0 .../modules/base_env/variables.tf | 2 +- test/restore_tf_files.sh | 1 - 122 files changed, 214 insertions(+), 220 deletions(-) rename 4-projects/{business_unit_3 => ml_business_unit}/development/README.md (100%) rename 4-projects/{business_unit_3/production => ml_business_unit/development}/backend.tf (91%) rename 4-projects/{business_unit_3/production => ml_business_unit/development}/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/common.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/development.auto.tfvars (100%) rename 4-projects/{business_unit_3/non-production => ml_business_unit/development}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/development/versions.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/README.md (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/backend.tf (91%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/backend.tf.cloud.example (94%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/common.auto.tfvars (100%) rename 4-projects/{business_unit_3/production => ml_business_unit/non-production}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/non-production.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/outputs.tf.backup (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/non-production/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/README.md (100%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/backend.tf (91%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/common.auto.tfvars (100%) rename 4-projects/{business_unit_3/development => ml_business_unit/production}/locals.tf (86%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/main.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/outputs.tf.backup (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/production.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/production/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/README.md (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/backend.tf (89%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/backend.tf.cloud.example (95%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/common.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/example_infra_pipeline.tf (96%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/ml_infra_projects.tf (91%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/remote.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/remote.tf.cloud.example (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/shared.auto.tfvars (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/variables.tf (100%) rename 4-projects/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/README.md (86%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/backend.tf (91%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/common.auto.tfvars (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/locals.tf (94%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/publish_artifacts.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/remote.tf (100%) rename 5-app-infra/projects/{service-catalog/business_unit_3 => artifact-publish/ml_business_unit}/shared/variables.tf (100%) rename 5-app-infra/projects/artifact-publish/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) create mode 100644 5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/backend.tf (91%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/common.auto.tfvars (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/locals.tf (94%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/outputs.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/remote.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/service_catalog.tf (100%) rename 5-app-infra/projects/{artifact-publish/business_unit_3 => service-catalog/ml_business_unit}/shared/variables.tf (100%) rename 5-app-infra/projects/service-catalog/{business_unit_3 => ml_business_unit}/shared/versions.tf (100%) rename {5-app-infra/projects/service-catalog/business_unit_3 => docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit}/shared/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3/production => ml_business_unit/development}/backend.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/development/versions.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/backend.tf (90%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/non-production/versions.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/README.md (100%) rename examples/machine-learning-pipeline/{business_unit_3/development => ml_business_unit/production}/backend.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/common.auto.tfvars (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/locals.tf (91%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/main.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/outputs.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/remote.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/variables.tf (100%) rename examples/machine-learning-pipeline/{business_unit_3 => ml_business_unit}/production/versions.tf (100%) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index ca36fb98..b6d9f3b8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -48,10 +48,10 @@ jobs: env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh env: DISABLE_TFLINT: 1 ENABLE_PARALLEL: 0 - EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared + EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared diff --git a/0-bootstrap/README-GitHub.md b/0-bootstrap/README-GitHub.md index f7e635e4..cdd6e4d2 100644 --- a/0-bootstrap/README-GitHub.md +++ b/0-bootstrap/README-GitHub.md @@ -843,8 +843,8 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from bootstrap output. @@ -862,7 +862,7 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th git commit -m 'Initialize projects repo' ``` -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. 1. Use `terraform output` to get the CI/CD project ID and the projects step Terraform Service Account from gcp-bootstrap output. 1. The CI/CD project ID will be used in the [validation](https://cloud.google.com/docs/terraform/policy-validation/quickstart) of the Terraform configuration diff --git a/0-bootstrap/README-Jenkins.md b/0-bootstrap/README-Jenkins.md index 77e081ef..b840094d 100644 --- a/0-bootstrap/README-Jenkins.md +++ b/0-bootstrap/README-Jenkins.md @@ -872,8 +872,8 @@ Here you will configure a VPN Network tunnel to enable connectivity between the mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from 0-bootstrap output. ```bash diff --git a/0-bootstrap/README-Terraform-Cloud.md b/0-bootstrap/README-Terraform-Cloud.md index 935f3e78..a7338b3d 100644 --- a/0-bootstrap/README-Terraform-Cloud.md +++ b/0-bootstrap/README-Terraform-Cloud.md @@ -771,16 +771,15 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](../4-projects/ml_business_unit/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/ml_business_unit/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. 1. In order to manually run the apply for shared workspace from your local we need to temporary unset the TFC backend by renaming `envs/shared/backend.tf` to `envs/shared/backend.tf.temporary_disabled`. ```bash - mv business_unit_1/shared/backend.tf business_unit_1/shared/backend.tf.temporary_disabled - mv business_unit_2/shared/backend.tf business_unit_2/shared/backend.tf.temporary_disabled + mv ml_business_unit/shared/backend.tf ml_business_unit/shared/backend.tf.temporary_disabled ``` 1. Use `terraform output` to get the CI/CD project ID and the projects step Terraform Service Account from gcp-bootstrap output. @@ -831,10 +830,8 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th 1. In order to set the TFC backend for shared workspace we now can rename `envs/shared/backend.tf.temporary_disabled` to `envs/shared/backend.tf` and run `terraform init`. When you're prompted, agree to copy Terraform state to Terraform Cloud. ```bash - mv business_unit_1/shared/backend.tf.temporary_disabled business_unit_1/shared/backend.tf - mv business_unit_2/shared/backend.tf.temporary_disabled business_unit_2/shared/backend.tf - terraform -chdir="business_unit_1/shared/" init - terraform -chdir="business_unit_2/shared/" init + mv ml_business_unit/shared/backend.tf.temporary_disabled ml_business_unit/shared/backend.tf + terraform -chdir="ml_business_unit/shared/" init ``` 1. Commit changes diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 7048283f..d6d3e6c1 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the business unit projects using the infra pipeline setup in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/0-bootstrap/terraform_cloud.tf.example b/0-bootstrap/terraform_cloud.tf.example index ce45ab1a..3aa7b54e 100644 --- a/0-bootstrap/terraform_cloud.tf.example +++ b/0-bootstrap/terraform_cloud.tf.example @@ -63,14 +63,10 @@ locals { "3-shared" = { vcs_branch = "production", directory = "/envs/shared" }, }, "proj" = { - "4-bu1-production" = { vcs_branch = "production", directory = "/business_unit_1/production" }, - "4-bu1-non-production" = { vcs_branch = "non-production", directory = "/business_unit_1/non-production" }, - "4-bu1-development" = { vcs_branch = "development", directory = "/business_unit_1/development" }, - "4-bu1-shared" = { vcs_branch = "production", directory = "/business_unit_1/shared" }, - "4-bu2-production" = { vcs_branch = "production", directory = "/business_unit_2/production" }, - "4-bu2-non-production" = { vcs_branch = "non-production", directory = "/business_unit_2/non-production" }, - "4-bu2-development" = { vcs_branch = "development", directory = "/business_unit_2/development" }, - "4-bu2-shared" = { vcs_branch = "production", directory = "/business_unit_2/shared" }, + "4-ml-production" = { vcs_branch = "production", directory = "/ml_business_unit/production" }, + "4-ml-non-production" = { vcs_branch = "non-production", directory = "/ml_business_unit/non-production" }, + "4-ml-development" = { vcs_branch = "development", directory = "/ml_business_unit/development" }, + "4-ml-shared" = { vcs_branch = "production", directory = "/ml_business_unit/shared" }, }, } @@ -218,14 +214,9 @@ resource "tfe_run_trigger" "networks_shared_production" { sourceable_id = tfe_workspace.main["3-shared"].id } -resource "tfe_run_trigger" "projects_bu1_shared_production" { - workspace_id = tfe_workspace.main["4-bu1-production"].id - sourceable_id = tfe_workspace.main["4-bu1-shared"].id -} - -resource "tfe_run_trigger" "projects_bu2_shared_production" { - workspace_id = tfe_workspace.main["4-bu2-production"].id - sourceable_id = tfe_workspace.main["4-bu2-shared"].id +resource "tfe_run_trigger" "projects_ml_shared_production" { + workspace_id = tfe_workspace.main["4-ml-production"].id + sourceable_id = tfe_workspace.main["4-ml-shared"].id } module "tfc_cicd" { diff --git a/1-org/README.md b/1-org/README.md index d528b9db..c774aabe 100644 --- a/1-org/README.md +++ b/1-org/README.md @@ -45,7 +45,7 @@ hub-and-spoke network model. It also sets up the global DNS hub. 5-app-infra -Deploy a Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy a service catalog and artifacts pipeline. diff --git a/2-environments/README.md b/2-environments/README.md index ccabb5fd..fd5a43bd 100644 --- a/2-environments/README.md +++ b/2-environments/README.md @@ -45,7 +45,8 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. +Deploy a service catalog and artifacts pipeline. diff --git a/3-networks-dual-svpc/README.md b/3-networks-dual-svpc/README.md index cb47dcf7..12e0e96c 100644 --- a/3-networks-dual-svpc/README.md +++ b/3-networks-dual-svpc/README.md @@ -45,7 +45,8 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. +Deploy a service catalog and artifacts pipeline. diff --git a/4-projects/README.md b/4-projects/README.md index f5ff9c88..45b203ca 100644 --- a/4-projects/README.md +++ b/4-projects/README.md @@ -45,7 +45,7 @@ Hub and Spoke network model. It also sets up the global DNS hub 5-app-infra -Deploy a simple Compute Engine instance in one of the business unit projects using the infra pipeline set up in 4-projects. +Deploy service catalog and artifacts pipeline. @@ -57,10 +57,9 @@ For an overview of the architecture and the parts, see the The purpose of this step is to set up the folder structure, projects, and infrastructure pipelines for applications that are connected as service projects to the shared VPC created in the previous stage. -For each business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. +For machine learning business unit, a shared `infra-pipeline` project is created along with Cloud Build triggers, CSRs for application infrastructure code and Google Cloud Storage buckets for state storage. This step follows the same [conventions](https://github.com/terraform-google-modules/terraform-google-enterprise-genai#branching-strategy) as the Foundation pipeline deployed in [0-bootstrap](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/0-bootstrap/README.md). -A custom [workspace](https://github.com/terraform-google-modules/terraform-google-bootstrap/blob/master/modules/tf_cloudbuild_workspace/README.md) (`bu1-example-app`) is created by this pipeline and necessary roles are granted to the Terraform Service Account of this workspace by enabling variable `sa_roles` as shown in this [example](https://github.com/terraform-google-modules/terraform-google-enterprise-genai/blob/master/4-projects/modules/base_env/example_base_shared_vpc_project.tf). This pipeline is utilized to deploy resources in projects across development/non-production/production in step [5-app-infra](../5-app-infra/README.md). Other Workspaces can also be created to isolate deployments if needed. @@ -122,8 +121,9 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. -1. See any of the shared folder [README.md](./business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. + +1. See any of the shared folder [README.md](./ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. 1. Use `terraform output` to get the backend bucket value from 0-bootstrap output. @@ -141,8 +141,10 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get git commit -m 'Initialize projects repo' ``` -1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. +1. You need to manually plan and apply only once the `ml_business_unit/shared` environments since `development`, `non-production`, and `production` depend on them. + 1. To use the `validate` option of the `tf-wrapper.sh` script, please follow the [instructions](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) to install the terraform-tools component. + 1. Use `terraform output` to get the Cloud Build project ID and the projects step Terraform Service Account from 0-bootstrap output. An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set using the Terraform Service Account to enable impersonation. ```bash @@ -253,8 +255,8 @@ See `0-bootstrap` [README-GitHub.md](../0-bootstrap/README-GitHub.md#deploying-s mv production.auto.example.tfvars production.auto.tfvars ``` -1. See any of the envs folder [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. - See any of the shared folder [README.md](./business_unit_1/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. +1. See any of the envs folder [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. + See any of the shared folder [README.md](./ml_business_unit/shared/README.md) files for additional information on the values in the `shared.auto.tfvars` file. Use `terraform output` to get the remote state bucket (the backend bucket used by previous steps) value from `0-bootstrap` output. ```bash diff --git a/4-projects/business_unit_3/development/README.md b/4-projects/ml_business_unit/development/README.md similarity index 100% rename from 4-projects/business_unit_3/development/README.md rename to 4-projects/ml_business_unit/development/README.md diff --git a/4-projects/business_unit_3/production/backend.tf b/4-projects/ml_business_unit/development/backend.tf similarity index 91% rename from 4-projects/business_unit_3/production/backend.tf rename to 4-projects/ml_business_unit/development/backend.tf index c035921a..e5ccedf9 100644 --- a/4-projects/business_unit_3/production/backend.tf +++ b/4-projects/ml_business_unit/development/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/production" + prefix = "terraform/projects/ml_business_unit/development" } } diff --git a/4-projects/business_unit_3/production/backend.tf.cloud.example b/4-projects/ml_business_unit/development/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/production/backend.tf.cloud.example rename to 4-projects/ml_business_unit/development/backend.tf.cloud.example index 23cd6e71..c09ed6d6 100644 --- a/4-projects/business_unit_3/production/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/development/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-production" + name = "4-ml-development" } } } diff --git a/4-projects/business_unit_3/development/common.auto.tfvars b/4-projects/ml_business_unit/development/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/development/common.auto.tfvars rename to 4-projects/ml_business_unit/development/common.auto.tfvars diff --git a/4-projects/business_unit_3/development/development.auto.tfvars b/4-projects/ml_business_unit/development/development.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/development/development.auto.tfvars rename to 4-projects/ml_business_unit/development/development.auto.tfvars diff --git a/4-projects/business_unit_3/non-production/locals.tf b/4-projects/ml_business_unit/development/locals.tf similarity index 86% rename from 4-projects/business_unit_3/non-production/locals.tf rename to 4-projects/ml_business_unit/development/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/non-production/locals.tf +++ b/4-projects/ml_business_unit/development/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/development/main.tf b/4-projects/ml_business_unit/development/main.tf similarity index 100% rename from 4-projects/business_unit_3/development/main.tf rename to 4-projects/ml_business_unit/development/main.tf diff --git a/4-projects/business_unit_3/development/outputs.tf b/4-projects/ml_business_unit/development/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/development/outputs.tf rename to 4-projects/ml_business_unit/development/outputs.tf diff --git a/4-projects/business_unit_3/development/remote.tf b/4-projects/ml_business_unit/development/remote.tf similarity index 100% rename from 4-projects/business_unit_3/development/remote.tf rename to 4-projects/ml_business_unit/development/remote.tf diff --git a/4-projects/business_unit_3/development/variables.tf b/4-projects/ml_business_unit/development/variables.tf similarity index 100% rename from 4-projects/business_unit_3/development/variables.tf rename to 4-projects/ml_business_unit/development/variables.tf diff --git a/4-projects/business_unit_3/development/versions.tf b/4-projects/ml_business_unit/development/versions.tf similarity index 100% rename from 4-projects/business_unit_3/development/versions.tf rename to 4-projects/ml_business_unit/development/versions.tf diff --git a/4-projects/business_unit_3/non-production/README.md b/4-projects/ml_business_unit/non-production/README.md similarity index 100% rename from 4-projects/business_unit_3/non-production/README.md rename to 4-projects/ml_business_unit/non-production/README.md diff --git a/4-projects/business_unit_3/non-production/backend.tf b/4-projects/ml_business_unit/non-production/backend.tf similarity index 91% rename from 4-projects/business_unit_3/non-production/backend.tf rename to 4-projects/ml_business_unit/non-production/backend.tf index 3872830c..0adc3933 100644 --- a/4-projects/business_unit_3/non-production/backend.tf +++ b/4-projects/ml_business_unit/non-production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/non-production" + prefix = "terraform/projects/ml_business_unit/non-production" } } diff --git a/4-projects/business_unit_3/non-production/backend.tf.cloud.example b/4-projects/ml_business_unit/non-production/backend.tf.cloud.example similarity index 94% rename from 4-projects/business_unit_3/non-production/backend.tf.cloud.example rename to 4-projects/ml_business_unit/non-production/backend.tf.cloud.example index ccefb79c..c6fa4995 100644 --- a/4-projects/business_unit_3/non-production/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/non-production/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-non-production" + name = "4-ml-non-production" } } } diff --git a/4-projects/business_unit_3/non-production/common.auto.tfvars b/4-projects/ml_business_unit/non-production/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/non-production/common.auto.tfvars rename to 4-projects/ml_business_unit/non-production/common.auto.tfvars diff --git a/4-projects/business_unit_3/production/locals.tf b/4-projects/ml_business_unit/non-production/locals.tf similarity index 86% rename from 4-projects/business_unit_3/production/locals.tf rename to 4-projects/ml_business_unit/non-production/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/production/locals.tf +++ b/4-projects/ml_business_unit/non-production/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/non-production/main.tf b/4-projects/ml_business_unit/non-production/main.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/main.tf rename to 4-projects/ml_business_unit/non-production/main.tf diff --git a/4-projects/business_unit_3/non-production/non-production.auto.tfvars b/4-projects/ml_business_unit/non-production/non-production.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/non-production/non-production.auto.tfvars rename to 4-projects/ml_business_unit/non-production/non-production.auto.tfvars diff --git a/4-projects/business_unit_3/non-production/outputs.tf b/4-projects/ml_business_unit/non-production/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/outputs.tf rename to 4-projects/ml_business_unit/non-production/outputs.tf diff --git a/4-projects/business_unit_3/non-production/outputs.tf.backup b/4-projects/ml_business_unit/non-production/outputs.tf.backup similarity index 100% rename from 4-projects/business_unit_3/non-production/outputs.tf.backup rename to 4-projects/ml_business_unit/non-production/outputs.tf.backup diff --git a/4-projects/business_unit_3/non-production/remote.tf b/4-projects/ml_business_unit/non-production/remote.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/remote.tf rename to 4-projects/ml_business_unit/non-production/remote.tf diff --git a/4-projects/business_unit_3/non-production/variables.tf b/4-projects/ml_business_unit/non-production/variables.tf similarity index 100% rename from 4-projects/business_unit_3/non-production/variables.tf rename to 4-projects/ml_business_unit/non-production/variables.tf diff --git a/4-projects/business_unit_3/production/README.md b/4-projects/ml_business_unit/production/README.md similarity index 100% rename from 4-projects/business_unit_3/production/README.md rename to 4-projects/ml_business_unit/production/README.md diff --git a/4-projects/business_unit_3/development/backend.tf b/4-projects/ml_business_unit/production/backend.tf similarity index 91% rename from 4-projects/business_unit_3/development/backend.tf rename to 4-projects/ml_business_unit/production/backend.tf index d22c4775..f900eb0c 100644 --- a/4-projects/business_unit_3/development/backend.tf +++ b/4-projects/ml_business_unit/production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/development" + prefix = "terraform/projects/ml_business_unit/production" } } diff --git a/4-projects/business_unit_3/development/backend.tf.cloud.example b/4-projects/ml_business_unit/production/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/development/backend.tf.cloud.example rename to 4-projects/ml_business_unit/production/backend.tf.cloud.example index e20a0d82..704da6c2 100644 --- a/4-projects/business_unit_3/development/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/production/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu3-development" + name = "4-ml-production" } } } diff --git a/4-projects/business_unit_3/production/common.auto.tfvars b/4-projects/ml_business_unit/production/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/production/common.auto.tfvars rename to 4-projects/ml_business_unit/production/common.auto.tfvars diff --git a/4-projects/business_unit_3/development/locals.tf b/4-projects/ml_business_unit/production/locals.tf similarity index 86% rename from 4-projects/business_unit_3/development/locals.tf rename to 4-projects/ml_business_unit/production/locals.tf index 1fa2b16a..b10bf71e 100644 --- a/4-projects/business_unit_3/development/locals.tf +++ b/4-projects/ml_business_unit/production/locals.tf @@ -13,7 +13,7 @@ # limitations under the License. # locals { - repo_name = "bu3-composer" - business_code = "bu3" - business_unit = "business_unit_3" + repo_name = "ml-composer" + business_code = "ml" + business_unit = "ml_business_unit" } diff --git a/4-projects/business_unit_3/production/main.tf b/4-projects/ml_business_unit/production/main.tf similarity index 100% rename from 4-projects/business_unit_3/production/main.tf rename to 4-projects/ml_business_unit/production/main.tf diff --git a/4-projects/business_unit_3/production/outputs.tf b/4-projects/ml_business_unit/production/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/production/outputs.tf rename to 4-projects/ml_business_unit/production/outputs.tf diff --git a/4-projects/business_unit_3/production/outputs.tf.backup b/4-projects/ml_business_unit/production/outputs.tf.backup similarity index 100% rename from 4-projects/business_unit_3/production/outputs.tf.backup rename to 4-projects/ml_business_unit/production/outputs.tf.backup diff --git a/4-projects/business_unit_3/production/production.auto.tfvars b/4-projects/ml_business_unit/production/production.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/production/production.auto.tfvars rename to 4-projects/ml_business_unit/production/production.auto.tfvars diff --git a/4-projects/business_unit_3/production/remote.tf b/4-projects/ml_business_unit/production/remote.tf similarity index 100% rename from 4-projects/business_unit_3/production/remote.tf rename to 4-projects/ml_business_unit/production/remote.tf diff --git a/4-projects/business_unit_3/production/variables.tf b/4-projects/ml_business_unit/production/variables.tf similarity index 100% rename from 4-projects/business_unit_3/production/variables.tf rename to 4-projects/ml_business_unit/production/variables.tf diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/ml_business_unit/shared/README.md similarity index 100% rename from 4-projects/business_unit_3/shared/README.md rename to 4-projects/ml_business_unit/shared/README.md diff --git a/4-projects/business_unit_3/shared/backend.tf b/4-projects/ml_business_unit/shared/backend.tf similarity index 89% rename from 4-projects/business_unit_3/shared/backend.tf rename to 4-projects/ml_business_unit/shared/backend.tf index 997695e5..130bb23c 100644 --- a/4-projects/business_unit_3/shared/backend.tf +++ b/4-projects/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_PROJECTS_BACKEND" - prefix = "terraform/projects/business_unit_3/shared" + prefix = "terraform/projects/ml_business_unit/shared" } } diff --git a/4-projects/business_unit_3/shared/backend.tf.cloud.example b/4-projects/ml_business_unit/shared/backend.tf.cloud.example similarity index 95% rename from 4-projects/business_unit_3/shared/backend.tf.cloud.example rename to 4-projects/ml_business_unit/shared/backend.tf.cloud.example index 5dfb05be..2f8429e2 100644 --- a/4-projects/business_unit_3/shared/backend.tf.cloud.example +++ b/4-projects/ml_business_unit/shared/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu2-shared" + name = "4-ml-shared" } } } diff --git a/4-projects/business_unit_3/shared/common.auto.tfvars b/4-projects/ml_business_unit/shared/common.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/shared/common.auto.tfvars rename to 4-projects/ml_business_unit/shared/common.auto.tfvars diff --git a/4-projects/business_unit_3/shared/example_infra_pipeline.tf b/4-projects/ml_business_unit/shared/example_infra_pipeline.tf similarity index 96% rename from 4-projects/business_unit_3/shared/example_infra_pipeline.tf rename to 4-projects/ml_business_unit/shared/example_infra_pipeline.tf index 0c1a33dd..476c1191 100644 --- a/4-projects/business_unit_3/shared/example_infra_pipeline.tf +++ b/4-projects/ml_business_unit/shared/example_infra_pipeline.tf @@ -16,9 +16,9 @@ locals { repo_names = [ - "bu3-artifact-publish", - "bu3-service-catalog", - "bu3-machine-learning", + "ml-artifact-publish", + "ml-service-catalog", + "ml-machine-learning", ] } @@ -50,7 +50,7 @@ module "app_infra_cloudbuild_project" { billing_code = "1234" primary_contact = "example@example.com" secondary_contact = "example2@example.com" - business_code = "bu3" + business_code = "ml" } module "infra_pipelines" { diff --git a/4-projects/business_unit_3/shared/ml_infra_projects.tf b/4-projects/ml_business_unit/shared/ml_infra_projects.tf similarity index 91% rename from 4-projects/business_unit_3/shared/ml_infra_projects.tf rename to 4-projects/ml_business_unit/shared/ml_infra_projects.tf index 431b0c2a..7640d30e 100644 --- a/4-projects/business_unit_3/shared/ml_infra_projects.tf +++ b/4-projects/ml_business_unit/shared/ml_infra_projects.tf @@ -22,15 +22,15 @@ module "ml_infra_projects" { billing_account = local.billing_account environment = "common" key_rings = local.shared_kms_key_ring - business_code = "bu3" + business_code = "ml" billing_code = "1234" primary_contact = "example@example.com" secondary_contact = "example2@example.com" cloud_source_artifacts_repo_name = var.cloud_source_artifacts_repo_name cloud_source_service_catalog_repo_name = var.cloud_source_service_catalog_repo_name remote_state_bucket = var.remote_state_bucket - artifacts_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["bu3-artifact-publish"] - service_catalog_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"] + artifacts_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["ml-artifact-publish"] + service_catalog_infra_pipeline_sa = module.infra_pipelines[0].terraform_service_accounts["ml-service-catalog"] environment_kms_project_id = "" prevent_destroy = var.prevent_destroy } diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/ml_business_unit/shared/outputs.tf similarity index 100% rename from 4-projects/business_unit_3/shared/outputs.tf rename to 4-projects/ml_business_unit/shared/outputs.tf diff --git a/4-projects/business_unit_3/shared/remote.tf b/4-projects/ml_business_unit/shared/remote.tf similarity index 100% rename from 4-projects/business_unit_3/shared/remote.tf rename to 4-projects/ml_business_unit/shared/remote.tf diff --git a/4-projects/business_unit_3/shared/remote.tf.cloud.example b/4-projects/ml_business_unit/shared/remote.tf.cloud.example similarity index 100% rename from 4-projects/business_unit_3/shared/remote.tf.cloud.example rename to 4-projects/ml_business_unit/shared/remote.tf.cloud.example diff --git a/4-projects/business_unit_3/shared/shared.auto.tfvars b/4-projects/ml_business_unit/shared/shared.auto.tfvars similarity index 100% rename from 4-projects/business_unit_3/shared/shared.auto.tfvars rename to 4-projects/ml_business_unit/shared/shared.auto.tfvars diff --git a/4-projects/business_unit_3/shared/variables.tf b/4-projects/ml_business_unit/shared/variables.tf similarity index 100% rename from 4-projects/business_unit_3/shared/variables.tf rename to 4-projects/ml_business_unit/shared/variables.tf diff --git a/4-projects/business_unit_3/shared/versions.tf b/4-projects/ml_business_unit/shared/versions.tf similarity index 100% rename from 4-projects/business_unit_3/shared/versions.tf rename to 4-projects/ml_business_unit/shared/versions.tf diff --git a/4-projects/modules/composer_env/variables.tf b/4-projects/modules/composer_env/variables.tf index 2a20f9c3..867bb69c 100644 --- a/4-projects/modules/composer_env/variables.tf +++ b/4-projects/modules/composer_env/variables.tf @@ -105,6 +105,6 @@ variable "shared_kms_key_ring" { } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/4-projects/modules/env_folders/variables.tf b/4-projects/modules/env_folders/variables.tf index 2297d829..3661fa96 100644 --- a/4-projects/modules/env_folders/variables.tf +++ b/4-projects/modules/env_folders/variables.tf @@ -15,7 +15,7 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } diff --git a/4-projects/modules/ml_env/example_vertex.tf b/4-projects/modules/ml_env/example_vertex.tf index 547bc932..0c1bd6e9 100644 --- a/4-projects/modules/ml_env/example_vertex.tf +++ b/4-projects/modules/ml_env/example_vertex.tf @@ -59,7 +59,7 @@ module "machine_learning_project" { // Map for the roles where the key is the repository name ("${var.business_code}-example-app") // and the value is the list of roles that this SA need to deploy step 5-app-infra sa_roles = { - "bu3-machine-learning" = [ + "ml-machine-learning" = [ "roles/aiplatform.admin", "roles/artifactregistry.admin", "roles/bigquery.admin", @@ -162,7 +162,7 @@ resource "google_kms_crypto_key_iam_member" "kms_admin" { for_each = module.machine_learning_project.kms_keys crypto_key_id = each.value.id role = "roles/cloudkms.admin" - member = "serviceAccount:${local.app_infra_pipeline_service_accounts["bu3-machine-learning"]}" + member = "serviceAccount:${local.app_infra_pipeline_service_accounts["ml-machine-learning"]}" } // Add crypto key viewer role to kms environment project @@ -212,7 +212,7 @@ resource "google_sourcerepo_repository_iam_member" "read" { project = local.service_catalog_project_id repository = local.service_catalog_repo_name role = "roles/viewer" - member = "serviceAccount:${local.app_infra_pipeline_service_accounts["bu3-machine-learning"]}" + member = "serviceAccount:${local.app_infra_pipeline_service_accounts["ml-machine-learning"]}" } // Add Browser Role to CloudBuild at Env Folder diff --git a/4-projects/modules/ml_env/variables.tf b/4-projects/modules/ml_env/variables.tf index b30f7c90..ad1509e2 100644 --- a/4-projects/modules/ml_env/variables.tf +++ b/4-projects/modules/ml_env/variables.tf @@ -15,12 +15,12 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/5-app-infra/README.md b/5-app-infra/README.md index b0c83588..507dcbdb 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -84,7 +84,7 @@ Clone the repo at the same level of the `terraform-google-enterprise-genai` fold Run `terraform output cloudbuild_project_id` in the `4-projects` folder to get the Cloud Build Project ID. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} gcloud source repos clone gcp-policies gcp-policies-app-infra --project=${INFRA_PIPELINE_PROJECT_ID} @@ -141,21 +141,21 @@ each folder under `images` has the full name and tag of the image that must be b Once pushed, the pipeline build logs can be accessed by navigating to the artifacts project name created in step-4: ```bash - terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw common_artifacts_project_id + terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw common_artifacts_project_id ``` -1. Clone the `bu3-artifact-publish` repo. +1. Clone the `ml-artifact-publish` repo. ```bash - gcloud source repos clone bu3-artifact-publish --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-artifact-publish --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of genAI to new repo. - All subsequent steps assume you are running them from the bu3-artifact-publish directory. + All subsequent steps assume you are running them from the ml-artifact-publish directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-artifact-publish/ + cd ml-artifact-publish/ git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ . @@ -171,7 +171,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa mv common.auto.example.tfvars common.auto.tfvars ``` -1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. +1. Update the file with values from your environment and 0-bootstrap. See machine learning business unit env folder [README.md](./ml_business_unit/production/README.md) file for additional information on the values in the `common.auto.tfvars` file. ```bash export remote_state_bucket=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw projects_gcs_bucket_tfstate) @@ -182,7 +182,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-artifact-publish"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-artifact-publish"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -211,7 +211,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa git push origin production ``` -1. `cd` out of the `bu3-artifacts-publish` repository. +1. `cd` out of the `ml-artifacts-publish` repository. ```bash cd .. @@ -224,7 +224,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa 1. Grab the Artifact Project ID ```bash - export ARTIFACT_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared" output -raw common_artifacts_project_id) + export ARTIFACT_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared" output -raw common_artifacts_project_id) echo ${ARTIFACT_PROJECT_ID} ``` @@ -273,7 +273,7 @@ The resoning behind utilizing one repository with two deployment methodologies i The repository has the structure (truncated for brevity): ``` - business_unit_3 + ml_business_unit ├── development ├── non-production ├── production @@ -316,18 +316,18 @@ This pipeline is listening to the `main` branch of this repository for changes i The pipeline also listens for changes made to `plan`, `development`, `non-production` & `production` branches, this is used for deploying infrastructure to each project. -1. Clone the `bu3-service-catalog` repo. +1. Clone the `ml-service-catalog` repo. ```bash - gcloud source repos clone bu3-service-catalog --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-service-catalog --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of foundation to new repo. - All subsequent steps assume you are running them from the bu3-service-catalog directory. + All subsequent steps assume you are running them from the ml-service-catalog directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-service-catalog + cd ml-service-catalog git checkout -b plan cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ . @@ -343,7 +343,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc mv common.auto.example.tfvars common.auto.tfvars ``` -1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./business_unit_1/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. +1. Update the file with values from your environment and 0-bootstrap. See any of the business unit 1 envs folders [README.md](./ml_business_unit/production/README.md) files for additional information on the values in the `common.auto.tfvars` file. ```bash export remote_state_bucket=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw projects_gcs_bucket_tfstate) @@ -354,7 +354,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-service-catalog"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-service-catalog"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -383,7 +383,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc git push origin production ``` -1. `cd` out of the `bu3-service-catalog` repository. +1. `cd` out of the `ml-service-catalog` repository. ```bash cd .. @@ -394,7 +394,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Grab the Service Catalogs ID ```bash - export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared" output -raw service_catalog_project_id) + export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared" output -raw service_catalog_project_id) echo ${SERVICE_CATALOG_PROJECT_ID} ``` @@ -458,7 +458,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -466,10 +466,10 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-artifact-publish"' --raw-output) + terraform_sa=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-artifact-publish"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -478,7 +478,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-artifact-publish"' --raw-output) + export backend_bucket=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-artifact-publish"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -492,10 +492,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-artifact-publish"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-artifact-publish"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -541,7 +541,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT 1. Grab the Artifact Project ID ```bash - export ARTIFACT_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/business_unit_3/shared" output -raw common_artifacts_project_id) + export ARTIFACT_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/ml_business_unit/shared" output -raw common_artifacts_project_id) echo ${ARTIFACT_PROJECT_ID} ``` @@ -602,7 +602,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -610,10 +610,10 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-service-catalog"' --raw-output) + terraform_sa=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-service-catalog"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -630,7 +630,7 @@ unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-service-catalog"' --raw-output) + export backend_bucket=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-service-catalog"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -644,10 +644,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-service-catalog"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-service-catalog"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -685,7 +685,7 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Grab the Service Catalogs ID ```bash - export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/business_unit_3/shared" output -raw service_catalog_project_id) + export SERVICE_CATALOG_PROJECT_ID=$(terraform -chdir="terraform-google-enterprise-genai/4-projects/ml_business_unit/shared" output -raw service_catalog_project_id) echo ${SERVICE_CATALOG_PROJECT_ID} ``` diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/README.md b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/README.md similarity index 86% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/README.md rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/README.md index c255b2ef..210f623d 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/README.md +++ b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/README.md @@ -4,6 +4,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | instance\_region | The region where compute instance will be created. A subnetwork must exists in the instance region. | `string` | n/a | yes | +| log\_bucket | Log bucket to be used by Service Catalog Bucket | `string` | n/a | yes | | remote\_state\_bucket | Backend bucket to load remote state information from previous steps. | `string` | n/a | yes | ## Outputs diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf similarity index 91% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf index 023a3c07..0b671313 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/backend.tf +++ b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/shared" + prefix = "terraform/app-infra/ml_business_unit/shared" } } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/common.auto.tfvars b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/common.auto.tfvars similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/common.auto.tfvars rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/common.auto.tfvars diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf similarity index 94% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf index d708d36e..d049424c 100644 --- a/5-app-infra/projects/artifact-publish/business_unit_3/shared/locals.tf +++ b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/locals.tf @@ -15,6 +15,6 @@ */ locals { - business_unit = "business_unit_3" + business_unit = "ml_business_unit" environment = "common" } diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/outputs.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/outputs.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/outputs.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/outputs.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/publish_artifacts.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/publish_artifacts.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/remote.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/remote.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/variables.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/variables.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/variables.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/variables.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/versions.tf b/5-app-infra/projects/artifact-publish/ml_business_unit/shared/versions.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/versions.tf rename to 5-app-infra/projects/artifact-publish/ml_business_unit/shared/versions.tf diff --git a/5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md b/5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md new file mode 100644 index 00000000..849fc76c --- /dev/null +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/README.md @@ -0,0 +1,16 @@ + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| instance\_region | The region where compute instance will be created. A subnetwork must exists in the instance region. | `string` | n/a | yes | +| remote\_state\_bucket | Backend bucket to load remote state information from previous steps. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cloudbuild\_trigger\_id | Id of Cloud Build Trigger | +| storage\_bucket\_name | Name of storage bucket created | + + diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf similarity index 91% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf index 023a3c07..0b671313 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/backend.tf +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/shared" + prefix = "terraform/app-infra/ml_business_unit/shared" } } diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/common.auto.tfvars b/5-app-infra/projects/service-catalog/ml_business_unit/shared/common.auto.tfvars similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/common.auto.tfvars rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/common.auto.tfvars diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf similarity index 94% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf index e4a9236a..09352bd6 100644 --- a/5-app-infra/projects/service-catalog/business_unit_3/shared/locals.tf +++ b/5-app-infra/projects/service-catalog/ml_business_unit/shared/locals.tf @@ -15,7 +15,7 @@ */ locals { - business_unit = "business_unit_3" + business_unit = "ml_business_unit" environment = "common" } diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/outputs.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/outputs.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/outputs.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/outputs.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/remote.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/remote.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/service_catalog.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/service_catalog.tf diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/variables.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/variables.tf similarity index 100% rename from 5-app-infra/projects/artifact-publish/business_unit_3/shared/variables.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/variables.tf diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/versions.tf b/5-app-infra/projects/service-catalog/ml_business_unit/shared/versions.tf similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/versions.tf rename to 5-app-infra/projects/service-catalog/ml_business_unit/shared/versions.tf diff --git a/5-app-infra/source_repos/service-catalog/modules/pubsub/locals.tf b/5-app-infra/source_repos/service-catalog/modules/pubsub/locals.tf index f92bce3f..9a1fcf38 100644 --- a/5-app-infra/source_repos/service-catalog/modules/pubsub/locals.tf +++ b/5-app-infra/source_repos/service-catalog/modules/pubsub/locals.tf @@ -17,4 +17,3 @@ locals { keyring_name = "sample-keyring" } - diff --git a/7-composer/dag.py b/7-composer/dag.py index 949fb8dc..40260767 100644 --- a/7-composer/dag.py +++ b/7-composer/dag.py @@ -36,7 +36,7 @@ RUNNER = "DataflowRunner" REGION = "us-central1" JOB_NAME = "census-ingest-composer" -default_kms_key_name = "projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning" +default_kms_key_name = "projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning" deployment_image = "us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-8:latest" service_account = "728034955955-compute@developer.gserviceaccount.com" prod_service_account = "728034955955-compute@developer.gserviceaccount.com" diff --git a/7-composer/us-central1-test-census-034e6abc-bucket/dags b/7-composer/us-central1-test-census-034e6abc-bucket/dags index 80d289fb..ba51813b 100644 --- a/7-composer/us-central1-test-census-034e6abc-bucket/dags +++ b/7-composer/us-central1-test-census-034e6abc-bucket/dags @@ -12,7 +12,7 @@ EVAL_TABLE_ID = 'census_eval_table_composer' RUNNER = "DataflowRunner" REGION = "us-central1" JOB_NAME = "census-ingest-composer" -default_kms_key_name="projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning" +default_kms_key_name="projects/prj-d-kms-cgvl/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning" default_args = { 'owner' : 'airflow', diff --git a/7-vertexpipeline/Readme.md b/7-vertexpipeline/Readme.md index cc157d92..4921fe5d 100644 --- a/7-vertexpipeline/Readme.md +++ b/7-vertexpipeline/Readme.md @@ -1,6 +1,5 @@ Machine learning pipeline from development to production - # Use case This example illustrates the promotion of a a machine learning pipeline from an interactive tenant to a production tenant. The example specifically trains a model on a [UCI census dataset](%28https://archive.ics.uci.edu/dataset/20/census+income%29) for binary classification. @@ -19,7 +18,6 @@ In the first step, a bigquery dataset is created using a bigquery operator offer location=region, ) - Note that the default encryption key for bigquery is set after the projecet inflation so you don't have to pass the key in every query. ## Dataflow for data ingestion @@ -127,7 +125,7 @@ The following method runs the pipeline. Note that a kms encryption key is suppli display_name=f"census_income_{timestamp}", template_path='./common/vertex-ai-pipeline/pipeline_package.yaml', pipeline_root=pipelineroot, - encryption_spec_key_name='projects/prj-d-kms-ui2h/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning', + encryption_spec_key_name='projects/prj-d-kms-ui2h/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning', parameter_values={ "create_bq_dataset_query": create_bq_dataset_query, "bq_dataset": data_config['bq_dataset'], @@ -188,6 +186,3 @@ Note that the is triggered by cloud build (for the first time) and cloud compose - The bigquery service agent on the non-prod project will need EncryptDecrypt permission on the kms key so that it can create the dataset using the CMEK key. - First, a non-prod service account to take care of components that run in non-prod (dataset creation, dataflow, training, and evaluation). This could simply be the default compute engine service account for the non-prod tenant. This service account needs write permission to upload the trained model from the non-prod bucket to the Vertex environment of prod. - Another service account that has permissions on the prod tenant in order to deploy the model and the model monitoring job. This could simply be the default service account for the prod tenant. This service account will also need read permission on bigquery of non-prod where the data exists so that the monitoring job deployed by this service account in prod - - - diff --git a/7-vertexpipeline/census_pipeline.ipynb b/7-vertexpipeline/census_pipeline.ipynb index 9412fc12..1e10de64 100644 --- a/7-vertexpipeline/census_pipeline.ipynb +++ b/7-vertexpipeline/census_pipeline.ipynb @@ -27,7 +27,7 @@ }, "outputs": [], "source": [ - "PROJECT_ID = \"prj-d-bu3machine-learning-gxcv\"\n", + "PROJECT_ID = \"prj-d-ml-machine-learning-gxcv\"\n", "REGION = \"us-central1\"\n", "BUCKET_URI = \"gs://bkt-d-vertexpipe-test-dev\"" ] @@ -72,7 +72,7 @@ "metadata": {}, "source": [ "This is the image we will use to run pipeline components. Replace the name of the artifact project with that of yours, e.g.:\n", - "##### \"us-central1-docker.pkg.dev/{prj-c-bu3artifacts-####}/c-publish-artifacts/vertexpipeline:v2\"\n", + "##### \"us-central1-docker.pkg.dev/{prj-c-ml-artifacts-####}/c-publish-artifacts/vertexpipeline:v2\"\n", "As part of the project inflation pipelines, the image from the Dockerfile in this repository is built and pushed to project" ] }, @@ -85,7 +85,7 @@ }, "outputs": [], "source": [ - "Image = \"us-central1-docker.pkg.dev/prj-c-bu3artifacts-5wdo/c-publish-artifacts/vertexpipeline:v2\"" + "Image = \"us-central1-docker.pkg.dev/prj-c-ml-artifacts-5wdo/c-publish-artifacts/vertexpipeline:v2\"" ] }, { @@ -1555,8 +1555,8 @@ " 'max_nodes': 4,\n", " 'deployment_project': PROJECT_ID,\n", " # important to replace the envryption key here with the key in your own dev environment.\n", - " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning\n", - " \"encryption\": 'projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning',\n", + " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning\n", + " \"encryption\": 'projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning',\n", " \"service_account\": service_account,\n", " \"prod_service_account\": prod_service_account,\n", "}\n", @@ -1573,8 +1573,8 @@ " template_path='./common/vertex-ai-pipeline/pipeline_package.yaml',\n", " pipeline_root=pipelineroot,\n", " # important to replace the envryption key here with the key in your own dev environment.\n", - " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning\n", - " encryption_spec_key_name='projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-bu3machine-learning',\n", + " # format would be: projects/prj-d-kms-####/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning\n", + " encryption_spec_key_name='projects/prj-d-kms-3yzc/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-d-ml-machine-learning',\n", " parameter_values={\n", " \"create_bq_dataset_query\": create_bq_dataset_query,\n", " \"bq_dataset\": data_config['bq_dataset'],\n", diff --git a/7-vertexpipeline/runpipeline.py b/7-vertexpipeline/runpipeline.py index 59c09d06..b737ebf5 100644 --- a/7-vertexpipeline/runpipeline.py +++ b/7-vertexpipeline/runpipeline.py @@ -117,7 +117,7 @@ def __init__(self, 'max_nodes': 4, 'deployment_project': self.PROD_PROJECT_ID, # Raplace encryption with the name of the kms key in the kms project of the prod folder - "encryption": 'projects/prj-p-kms-lkuy/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-p-bu3machine-learning', + "encryption": 'projects/prj-p-kms-lkuy/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-p-ml-machine-learning', "service_account": self.SERVICE_ACCOUNT, "prod_service_account": self.PROD_SERVICE_ACCOUNT } @@ -135,7 +135,7 @@ def execute(self): template_path=self.yaml_file_path, pipeline_root=self.pipelineroot, # Raplace encryption with the name of the kms key in the kms project of the non-prod folder - encryption_spec_key_name='projects/prj-n-kms-gi2r/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-n-bu3machine-learning', + encryption_spec_key_name='projects/prj-n-kms-gi2r/locations/us-central1/keyRings/sample-keyring/cryptoKeys/prj-n-ml-machine-learning', parameter_values={ "create_bq_dataset_query": self.create_bq_dataset_query, "bq_dataset": self.data_config['bq_dataset'], @@ -179,7 +179,7 @@ def execute(self): pipeline = vertex_ai_pipeline( # Replace with your non-prod project Id PROJECT_ID="prj-n-bu3machine-learning-brk1", \ - PROD_PROJECT_ID='prj-p-bu3machine-learning-skc4', \ # Replace with your prod project Id + PROD_PROJECT_ID='prj-p-ml-machine-learning-skc4', \ # Replace with your prod project Id REGION="us-central1", \ BUCKET_URI="gs://bkt-n-ml-storage-akdv", \ # Replace with your bucket in non-prod DATA_PATH="data", \ diff --git a/ERRATA.md b/ERRATA.md index a00986a7..2c8bec34 100644 --- a/ERRATA.md +++ b/ERRATA.md @@ -24,7 +24,7 @@ This will be addressed in the next version of the blueprint guide. #### Naming - The Service Account naming is not aligned to the blueprint guide. Naming will be modified accordingly in a future release. -- The infrastructure pipeline project naming (`prj-buN-c-infra-pipeline`) is not aligned to the blueprint guide(`prj-buN-c-sample-infra-pipeline`). Naming will be modified accordingly in a future release. +- The infrastructure pipeline project naming (`prj-ml-buN-c-infra-pipeline`) is not aligned to the blueprint guide(`prj-ml-buN-c-sample-infra-pipeline`). Naming will be modified accordingly in a future release. #### Networking - The “allow-windows-activation” rule that exists in the code is not explicitly called out in the guide. diff --git a/Makefile b/Makefile index a06f9d1b..52ba3fa8 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker_test_lint: docker run --rm -it \ -e ENABLE_PARALLEL=0 \ -e DISABLE_TFLINT=1 \ - -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared" \ + -e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/ml_business_unit/shared|\./5-app-infra/projects/artifact-publish/ml_business_unit/shared" \ -v $(CURDIR):/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 8f4637a4..aef67481 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -269,7 +269,7 @@ You will need to mark some Terraform resources as **tainted** in order to trigge 1. In a terminal, navigate to the path where the error is being reported. - For example, if the unknown project ID is `prj-bu1-p-sample-base-abcd`, you should go to ./gcp-projects/business_unit_1/production (`business_unit_1` due to `bu1` and `production` due to `p`, see the Security Foundations [naming conventions](https://cloud.google.com/architecture/security-foundations/using-example-terraform#naming_conventions) for more information on the projects naming guideline). + For example, if the unknown project ID is `prj-ml-p-sample-base-abcd`, you should go to ./gcp-projects/ml_business_unit/production (`ml_business_unit` due to `ml` and `production` due to `p`, see the Security Foundations [naming conventions](https://cloud.google.com/architecture/security-foundations/using-example-terraform#naming_conventions) for more information on the projects naming guideline). ```bash cd ./gcp-projects// diff --git a/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example b/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example index 5dfb05be..2f8429e2 100644 --- a/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example +++ b/docs/assets/terraform/4-projects/ml_business_unit/shared/backend.tf.cloud.example @@ -17,7 +17,7 @@ terraform { cloud { workspaces { - name = "4-bu2-shared" + name = "4-ml-shared" } } } diff --git a/docs/assets/terraform/4-projects/modules/env_folders/variables.tf b/docs/assets/terraform/4-projects/modules/env_folders/variables.tf index 2297d829..3661fa96 100644 --- a/docs/assets/terraform/4-projects/modules/env_folders/variables.tf +++ b/docs/assets/terraform/4-projects/modules/env_folders/variables.tf @@ -15,7 +15,7 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } diff --git a/docs/assets/terraform/4-projects/modules/ml_env/variables.tf b/docs/assets/terraform/4-projects/modules/ml_env/variables.tf index 592d10fc..dc374ee0 100644 --- a/docs/assets/terraform/4-projects/modules/ml_env/variables.tf +++ b/docs/assets/terraform/4-projects/modules/ml_env/variables.tf @@ -15,12 +15,12 @@ */ variable "business_code" { - description = "The business code (ex. bu1)." + description = "The business code (ex. ml)." type = string } variable "business_unit" { - description = "The business (ex. business_unit_1)." + description = "The business (ex. ml_business_unit)." type = string } diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/README.md b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md similarity index 100% rename from 5-app-infra/projects/service-catalog/business_unit_3/shared/README.md rename to docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md diff --git a/docs/change_resource_hierarchy.md b/docs/change_resource_hierarchy.md index 34e0fae1..d13fb1eb 100644 --- a/docs/change_resource_hierarchy.md +++ b/docs/change_resource_hierarchy.md @@ -180,24 +180,16 @@ example-organization/ 1. **(Optional)** To simplify the below changes renaming business_units here is helper script. **Remember to review the changes**. The below script assumes you are in `gcp-projects` folder: ```bash - for i in `find "./business_unit_1" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu1//" $i; done + for i in `find "./ml_business_unit" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/ml//" $i; done - for i in `find "./business_unit_1" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_1//" $i; done - - for i in `find "./business_unit_2" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu2//" $i; done - - for i in `find "./business_unit_2" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_2//" $i; done + for i in `find "./ml_business_unit" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/ml_business_unit//" $i; done for i in `find "./business_unit_" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/bu//" $i; done for i in `find "./business_unit_" -type f -not -path "*/.terraform/*" -name '*.tf'`; do sed -i "s/business_unit_//" $i; done ``` -1. For this example, just rename folders business_unit_1 and business_unit_2 to your Business Units names, i.e: finance and retail, to match the example folder hierarchy. - - - - +1. For this example, just rename folder ml_business_unit to your Business Units name, i.e: finance and retail, to match the example folder hierarchy. 1. Change backend gcs prefix for each business unit shared resources. Example: diff --git a/examples/machine-learning-pipeline/README.md b/examples/machine-learning-pipeline/README.md index 7ce16ee4..0161c9ed 100644 --- a/examples/machine-learning-pipeline/README.md +++ b/examples/machine-learning-pipeline/README.md @@ -76,15 +76,15 @@ Below, you can find the values that will need to be applied to `common.auto.tfva In `common.auto.tfvars` update your `perimeter_additional_members` to include: ``` - "serviceAccount:sa-tf-cb-bu3-machine-learning@[prj_c_bu3infra_pipeline_project_id].iam.gserviceaccount.com" + "serviceAccount:sa-tf-cb-ml-machine-learning@[prj_c_ml_infra_pipeline_project_id].iam.gserviceaccount.com" "serviceAccount:sa-terraform-env@[prj_b_seed_project_id].iam.gserviceaccount.com" "serviceAccount:service-[prj_d_logging_project_number]@gs-project-accounts.iam.gserviceaccount.com" "serviceAccount:[prj_d_machine_learning_project_number]@cloudbuild.gserviceaccount.com" ``` ```bash - export prj_c_bu3infra_pipeline_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) - echo "prj_c_bu3infra_pipeline_project_id = ${prj_c_bu3infra_pipeline_project_id}" + export prj_c_ml-infra_pipeline_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) + echo "prj_c_ml-infra_pipeline_project_id = ${prj_c_ml_infra_pipeline_project_id}" export prj_b_seed_project_id=$(terraform -chdir="../terraform-google-enterprise-genai/0-bootstrap/" output -raw seed_project_id) echo "prj_b_seed_project_id = ${prj_b_seed_project_id}" @@ -104,7 +104,7 @@ In `common.auto.tfvars` update your `perimeter_additional_members` to include: export project_d_logging_project_number=$(gsutil cat gs://$backend_bucket/terraform/environments/development/default.tfstate | jq -r '.outputs.env_log_project_number.value') echo "project_d_logging_project_number = ${project_d_logging_project_number}" - prj_d_machine_learning_project_number=$(gsutil cat gs://$backend_bucket_projects/terraform/projects/business_unit_3/development/default.tfstate | jq -r '.outputs.machine_learning_project_number.value') + prj_d_machine_learning_project_number=$(gsutil cat gs://$backend_bucket_projects/terraform/projects/ml_business_unit/development/default.tfstate | jq -r '.outputs.machine_learning_project_number.value') echo "project_d_machine_learning_number = ${prj_d_machine_learning_project_number}" ``` @@ -131,7 +131,7 @@ Once there, select the perimeter that is associated with the environment (eg. `d "resources" = [ "projects/[your-environment-shared-restricted-project-number]", "projects/[your-environment-kms-project-number]", - "projects/[your-environment-bu3machine-learning-number]", + "projects/[your-environment-mlmachine-learning-number]", ] "operations" = { "compute.googleapis.com" = { @@ -190,8 +190,8 @@ For your DEVELOPMENT.AUTO.TFVARS file, also include this as an egress policy: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@compute-system.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@compute-system.iam.gserviceaccount.com", ] }, "to" = { @@ -232,7 +232,7 @@ Add in your dags in the `dags` folder. Any changes to this folder will trigger Have a github token for access to your repository ready, along with an [Application Installation Id](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github#connecting_a_github_host_programmatically) and the remote uri to your repository. -These environmental project inflations are closely tied to the `service-catalog` project that have already deployed. By now, the `bu3-service-catalog` should have been inflated. `service-catalog` contains modules that are being deployed in an interactive (development) environment. Since they already exist; they can be used as terraform modules for operational (non-production, production) environments. This was done in order to avoid code redundancy. One area for all `machine-learning` deployments. +These environmental project inflations are closely tied to the `service-catalog` project that have already deployed. By now, the `ml-service-catalog` should have been inflated. `service-catalog` contains modules that are being deployed in an interactive (development) environment. Since they already exist; they can be used as terraform modules for operational (non-production, production) environments. This was done in order to avoid code redundancy. One area for all `machine-learning` deployments. Under `modules/base_env/main.tf` you will notice all module calls are using `git` links as sources. These links refer to the `service-catalog` cloud source repository we have already set up. @@ -245,7 +245,7 @@ Clone the repo at the same level of the `terraform-google-enterprise-genai` fold Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get the Cloud Build Project ID. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="gcp-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} gcloud source repos clone gcp-policies gcp-policies-app-infra --project=${INFRA_PIPELINE_PROJECT_ID} @@ -279,18 +279,18 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get cd .. ``` -1. Clone the `bu3-machine-learning` repo. +1. Clone the `ml-machine-learning` repo. ```bash - gcloud source repos clone bu3-machine-learning --project=${INFRA_PIPELINE_PROJECT_ID} + gcloud source repos clone ml-machine-learning --project=${INFRA_PIPELINE_PROJECT_ID} ``` 1. Navigate into the repo, change to non-main branch and copy contents of foundation to new repo. - All subsequent steps assume you are running them from the bu3-machine-learning directory. + All subsequent steps assume you are running them from the ml-machine-learning directory. If you run them from another directory, adjust your copy paths accordingly. ```bash - cd bu3-machine-learning + cd ml-machine-learning git checkout -b plan cp -RT ../terraform-google-enterprise-genai/examples/machine-learning-pipeline . @@ -326,7 +326,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Use `terraform output` to retrieve the Service Catalog project-id from the projects step and update values in `module/base_env`. ```bash - export service_catalog_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) echo "service_catalog_project_id = ${service_catalog_project_id}" ## Linux @@ -336,7 +336,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-machine-learning"' --raw-output) + export backend_bucket=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-machine-learning"' --raw-output) echo "backend_bucket = ${backend_bucket}" ## Linux @@ -349,7 +349,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `modules/base_env/main.tf` with the name of service catalog project id to complete the git fqdn for module sources: ```bash - export service_catalog_project_id=$(terraform -chdir="../gcp-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../gcp-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) ##LINUX sed -i "s/SERVICE-CATALOG-PROJECT-ID/${service_catalog_project_id}/" ./modules/base_env/main.tf @@ -373,7 +373,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get github_token="YOUR-GITHUB-TOKEN" for env in "${envs[@]}"; do - output=$(terraform -chdir="../gcp-projects/business_unit_3/${env}" output -raw machine_learning_project_id) + output=$(terraform -chdir="../gcp-projects/ml_business_unit/${env}" output -raw machine_learning_project_id) project_ids+=("$output") done @@ -446,7 +446,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars ``` -1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the bu3 Terraform service account. +1. Provide the user that will be running `./tf-wrapper.sh` the Service Account Token Creator role to the ml Terraform service account. 1. Provide the user permissions to run the terraform locally with the `serviceAccountTokenCreator` permission. @@ -454,10 +454,10 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get member="user:$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")" echo ${member} - project_id=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + project_id=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${project_id} - terraform_sa=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-machine-learning"' --raw-output) + terraform_sa=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-machine-learning"' --raw-output) echo ${terraform_sa} gcloud iam service-accounts add-iam-policy-binding ${terraform_sa} --project ${project_id} --member="${member}" --role="roles/iam.serviceAccountTokenCreator" @@ -466,7 +466,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `backend.tf` with your bucket from the infra pipeline output. ```bash - export backend_bucket=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json state_buckets | jq '."bu3-machine-learning"' --raw-output) + export backend_bucket=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json state_buckets | jq '."ml-machine-learning"' --raw-output) echo "backend_bucket = ${backend_bucket}" for i in `find -name 'backend.tf'`; do sed -i "s/UPDATE_APP_INFRA_BUCKET/${backend_bucket}/" $i; done @@ -475,7 +475,7 @@ Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to get 1. Update `modules/base_env/main.tf` with Service Catalog Project Id. ```bash - export service_catalog_project_id=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw service_catalog_project_id) + export service_catalog_project_id=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw service_catalog_project_id) echo "service_catalog_project_id = ${service_catalog_project_id}" ## Linux @@ -490,10 +490,10 @@ To use the `validate` option of the `tf-wrapper.sh` script, please follow the [i 1. Use `terraform output` to get the Infra Pipeline Project ID from 4-projects output. ```bash - export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -raw cloudbuild_project_id) + export INFRA_PIPELINE_PROJECT_ID=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -raw cloudbuild_project_id) echo ${INFRA_PIPELINE_PROJECT_ID} - export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../4-projects/business_unit_3/shared/" output -json terraform_service_accounts | jq '."bu3-machine-learning"' --raw-output) + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../../4-projects/ml_business_unit/shared/" output -json terraform_service_accounts | jq '."ml-machine-learning"' --raw-output) echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} ``` @@ -568,9 +568,9 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir In order to avoid having to specify a kms key for every query against a bigquery resource, we set the default project encryption key to the corresponding environment key in advance ```bash - ml_project_dev=$(terraform -chdir="gcp-projects/business_unit_3/development" output -json) - ml_project_nonprd=$(terraform -chdir="gcp-projects/business_unit_3/non-production" output -json) - ml_project_prd=$(terraform -chdir="gcp-projects/business_unit_3/production" output -json) + ml_project_dev=$(terraform -chdir="gcp-projects/ml_business_unit/development" output -json) + ml_project_nonprd=$(terraform -chdir="gcp-projects/ml_business_unit/non-production" output -json) + ml_project_prd=$(terraform -chdir="gcp-projects/ml_business_unit/production" output -json) projects=( "$ml_project_dev" "$ml_project_nonprd" "$ml_project_prd" ) @@ -586,18 +586,18 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Now that machine learning's projects have all been inflated, please _return to gcp-projects_ and update COMMON.AUTO.TFVARS with this __additional__ information under `perimeter_additional_members`: ``` - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", - "serviceAccount:[prj-n-bu3machine-learning-number]@cloudbuild.gserviceaccount.com", - "serviceAccount:[prj-n-bu3machine-learning-number]-compute@developer.gserviceaccount.com", - "serviceAccount:[prj-p-bu3machine-learning-number]@cloudbuild.gserviceaccount.com", - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]@cloudbuild.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]-compute@developer.gserviceaccount.com", + "serviceAccount:[prj-p-ml-machine-learning-number]@cloudbuild.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", ``` 2. optional - run the below command to generate a list of the above changes needed to COMMON.AUTO.TFVARS ```bash - ml_n=$(terraform -chdir="gcp-projects/business_unit_3/non-production" output -raw machine_learning_project_number) - ml_p=$(terraform -chdir="gcp-projects/business_unit_3/production" output -raw machine_learning_project_number) + ml_n=$(terraform -chdir="gcp-projects/ml_business_unit/non-production" output -raw machine_learning_project_number) + ml_p=$(terraform -chdir="gcp-projects/ml_business_unit/production" output -raw machine_learning_project_number) echo "serviceAccount:service-${ml_n}@dataflow-service-producer-prod.iam.gserviceaccount.com", echo "serviceAccount:${ml_n}@cloudbuild.gserviceaccount.com", @@ -611,7 +611,7 @@ After executing this stage, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` envir 1. Add in more agents to the DEVELOPMENT.AUTO.TFVARS file under `egress_policies`. Notably: - * "serviceAccount:bq-[prj-d-bu3machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" + * "serviceAccount:bq-[prj-d-ml-machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" This should be added under identities. It should look like this:: @@ -622,9 +622,9 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:bq-[prj-d-bu3machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" << New Addition - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-project-number]@compute-system.iam.gserviceaccount.com", + "serviceAccount:bq-[prj-d-ml-machine-learning-project-number]@bigquery-encryption.iam.gserviceaccount.com" << New Addition + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@gcp-sa-notebooks.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-project-number]@compute-system.iam.gserviceaccount.com", ] }, "to" = { @@ -650,11 +650,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-d-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-c-bu3artifacts-number]"] + "resources" = ["projects/[prj-c-ml-artifacts-number]"] "operations" = { "artifactregistry.googleapis.com" = { "methods" = ["*"] @@ -667,11 +667,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-n-bu3machine-learning-number]"] + "resources" = ["projects/[prj-n-ml-machine-learning-number]"] "operations" = { "compute.googleapis.com" = { "methods" = ["*"] @@ -691,7 +691,7 @@ Notably: }, "to" = { "resources" = [ - "projects/[prj-c-bu3artifacts-number]" + "projects/[prj-c-ml-artifacts-number]" ] "operations" = { "artifactregistry.googleapis.com" = { @@ -705,11 +705,11 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-c-bu3artifacts-number]"] + "resources" = ["projects/[prj-c-ml-artifacts-number]"] "operations" = { "artifactregistry.googleapis.com" = { "methods" = ["*"] @@ -722,7 +722,7 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-n-bu3machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", + "serviceAccount:service-[prj-n-ml-machine-learning-number]@dataflow-service-producer-prod.iam.gserviceaccount.com", ] }, "to" = { @@ -738,12 +738,12 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:[prj-n-bu3machine-learning-number]-compute@developer.gserviceaccount.com", - "serviceAccount:service-[prj-d-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:[prj-n-ml-machine-learning-number]-compute@developer.gserviceaccount.com", + "serviceAccount:service-[prj-d-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", ] }, "to" = { - "resources" = ["projects/[prj-p-bu3machine-learning-number]"] + "resources" = ["projects/[prj-p-ml-machine-learning-number]"] "operations" = { "aiplatform.googleapis.com" = { "methods" = ["*"] @@ -766,15 +766,15 @@ Notably: "from" = { "identity_type" = "" "identities" = [ - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", - "serviceAccount:service-[prj-p-bu3machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform.iam.gserviceaccount.com", + "serviceAccount:service-[prj-p-ml-machine-learning-number]@gcp-sa-aiplatform-cc.iam.gserviceaccount.com", "serviceAccount:cloud-cicd-artifact-registry-copier@system.gserviceaccount.com", ] }, "to" = { "resources" = [ - "projects/[prj-n-bu3machine-learning-number]", - "projects/[prj-c-bu3artifacts-number]", + "projects/[prj-n-ml-machine-learning-number]", + "projects/[prj-c-ml-artifacts-number]", ] "operations" = { "artifactregistry.googleapis.com" = { @@ -795,7 +795,7 @@ Notably: Once you have set up service catalog and attempt to deploy out terraform code, there is a high chance you will encounter this error: `Permission denied; please check you have the correct IAM permissions and APIs enabled.` -This is due to a VPC Service control error that until now, is impossible to add into the egress policy. Go to `prj-d-bu3machine-learning` project and view the logs, filtering for ERRORS. There will be a VPC Service Controls entry that has an `egressViolation`. It should look something like the following: +This is due to a VPC Service control error that until now, is impossible to add into the egress policy. Go to `prj-d-ml-machine-learning` project and view the logs, filtering for ERRORS. There will be a VPC Service Controls entry that has an `egressViolation`. It should look something like the following: ``` egressViolations: [ 0: { @@ -926,14 +926,14 @@ Also make sure to have a gcs bucket ready to store the artifacts for the tutoria |variable|definition|example value|How to obtain| |--------|----------|-------------|-------------| - |PROJECT_ID|The id of the non-prod project|`{none-prod-project-id}`|From the project's menu in console navigate to the `fldr-non-production/fldr-non-production-bu3` folder; here you can find the machine learning project in non-prod (`prj-n-bu3machine-learning`) and obtain its' ID| - |BUCKET_URI|URI of the non-prod bucket|`gs://non-prod-bucket`|From the project menu in console navigate to the non-prod ML project `fldr-non-production/fldr-non-production-bu3/prj-n-bu3machine-learning` project, navigate to cloud storage and copy the name of the bucket available there| + |PROJECT_ID|The id of the non-prod project|`{none-prod-project-id}`|From the project's menu in console navigate to the `fldr-non-production/fldr-non-production-ml` folder; here you can find the machine learning project in non-prod (`prj-n-ml-machine-learning`) and obtain its' ID| + |BUCKET_URI|URI of the non-prod bucket|`gs://non-prod-bucket`|From the project menu in console navigate to the non-prod ML project `fldr-non-production/fldr-non-production-ml/prj-n-ml-machine-learning` project, navigate to cloud storage and copy the name of the bucket available there| |REGION|The region for pipeline jobs|Can be left as default `us-central1`| - |PROD_PROJECT_ID|ID of the prod project|`prod-project-id`|In console's project menu, navigate to the `fldr-production/fldr-production-bu3` folder; here you can find the machine learning project in prod (`prj-p-bu3machine-learning`) and obtain its' ID| - |Image|The image artifact used to run the pipeline components. The image is already built and pushed to the artifact repository in your artifact project under the common folder|`f"us-central1-docker.pkg.dev/{{artifact-project}}/{{artifact-repository}}/vertexpipeline:v2"`|Navigate to `fldr-common/prj-c-bu3artifacts` project. Navigate to the artifact registry repositories in the project to find the full name of the image artifact.| + |PROD_PROJECT_ID|ID of the prod project|`prod-project-id`|In console's project menu, navigate to the `fldr-production/fldr-production-ml` folder; here you can find the machine learning project in prod (`prj-p-ml-machine-learning`) and obtain its' ID| + |Image|The image artifact used to run the pipeline components. The image is already built and pushed to the artifact repository in your artifact project under the common folder|`f"us-central1-docker.pkg.dev/{{artifact-project}}/{{artifact-repository}}/vertexpipeline:v2"`|Navigate to `fldr-common/prj-c-ml-artifacts` project. Navigate to the artifact registry repositories in the project to find the full name of the image artifact.| |DATAFLOW_SUBNET|The shared subnet in non-prod env required to run the dataflow job|`https://www.googleapis.com/compute/v1/projects/{non-prod-network-project}/regions/us-central1/subnetworks/{subnetwork-name}`|Navigate to the `fldr-network/prj-n-shared-restricted` project. Navigate to the VPC networks and under the subnets tab, find the name of the network associated with your region (us-central1)| - |SERVICE_ACCOUNT|The service account used to run the pipeline and it's components such as the model monitoring job. This is the compute default service account of non-prod if you don't plan on using another costume service account|`{non-prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the non-prod project `fldr-non-production/fldr-non-production-bu3/prj-n-bu3machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| - |PROD_SERICE_ACCOUNT|The service account used to create endpoint, upload the model, and deploy the model in the prod project. This is the compute default service account of prod if you don't plan on using another costume service account|`{prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the prod project `fldr-production/fldr-production-bu3/prj-p-bu3machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| + |SERVICE_ACCOUNT|The service account used to run the pipeline and it's components such as the model monitoring job. This is the compute default service account of non-prod if you don't plan on using another costume service account|`{non-prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the non-prod project `fldr-non-production/fldr-non-production-ml/prj-n-ml-machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| + |PROD_SERICE_ACCOUNT|The service account used to create endpoint, upload the model, and deploy the model in the prod project. This is the compute default service account of prod if you don't plan on using another costume service account|`{prod-project_number}-compute@developer.gserviceaccount.com`|Head over to the IAM page in the prod project `fldr-production/fldr-production-ml/prj-p-ml-machine-learning`, check the box for `Include Google-provided role grants` and look for the service account with the `{project_number}-compute@developer.gserviceaccount.com`| |deployment_config['encryption']|The kms key for the prod env. This key is used to encrypt the vertex model, endpoint, model deployment, and model monitoring.|`projects/{prod-kms-project}/locations/us-central1/keyRings/{keyring-name}/cryptoKeys/{key-name}`|Navigate to `fldr-production/prj-n-kms`, navigate to the Security/Key management in that project to find the key in `sample-keyring` keyring of your target region `us-central1`| |encryption_spec_key_name|The name of the encryption key for the non-prod env. This key is used to create the vertex pipeline job and it's associated metadata store|`projects/{non-prod-kms-project}/locations/us-central1/keyRings/{keyring-name}/cryptoKeys/{key-name}`|Navigate to `fldr-non-production/prj-n-kms`, navigate to the Security/Key management in that project to find the key in `sample-keyring` keyring of your target region `us-central1`| |monitoring_config['email']|The email that Vertex AI monitoring will email alerts to|`your email`|your email associated with your gcp account| diff --git a/examples/machine-learning-pipeline/business_unit_3/development/README.md b/examples/machine-learning-pipeline/ml_business_unit/development/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/README.md rename to examples/machine-learning-pipeline/ml_business_unit/development/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/production/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/development/backend.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/production/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/backend.tf index 9a280f76..ebcfd862 100644 --- a/examples/machine-learning-pipeline/business_unit_3/production/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/development/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/production" + prefix = "terraform/app-infra/ml_business_unit/development" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/development/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/development/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/development/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/development/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/development/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/development/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/locals.tf index 39b3471f..593d0e47 100644 --- a/examples/machine-learning-pipeline/business_unit_3/development/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/development/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "development" environment_code = "d" } diff --git a/examples/machine-learning-pipeline/business_unit_3/development/main.tf b/examples/machine-learning-pipeline/ml_business_unit/development/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/development/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/development/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/development/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/development/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/development/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/development/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/development/versions.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/README.md b/examples/machine-learning-pipeline/ml_business_unit/non-production/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/README.md rename to examples/machine-learning-pipeline/ml_business_unit/non-production/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf similarity index 90% rename from examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf index a8c71ae7..d737ec7d 100644 --- a/examples/machine-learning-pipeline/business_unit_3/non-production/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/non-production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/non-production" + prefix = "terraform/app-infra/ml_business_unit/non-production" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/non-production/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/non-production/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf index 6600d7b7..e59d1a15 100644 --- a/examples/machine-learning-pipeline/business_unit_3/non-production/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/non-production/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "non-production" environment_code = "n" } diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/main.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/non-production/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/non-production/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/non-production/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/non-production/versions.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/README.md b/examples/machine-learning-pipeline/ml_business_unit/production/README.md similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/README.md rename to examples/machine-learning-pipeline/ml_business_unit/production/README.md diff --git a/examples/machine-learning-pipeline/business_unit_3/development/backend.tf b/examples/machine-learning-pipeline/ml_business_unit/production/backend.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/development/backend.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/backend.tf index 60060517..f8d1bace 100644 --- a/examples/machine-learning-pipeline/business_unit_3/development/backend.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/production/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/business_unit_3/development" + prefix = "terraform/app-infra/ml_business_unit/production" } } diff --git a/examples/machine-learning-pipeline/business_unit_3/production/common.auto.tfvars b/examples/machine-learning-pipeline/ml_business_unit/production/common.auto.tfvars similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/common.auto.tfvars rename to examples/machine-learning-pipeline/ml_business_unit/production/common.auto.tfvars diff --git a/examples/machine-learning-pipeline/business_unit_3/production/locals.tf b/examples/machine-learning-pipeline/ml_business_unit/production/locals.tf similarity index 91% rename from examples/machine-learning-pipeline/business_unit_3/production/locals.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/locals.tf index 5a1cd7c8..daf31502 100644 --- a/examples/machine-learning-pipeline/business_unit_3/production/locals.tf +++ b/examples/machine-learning-pipeline/ml_business_unit/production/locals.tf @@ -15,8 +15,8 @@ */ locals { - business_unit = "business_unit_3" - business_code = "bu3" + business_unit = "ml_business_unit" + business_code = "ml" env = "production" environment_code = "p" } diff --git a/examples/machine-learning-pipeline/business_unit_3/production/main.tf b/examples/machine-learning-pipeline/ml_business_unit/production/main.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/main.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/main.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/outputs.tf b/examples/machine-learning-pipeline/ml_business_unit/production/outputs.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/outputs.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/outputs.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/remote.tf b/examples/machine-learning-pipeline/ml_business_unit/production/remote.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/remote.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/remote.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/variables.tf b/examples/machine-learning-pipeline/ml_business_unit/production/variables.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/variables.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/variables.tf diff --git a/examples/machine-learning-pipeline/business_unit_3/production/versions.tf b/examples/machine-learning-pipeline/ml_business_unit/production/versions.tf similarity index 100% rename from examples/machine-learning-pipeline/business_unit_3/production/versions.tf rename to examples/machine-learning-pipeline/ml_business_unit/production/versions.tf diff --git a/examples/machine-learning-pipeline/modules/base_env/variables.tf b/examples/machine-learning-pipeline/modules/base_env/variables.tf index 55ff0fee..848a9ad4 100644 --- a/examples/machine-learning-pipeline/modules/base_env/variables.tf +++ b/examples/machine-learning-pipeline/modules/base_env/variables.tf @@ -29,7 +29,7 @@ variable "environment_code" { } variable "business_code" { - description = "Business unit code (ie. bu3)" + description = "Business unit code (ie. ml)" type = string } diff --git a/test/restore_tf_files.sh b/test/restore_tf_files.sh index d7022b09..0e91f3d0 100644 --- a/test/restore_tf_files.sh +++ b/test/restore_tf_files.sh @@ -142,7 +142,6 @@ function appinfra(){ mv 5-app-infra/business_unit_1/production/common.auto.tfvars.disabled 5-app-infra/business_unit_1/production/common.auto.tfvars } - # parse args for arg in "$@" do