diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/common.auto.example.tfvars b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/common.auto.example.tfvars deleted file mode 100644 index 7f83f05b..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/common.auto.example.tfvars +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -instance_region = "us-central1" // should be one of the regions used to create network on step 3-networks - -remote_state_bucket = "REMOTE_STATE_BUCKET" diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/README.md b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/README.md deleted file mode 100644 index c255b2ef..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/README.md +++ /dev/null @@ -1,15 +0,0 @@ - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| instance\_region | The region where compute instance will be created. A subnetwork must exists in the instance region. | `string` | n/a | yes | -| remote\_state\_bucket | Backend bucket to load remote state information from previous steps. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| cloudbuild\_trigger\_id | n/a | - - diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/backend.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/backend.tf deleted file mode 100644 index 0b671313..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/backend.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - backend "gcs" { - bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/ml_business_unit/shared" - } -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/common.auto.tfvars b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/common.auto.tfvars deleted file mode 120000 index 39aaa462..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/common.auto.tfvars +++ /dev/null @@ -1 +0,0 @@ -../../common.auto.tfvars \ No newline at end of file diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/locals.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/locals.tf deleted file mode 100644 index d049424c..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/locals.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - business_unit = "ml_business_unit" - environment = "common" -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/outputs.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/outputs.tf deleted file mode 100644 index 11e2aade..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/outputs.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# output "trigger_sa_account_id" { -# description = "Account id of service account cloudbuild." -# value = module.artifact_pipeline.trigger_sa_account_id -# } - -# output "cloudbuild_v2_repo_id" { -# description = "Repository ID of cloudbuild repository" -# value = module.artifact_pipeline.cloudbuild_v2_repo_id -# } - -# output "kms_key_id" { -# description = "Projects Key ID for encrytion" -# value = module.artifact_pipeline.kms_key_id -# } - -output "cloudbuild_trigger_id" { - value = module.artifact_publish.cloudbuild_trigger_id -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/publish_artifacts.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/publish_artifacts.tf deleted file mode 100644 index 2d535968..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/publish_artifacts.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] -} - -data "google_project" "common_artifacts" { - project_id = local.common_artifacts_project_id -} - -module "artifact_publish" { - source = "../../modules/publish_artifacts" - - environment = local.environment - description = "Publish Artifacts for ML Projects" - project_id = local.common_artifacts_project_id - name = local.artifacts_repo_name - format = "DOCKER" - region = var.instance_region - cleanup_policies = [{ - id = "keep-tagged-release" - action = "KEEP" - condition = [ - { - tag_state = "TAGGED", - tag_prefixes = ["release"], - package_name_prefixes = ["webapp", "mobile"] - } - ] - }] - - kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_artifacts.name}" -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/remote.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/remote.tf deleted file mode 100644 index c98f85b7..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/remote.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - common_artifacts_project_id = data.terraform_remote_state.projects_shared.outputs.common_artifacts_project_id - service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id - artifacts_repo_name = data.terraform_remote_state.projects_shared.outputs.artifacts_repo_name - shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings -} - -data "terraform_remote_state" "projects_shared" { - backend = "gcs" - - config = { - bucket = var.remote_state_bucket - prefix = "terraform/projects/${local.business_unit}/shared" - } -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/variables.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/variables.tf deleted file mode 100644 index 8e9bafd0..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/variables.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_region" { - description = "The region where compute instance will be created. A subnetwork must exists in the instance region." - type = string -} - -variable "remote_state_bucket" { - description = "Backend bucket to load remote state information from previous steps." - type = string -} diff --git a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/versions.tf b/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/versions.tf deleted file mode 100644 index baa38abb..00000000 --- a/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo/ml_business_unit/shared/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - - google = { - source = "hashicorp/google" - version = ">= 3.77, < 6" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 3.77, < 6" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.1" - } - - } -} diff --git a/docs/assets/terraform/5-appinfra/artifacts-pipeline/README.md b/docs/assets/terraform/5-appinfra/artifacts-pipeline/README.md deleted file mode 100644 index 365d29e8..00000000 --- a/docs/assets/terraform/5-appinfra/artifacts-pipeline/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# ml-foundations-docker -Dockerfile repository for ml-foundations artifacts project diff --git a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-13:0.1/Dockerfile b/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-13:0.1/Dockerfile deleted file mode 100644 index 731a8e02..00000000 --- a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-13:0.1/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-13:latest -RUN echo "Hello World" > helloworld.txt - diff --git a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-8:01/Dockerfile b/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-8:01/Dockerfile deleted file mode 100644 index d1d7fc14..00000000 --- a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-cpu.2-8:01/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-8:cmle_op_images_20240312_0210_RC00 - diff --git a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-gpu.2-13:0.1/Dockerfile b/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-gpu.2-13:0.1/Dockerfile deleted file mode 100644 index d8015e53..00000000 --- a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/tf2-gpu.2-13:0.1/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-13:latest -RUN echo "Hello World" > helloworld.txt diff --git a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/vertexpipeline:v2/Dockerfile b/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/vertexpipeline:v2/Dockerfile deleted file mode 100644 index b466e248..00000000 --- a/docs/assets/terraform/5-appinfra/artifacts-pipeline/images/vertexpipeline:v2/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM tensorflow/tensorflow:2.8.0 -RUN pip install tensorflow-io==0.25.0 protobuf==3.20.0 google-cloud-bigquery==3.13.0 pandas==2.0.3 db-dtypes==1.2.0 google-cloud-aiplatform==1.36.0 google-cloud-storage==2.14.0 kfp google-cloud-pipeline-components diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/data.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/data.tf deleted file mode 100644 index 4671af93..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/data.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/locals.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/locals.tf deleted file mode 100644 index 2e394621..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/locals.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - env_code = substr(var.environment, 0, 1) - name_var = format("%s-%s", local.env_code, var.name) - # key_ring_var = "projects/${var.cmek_project_id}/locations/${var.region}/keyRings/sample-keyring" - region_short_code = { - "us-central1" = "usc1" - "us-east4" = "use4" - } - # github_owner = split("/", split("https://github.com/", var.github_remote_uri)[1])[0] - # github_repo_name = trim(basename(var.github_remote_uri), ".git") - - trigger_sa_roles = [ - "roles/artifactregistry.reader", - "roles/artifactregistry.writer", - ] - # github_repository = replace(var.github_remote_uri, "https://", "") -} diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/main.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/main.tf deleted file mode 100644 index c8c8ed6f..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/main.tf +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -resource "google_project_service_identity" "artifact_registry_agent" { - provider = google-beta - - project = var.project_id - service = "artifactregistry.googleapis.com" -} - -resource "google_kms_crypto_key_iam_member" "artifact-kms-key-binding" { - crypto_key_id = var.kms_crypto_key - role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" - member = "serviceAccount:${google_project_service_identity.artifact_registry_agent.email}" -} - -resource "google_artifact_registry_repository" "repo" { - provider = google-beta - location = var.region - repository_id = local.name_var - description = var.description - format = var.format - cleanup_policy_dry_run = var.cleanup_policy_dry_run - project = data.google_project.project.project_id - - #Customer Managed Encryption Keys - #Control ID: COM-CO-2.3 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - - kms_key_name = var.kms_crypto_key - - #Cleanup policy - #Control ID: AR-CO-6.1 - #NIST 800-53: SI-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - dynamic "cleanup_policies" { - for_each = var.cleanup_policies - content { - id = cleanup_policies.value.id - action = cleanup_policies.value.action - - dynamic "condition" { - for_each = cleanup_policies.value.condition != null ? [cleanup_policies.value.condition] : [] - content { - tag_state = condition.value[0].tag_state - tag_prefixes = condition.value[0].tag_prefixes - package_name_prefixes = condition.value[0].package_name_prefixes - older_than = condition.value[0].older_than - } - } - - dynamic "most_recent_versions" { - for_each = cleanup_policies.value.most_recent_versions != null ? [cleanup_policies.value.most_recent_versions] : [] - content { - package_name_prefixes = most_recent_versions.value[0].package_name_prefixes - keep_count = most_recent_versions.value[0].keep_count - } - } - } - } - depends_on = [ - google_kms_crypto_key_iam_member.artifact-kms-key-binding, - - ] -} -resource "google_artifact_registry_repository_iam_member" "project" { - for_each = toset(local.trigger_sa_roles) - project = var.project_id - repository = google_artifact_registry_repository.repo.repository_id - location = var.region - role = each.key - # member = "serviceAccount:${google_service_account.trigger_sa.email}" - member = "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com" -} - -# resource "google_sourcerepo_repository" "artifact_repo" { -# project = var.project_id -# name = var.name -# } -resource "google_cloudbuild_trigger" "docker_build" { - name = "docker-build" - project = var.project_id - location = var.region - - trigger_template { - branch_name = "^main$" - repo_name = var.name - } - build { - timeout = "1800s" - step { - id = "unshallow" - name = "gcr.io/cloud-builders/git" - entrypoint = "/bin/bash" - args = [ - "-c", - "git fetch --unshallow" - ] - } - step { - id = "select-folder" - name = "gcr.io/cloud-builders/git" - entrypoint = "/bin/bash" - args = [ - "-c", - <<-EOT - changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r) - changed_folders=$(echo "$changed_files" | awk -F/ '{print $2}' | sort | uniq ) - - for folder in $changed_folders; do - echo "Found docker folder: $folder" - echo $folder >> /workspace/docker_build - done - EOT - ] - } - step { - id = "build-image" - wait_for = ["select-folder"] - name = "gcr.io/cloud-builders/docker" - entrypoint = "/bin/bash" - args = [ - "-c", - <<-EOT - build_path="/workspace/docker_build" - while IFS= read -r line; do - docker build -t ${var.region}-docker.pkg.dev/$PROJECT_ID/c-publish-artifacts/$line images/$line - done < "$build_path" - EOT - ] - } - - step { - id = "push-image" - wait_for = ["select-folder", "build-image"] - name = "gcr.io/cloud-builders/docker" - entrypoint = "/bin/bash" - args = [ - "-c", - <<-EOT - build_path="/workspace/docker_build" - while IFS= read -r line; do - docker push ${var.region}-docker.pkg.dev/$PROJECT_ID/c-publish-artifacts/$line - done < "$build_path" - EOT - ] - } - } -} diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/outputs.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/outputs.tf deleted file mode 100644 index 668f4e96..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cloudbuild_trigger_id" { - value = google_cloudbuild_trigger.docker_build.id -} - diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/variables.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/variables.tf deleted file mode 100644 index 18e8a528..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/variables.tf +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "region" { - description = "Location of the repository." - type = string -} -variable "name" { - description = "Name of the repository." - type = string -} - -variable "description" { - description = "Description of the repository." - type = string -} - -variable "format" { - description = "Format of the repository." - type = string -} - -variable "cleanup_policy_dry_run" { - description = "Whether to perform a dry run of the cleanup policy." - type = bool - default = false -} - -variable "cleanup_policies" { - description = "List of cleanup policies." - type = list(object({ - id = string - action = optional(string) - condition = optional(list(object({ - tag_state = optional(string) - tag_prefixes = optional(list(string)) - package_name_prefixes = optional(list(string)) - older_than = optional(string) - }))) - most_recent_versions = optional(list(object({ - package_name_prefixes = optional(list(string)) - keep_count = optional(number) - }))) - })) -} - -variable "environment" { - type = string - description = "development | staging | production | commmon" - validation { - condition = contains(["development", "staging", "production", "common"], var.environment) - error_message = "Environment must be one of [development, staging, production]." - } -} - -variable "project_id" { - description = "Project ID" -} - -variable "kms_crypto_key" { - description = "KMS Key to be used" - type = string -} diff --git a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/versions.tf b/docs/assets/terraform/5-appinfra/modules/publish_artifacts/versions.tf deleted file mode 100644 index 7fb6a704..00000000 --- a/docs/assets/terraform/5-appinfra/modules/publish_artifacts/versions.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.61" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 3.61" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.1" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-enterprise-genai:app_env_base/v0.0.1" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-enterprise-genai:app_env_base/v0.0.1" - } -} diff --git a/docs/assets/terraform/5-appinfra/modules/service_catalog/data.tf b/docs/assets/terraform/5-appinfra/modules/service_catalog/data.tf deleted file mode 100644 index 4671af93..00000000 --- a/docs/assets/terraform/5-appinfra/modules/service_catalog/data.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} diff --git a/docs/assets/terraform/5-appinfra/modules/service_catalog/locals.tf b/docs/assets/terraform/5-appinfra/modules/service_catalog/locals.tf deleted file mode 100644 index 1d4c7e5a..00000000 --- a/docs/assets/terraform/5-appinfra/modules/service_catalog/locals.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # github_repository = replace(var.github_remote_uri, "https://", "") - log_bucket_prefix = "bkt" - bucket_permissions = { - - "roles/storage.admin" = [ - "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com" - ], - "roles/storage.legacyObjectReader" = [ - "serviceAccount:${var.machine_learning_project_number}@cloudbuild.gserviceaccount.com", - ], - } - - bucket_roles = flatten([ - for role in keys(local.bucket_permissions) : [ - for sa in local.bucket_permissions[role] : - { - role = role - acct = sa - } - ] - ]) -} - - diff --git a/docs/assets/terraform/5-appinfra/modules/service_catalog/main.tf b/docs/assets/terraform/5-appinfra/modules/service_catalog/main.tf deleted file mode 100644 index 4aa6cde0..00000000 --- a/docs/assets/terraform/5-appinfra/modules/service_catalog/main.tf +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# resource "google_project_service_identity" "storage_agent" { -# provider = google-beta - -# project = var.project_id -# service = "storage.googleapis.com" -# } -# resource "google_kms_crypto_key_iam_member" "storage-kms-key-binding" { -# crypto_key_id = var.kms_crypto_key -# role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" -# member = "serviceAccount:${google_project_service_identity.storage_agent.email}" -# } - -resource "random_string" "bucket_name" { - length = 4 - upper = false - numeric = true - lower = true - special = false -} - -resource "google_storage_bucket" "bucket" { - location = var.region - name = "${var.gcs_bucket_prefix}-${var.project_id}-${lower(var.region)}-${random_string.bucket_name.result}" - project = var.project_id - uniform_bucket_level_access = true - - encryption { - default_kms_key_name = var.kms_crypto_key - } - versioning { - enabled = true - } - logging { - log_bucket = var.log_bucket - } - -} - -resource "google_storage_bucket_iam_member" "bucket_role" { - for_each = { for gcs in local.bucket_roles : "${gcs.role}-${gcs.acct}" => gcs } - bucket = google_storage_bucket.bucket.name - role = each.value.role - member = each.value.acct -} - -# resource "google_sourcerepo_repository" "service_catalog" { -# project = var.project_id -# name = var.name -# } - -resource "google_sourcerepo_repository_iam_member" "read" { - project = var.project_id - repository = var.name - role = "roles/viewer" - member = "serviceAccount:${var.tf_service_catalog_sa_email}" -} - -resource "google_cloudbuild_trigger" "zip_files" { - name = "zip-tf-files-trigger" - project = var.project_id - location = var.region - - # repository_event_config { - # repository = var.cloudbuild_repo_id - # push { - # branch = "^main$" - # } - # } - - trigger_template { - branch_name = "^main$" - repo_name = var.name - } - - build { - # step { - # id = "unshallow" - # name = "gcr.io/cloud-builders/git" - # secret_env = ["token"] - # entrypoint = "/bin/bash" - # args = [ - # "-c", - # "git fetch --unshallow https://$token@${local.github_repository}" - # ] - - # } - step { - id = "unshallow" - name = "gcr.io/cloud-builders/git" - entrypoint = "/bin/bash" - args = [ - "-c", - "git fetch --unshallow" - ] - - } - # available_secrets { - # secret_manager { - # env = "token" - # version_name = var.secret_version_name - # } - # } - step { - id = "find-folders-affected-in-push" - name = "gcr.io/cloud-builders/git" - entrypoint = "/bin/bash" - args = [ - "-c", - <<-EOT - changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r) - changed_folders=$(echo "$changed_files" | awk -F/ '{print $2}' | sort | uniq ) - - for folder in $changed_folders; do - if [[ "$folder" != *.* ]]; then - echo "Found change in folder: $folder" - (cd modules/$folder && find . -type f -name '*.tf' -exec tar -cvzPf "/workspace/$folder.tar.gz" {} +) - fi - done - EOT - ] - } - step { - id = "push-to-bucket" - name = "gcr.io/cloud-builders/gsutil" - args = ["cp", "/workspace/*.tar.gz", "gs://${google_storage_bucket.bucket.name}/modules/"] - } - } -} - diff --git a/docs/assets/terraform/5-appinfra/modules/service_catalog/outputs.tf b/docs/assets/terraform/5-appinfra/modules/service_catalog/outputs.tf deleted file mode 100644 index 9efe0568..00000000 --- a/docs/assets/terraform/5-appinfra/modules/service_catalog/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "storage_bucket_name" { - description = "Name of storage bucket created" - value = google_storage_bucket.bucket.name -} - -output "cloudbuild_trigger_id" { - description = "Id of Cloud Build Trigger" - value = google_cloudbuild_trigger.zip_files.id -} diff --git a/docs/assets/terraform/5-appinfra/modules/service_catalog/variables.tf b/docs/assets/terraform/5-appinfra/modules/service_catalog/variables.tf deleted file mode 100644 index c3e72086..00000000 --- a/docs/assets/terraform/5-appinfra/modules/service_catalog/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - description = "Name of the repository." - type = string -} -variable "region" { - description = "Location of the repository." - type = string -} - -variable "project_id" { - description = "Project ID" - type = string -} - -variable "gcs_bucket_prefix" { - description = "Prefix of the bucket name" - default = "bkt" -} - -variable "tf_service_catalog_sa_email" { - description = "Full email of the terraform service account for service-catalog" - type = string -} - -variable "machine_learning_project_number" { - description = "Project Number for the Machine Learning (Vertex) Project" - type = string -} - -variable "kms_crypto_key" { - description = "KMS Key to be used" - type = string -} - -variable "log_bucket" { - description = "Bucket to store logs from service catalog bucket" - type = string -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/README.md b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/common.auto.example.tfvars b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/common.auto.example.tfvars deleted file mode 100644 index a9f152f5..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/common.auto.example.tfvars +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -instance_region = "us-central1" // should be one of the regions used to create network on step 3-networks - -remote_state_bucket = "REMOTE_STATE_BUCKET" - -log_bucket = "REPLACE_LOG_BUCKET" - -# github_ api_ token = "PUT IN TOKEN" - -# github_app_installation_id = "18685983" - -# github_remote_uri = "https://github.com/badal-io/ml-foundations-tf-modules.git" - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md deleted file mode 100644 index 849fc76c..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/README.md +++ /dev/null @@ -1,16 +0,0 @@ - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| instance\_region | The region where compute instance will be created. A subnetwork must exists in the instance region. | `string` | n/a | yes | -| remote\_state\_bucket | Backend bucket to load remote state information from previous steps. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| cloudbuild\_trigger\_id | Id of Cloud Build Trigger | -| storage\_bucket\_name | Name of storage bucket created | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/backend.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/backend.tf deleted file mode 100644 index 0b671313..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/backend.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - backend "gcs" { - bucket = "UPDATE_APP_INFRA_BUCKET" - prefix = "terraform/app-infra/ml_business_unit/shared" - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/common.auto.tfvars b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/common.auto.tfvars deleted file mode 120000 index 39aaa462..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/common.auto.tfvars +++ /dev/null @@ -1 +0,0 @@ -../../common.auto.tfvars \ No newline at end of file diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/locals.tf deleted file mode 100644 index 09352bd6..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/locals.tf +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - business_unit = "ml_business_unit" - environment = "common" -} - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/outputs.tf deleted file mode 100644 index 22d794b2..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "storage_bucket_name" { - description = "Name of storage bucket created" - value = module.service_catalog.storage_bucket_name -} - -output "cloudbuild_trigger_id" { - description = "Id of Cloud Build Trigger" - value = module.service_catalog.cloudbuild_trigger_id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/remote.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/remote.tf deleted file mode 100644 index 366949d7..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/remote.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - service_catalog_project_id = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id - service_catalog_repo_name = data.terraform_remote_state.projects_shared.outputs.service_catalog_repo_name - machine_learning_project_number = data.terraform_remote_state.machine_learning_development.outputs.machine_learning_project_number - tf_service_catalog_sa_email = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["ml-service-catalog"] - shared_keyrings = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings -} - -data "terraform_remote_state" "projects_shared" { - backend = "gcs" - - config = { - bucket = var.remote_state_bucket - prefix = "terraform/projects/${local.business_unit}/shared" - } -} - -data "terraform_remote_state" "machine_learning_development" { - backend = "gcs" - config = { - bucket = var.remote_state_bucket - prefix = "terraform/projects/${local.business_unit}/development" - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/service_catalog.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/service_catalog.tf deleted file mode 100644 index 2a031be5..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/service_catalog.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region] -} - -data "google_project" "common_svc_catalog" { - project_id = local.service_catalog_project_id -} - -module "service_catalog" { - source = "../../modules/service_catalog" - - project_id = local.service_catalog_project_id - region = var.instance_region - name = local.service_catalog_repo_name - machine_learning_project_number = local.machine_learning_project_number - tf_service_catalog_sa_email = local.tf_service_catalog_sa_email - - log_bucket = var.log_bucket - kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_svc_catalog.name}" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/variables.tf deleted file mode 100644 index debeaa58..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/variables.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_region" { - description = "The region where compute instance will be created. A subnetwork must exists in the instance region." - type = string -} - -variable "remote_state_bucket" { - description = "Backend bucket to load remote state information from previous steps." - type = string -} - -variable "log_bucket" { - description = "Log bucket to be used by Service Catalog Bucket" - type = string -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/versions.tf b/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/versions.tf deleted file mode 100644 index baa38abb..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-infra-repo/ml_business_unit/shared/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - - google = { - source = "hashicorp/google" - version = ">= 3.77, < 6" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 3.77, < 6" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.1" - } - - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/README.md deleted file mode 100644 index 7dde3147..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/README.md +++ /dev/null @@ -1,135 +0,0 @@ -# The Service Catalog - -## Overview - -This repo provides a number of the [Google Service Catalog](https://cloud.google.com/service-catalog) Terraform-based solution modules: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
artifact_registryArtifact Registry is the next generation of Container Registry. Store, manage, and secure your build artifacts.
bigqueryBigQuery is a serverless and cost-effective enterprise data warehouse that works across clouds and scales with your data. Use built-in ML/AI and BI for insights at scale.
bucketCloud Storage is a managed service for storing unstructured data.
composerCloud Composer is a fully managed workflow orchestration service built on Apache Airflow.
metadataVertex ML Metadata is a service that provides capabilities for managing the lifecycle of metadata consumed and produced by machine-learning (ML) workflows.
notebookVertex AI Workbench is a Jupyter notebook-based development environment for the entire data science workflow.
pubsubPub/Sub is an asynchronous and scalable messaging service that decouples services producing messages from services processing those messages.
secretsSecret Manager lets you store, manage, and access secrets as binary blobs or text strings. With the appropriate permissions, you can view the contents of the secret.
tensorboardVertex AI TensorBoard is an enterprise-ready managed service for machine learning experiment visualization.
- -## Building with Cloud Build - -The Service Catalog solutions are built and published as artifacts to the Google Cloud Storage bucket with the Cloud Build CI/CD pipeline, in the `service-catalog` project in the `common`` environment, as per the diagram below: - -![Service Catalog Workflow](img/workflow.png "Service Catalog Workflow") - -The build is triggered by code push into the `service-catalog` Git (or Cloud Source Repositories). The pipeline builds the solution artifacts (one per module) and copies them into the Cloud Storage Bucket. - -Once the modules are copied to the bucket, they can be used to create Terraform-based Solutions on the Service Catalog Admin page, see [Creating the configuration in Service Catalog](https://cloud.google.com/service-catalog/docs/terraform-configuration#create_config). - -The Solutions then can be added to one or more [Service Catalogs](https://cloud.google.com/service-catalog/docs/create-catalog) and then [shared](https://cloud.google.com/service-catalog/docs/share-catalog) with the other projects to be used with the Service Catalog solution browser. - -## Usage - -### Google APIs - -For the Service Catalog to be able to deploy the solution with the Cloud Build the following Google Cloud APIs should be enabled on the project: - -```bash -aiplatform.googleapis.com -artifactregistry.googleapis.com -bigquery.googleapis.com -cloudkms.googleapis.com -composer.googleapis.com -compute.googleapis.com -container.googleapis.com -containerregistry.googleapis.com -dataflow.googleapis.com -dataform.googleapis.com -notebooks.googleapis.com -pubsub.googleapis.com -secretmanager.googleapis.com -storage.googleapis.com -``` - -### IAM Roles - -To launch a solution using Service Catalog, user must have the following IAM roles assigned: - -```bash -roles/cloudbuild.builds.editor -roles/storage.admin -roles/viewer -``` - -As well as the Cloud Build Service Agent `PROJECT_NUMBER@cloudbuild.gserviceaccount.com` show be granted the following IAM roles: - -```bash -roles/aiplatform.admin -roles/artifactregistry.admin -roles/bigquery.admin -roles/cloudbuild.connectionAdmin -roles/composer.admin -roles/compute.admin -roles/compute.instanceAdmin.v1 -roles/compute.networkAdmin -roles/iam.roleAdmin -roles/iam.serviceAccountAdmin -roles/iam.serviceAccountUser -roles/notebooks.admin -roles/pubsub.admin -roles/resourcemanager.projectIamAdmin -roles/secretmanager.admin -roles/serviceusage.serviceUsageConsumer -roles/storage.admin -``` - -Most of the Services deployed with the Service Catalog requires access to KMS service, to be able to encrypt data with the Customer Managed Encryption Key (CMEK), created as part of the project onboarding process. Hence, the Service Agent of the corresponding service requires the IAM role `roles/cloudkms.cryptoKeyEncrypterDecrypter` on the project's KMS key. - -### Launch a Solution - -In Service Catalog, you see a union of solutions from all the catalogs you have access to in your Google Cloud hierarchy. - -To view available solutions, go to the Google Cloud console Service Catalog page. Solutions that have been shared with you appear in the main panel, see [Launch a Terraform configuration](https://cloud.google.com/service-catalog/docs/view-and-launch#launch_terraform) for more information. - -## Resources - -* [Service Catalog](https://cloud.google.com/service-catalog/docs) -* [Cloud Build](https://cloud.google.com/build/docs) -* [Cloud Storage](https://cloud.google.com/storage/docs) -* [Artifact Registry](https://cloud.google.com/artifact-registry/docs) -* [Big Query](https://cloud.google.com/bigquery/docs) -* [Cloud Composer](https://cloud.google.com/composer/docs) -* [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata/introduction) -* [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) -* [Pub/Sub](https://cloud.google.com/pubsub/docs) -* [Secret Manager](https://cloud.google.com/secret-manager/docs) -* [Vertex AI TensorBoard](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-introduction) diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/img/workflow.png b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/img/workflow.png deleted file mode 100644 index eaafbf50..00000000 Binary files a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/img/workflow.png and /dev/null differ diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/README.md deleted file mode 100644 index 0031846c..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/README.md +++ /dev/null @@ -1,74 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_artifact_registry_repository.registry](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_artifact_registry_repository) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cleanup\_policies](#input\_cleanup\_policies) | List of cleanup policies. | 
list(object({
    id     = string
    action = optional(string)
    condition = optional(list(object({
      tag_state             = optional(string)
      tag_prefixes          = optional(list(string))
      package_name_prefixes = optional(list(string))
      older_than            = optional(string)
    })))
    most_recent_versions = optional(list(object({
      package_name_prefixes = optional(list(string))
      keep_count            = optional(number)
    })))
  }))
| 
[
  {
    "action": "DELETE",
    "condition": [
      {
        "older_than": "2592000s",
        "tag_prefixes": [
          "alpha",
          "v0"
        ],
        "tag_state": "TAGGED"
      }
    ],
    "id": "delete-prerelease"
  }
]
| no | -| [cleanup\_policy\_dry\_run](#input\_cleanup\_policy\_dry\_run) | Whether to perform a dry run of the cleanup policy. | `bool` | `false` | no | -| [description](#input\_description) | Description of the repository. | `string` | `""` | no | -| [format](#input\_format) | Format of the repository. | `string` | `"DOCKER"` | no | -| [name](#input\_name) | Name of the repository. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| cleanup\_policies | List of cleanup policies. | 
list(object({
    id     = string
    action = optional(string)
    condition = optional(list(object({
      tag_state             = optional(string)
      tag_prefixes          = optional(list(string))
      package_name_prefixes = optional(list(string))
      older_than            = optional(string)
    })))
    most_recent_versions = optional(list(object({
      package_name_prefixes = optional(list(string))
      keep_count            = optional(number)
    })))
  }))
| 
[
  {
    "action": "DELETE",
    "condition": [
      {
        "older_than": "2592000s",
        "tag_prefixes": [
          "alpha",
          "v0"
        ],
        "tag_state": "TAGGED"
      }
    ],
    "id": "delete-prerelease"
  }
]
| no | -| cleanup\_policy\_dry\_run | Whether to perform a dry run of the cleanup policy. | `bool` | `false` | no | -| description | Description of the repository. | `string` | `""` | no | -| format | Format of the repository. | `string` | `"DOCKER"` | no | -| name | Name of the repository. | `string` | n/a | yes | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/data.tf deleted file mode 100644 index 0ce9ad7b..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/data.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * 1Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/locals.tf deleted file mode 100644 index c11543ac..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/locals.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * 2Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - keyring_name = "sample-keyring" - region_short_code = { - "us-central1" = "usc1" - "us-east4" = "use4" - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/main.tf deleted file mode 100644 index b40cbe91..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/main.tf +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_artifact_registry_repository" "registry" { - provider = google-beta - - project = data.google_project.project.project_id - location = var.region - repository_id = var.name - description = var.description - format = var.format - cleanup_policy_dry_run = var.cleanup_policy_dry_run - - #Customer Managed Encryption Keys - #Control ID: COM-CO-2.3 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - - kms_key_name = data.google_kms_crypto_key.key.id - - #Cleanup policy - #Control ID: AR-CO-6.1 - #NIST 800-53: SI-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - dynamic "cleanup_policies" { - for_each = var.cleanup_policies - content { - id = cleanup_policies.value.id - action = cleanup_policies.value.action - - dynamic "condition" { - for_each = cleanup_policies.value.condition != null ? [cleanup_policies.value.condition] : [] - content { - tag_state = condition.value[0].tag_state - tag_prefixes = condition.value[0].tag_prefixes - package_name_prefixes = condition.value[0].package_name_prefixes - older_than = condition.value[0].older_than - } - } - - dynamic "most_recent_versions" { - for_each = cleanup_policies.value.most_recent_versions != null ? [cleanup_policies.value.most_recent_versions] : [] - content { - package_name_prefixes = most_recent_versions.value[0].package_name_prefixes - keep_count = most_recent_versions.value[0].keep_count - } - } - } - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/terraform.tfvars.example b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/terraform.tfvars.example deleted file mode 100644 index 5b8b92cb..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/terraform.tfvars.example +++ /dev/null @@ -1,43 +0,0 @@ -region = "us-central1" -environment = "development" -project_id = "prj-c-composer" -cmek_project_id = "prj-c-bu3cmek-wo8w" -name = "test-repo" -description = "Artifact Repository" -format = "DOCKER" -cleanup_policy_dry_run = false - -cleanup_policies = [ - { - id = "delete-prerelease" - action = "DELETE" - condition = [ - { - tag_state = "TAGGED" - tag_prefixes = ["alpha", "v0"] - older_than = "2592000s" - } - ] - }, - { - id = "keep-tagged-release" - action = "KEEP" - condition = [ - { - tag_state = "TAGGED" - tag_prefixes = ["release"] - package_name_prefixes = ["webapp", "mobile"] - } - ] - }, - { - id = "keep-minimum-versions" - action = "KEEP" - most_recent_versions = [ - { - package_name_prefixes = ["webapp", "mobile", "sandbox"] - keep_count = 5 - } - ] - } -] diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/variables.tf deleted file mode 100644 index 99cb1a45..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/artifact_registry/variables.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - */ - -variable "name" { - description = "Name of the repository." - type = string -} - -variable "description" { - description = "Description of the repository." - type = string - default = "" -} - -variable "format" { - description = "Format of the repository." - type = string - default = "DOCKER" -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "cleanup_policy_dry_run" { - description = "Whether to perform a dry run of the cleanup policy." - type = bool - default = false -} - -variable "cleanup_policies" { - description = "List of cleanup policies." - type = list(object({ - id = string - action = optional(string) - condition = optional(list(object({ - tag_state = optional(string) - tag_prefixes = optional(list(string)) - package_name_prefixes = optional(list(string)) - older_than = optional(string) - }))) - most_recent_versions = optional(list(object({ - package_name_prefixes = optional(list(string)) - keep_count = optional(number) - }))) - })) - default = [ - { - id = "delete-prerelease" - action = "DELETE" - condition = [ - { - tag_state = "TAGGED" - tag_prefixes = ["alpha", "v0"] - older_than = "2592000s" - } - ] - } - ] -} - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/README.md deleted file mode 100644 index 8e898e26..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/README.md +++ /dev/null @@ -1,76 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bigquery](#module\_bigquery) | terraform-google-modules/bigquery/google | 7.0.0 | - -## Resources - -| Name | Type | -|------|------| -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | A unique ID for this dataset, without the project name. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (\_). The maximum length is 1,024 characters. | `string` | n/a | yes | -| [default\_partition\_expiration\_ms](#input\_default\_partition\_expiration\_ms) | The default partition expiration for all partitioned tables in the dataset, in milliseconds. Once this property is set, all newly-created partitioned tables in the dataset will have an expirationMs property in the timePartitioning settings set to this value, and changing the value will only affect new tables, not existing ones. The storage in a partition will have an expiration time of its partition time plus this value. | `number` | `null` | no | -| [default\_table\_expiration\_ms](#input\_default\_table\_expiration\_ms) | The default lifetime of all tables in the dataset, in milliseconds. The minimum value is 3600000 milliseconds (one hour). Once this property is set, all newly-created tables in the dataset will have an expirationTime property set to the creation time plus the value in this property, and changing the value will only affect new tables, not existing ones. When the expirationTime for a given table is reached, that table will be deleted automatically. If a table's expirationTime is modified or removed before the table expires, or if you provide an explicit expirationTime when creating a table, that value takes precedence over the default expiration time indicated by this property. | `number` | `null` | no | -| [delete\_contents\_on\_destroy](#input\_delete\_contents\_on\_destroy) | If true, delete all the tables in the dataset when destroying the dataset; otherwise, destroying the dataset does not affect the tables in the dataset. If you try to delete a dataset that contains tables, and you set delete\_contents\_on\_destroy to false when you created the dataset, the request will fail. Always use this flag with caution. A missing value is treated as false. | `bool` | `false` | no | -| [description](#input\_description) | A user-friendly description of the dataset | `string` | `""` | no | -| [friendly\_name](#input\_friendly\_name) | A descriptive name for the dataset | `string` | `""` | no | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| dataset\_id | A unique ID for this dataset, without the project name. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (\_). The maximum length is 1,024 characters. | `string` | n/a | yes | -| default\_partition\_expiration\_ms | The default partition expiration for all partitioned tables in the dataset, in milliseconds. Once this property is set, all newly-created partitioned tables in the dataset will have an expirationMs property in the timePartitioning settings set to this value, and changing the value will only affect new tables, not existing ones. The storage in a partition will have an expiration time of its partition time plus this value. | `number` | `null` | no | -| default\_table\_expiration\_ms | The default lifetime of all tables in the dataset, in milliseconds. The minimum value is 3600000 milliseconds (one hour). Once this property is set, all newly-created tables in the dataset will have an expirationTime property set to the creation time plus the value in this property, and changing the value will only affect new tables, not existing ones. When the expirationTime for a given table is reached, that table will be deleted automatically. If a table's expirationTime is modified or removed before the table expires, or if you provide an explicit expirationTime when creating a table, that value takes precedence over the default expiration time indicated by this property. | `number` | `null` | no | -| delete\_contents\_on\_destroy | If true, delete all the tables in the dataset when destroying the dataset; otherwise, destroying the dataset does not affect the tables in the dataset. If you try to delete a dataset that contains tables, and you set delete\_contents\_on\_destroy to false when you created the dataset, the request will fail. Always use this flag with caution. A missing value is treated as false. | `bool` | `false` | no | -| description | A user-friendly description of the dataset | `string` | `""` | no | -| friendly\_name | A descriptive name for the dataset | `string` | `""` | no | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/data.tf deleted file mode 100644 index 035a39b9..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/data.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/locals.tf deleted file mode 100644 index 543a3276..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/locals.tf +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - project_hash = substr(sha256(data.google_project.project.project_id), 0, 6) - name_var = "bq-${var.dataset_id}-${data.google_project.project.labels.env_code}-${local.project_hash}" - keyring_name = "sample-keyring" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/main.tf deleted file mode 100644 index 4939412a..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/main.tf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "bigquery" { - source = "terraform-google-modules/bigquery/google" - version = "7.0.0" - dataset_id = var.dataset_id - project_id = data.google_project.project.project_id - location = var.region - encryption_key = data.google_kms_crypto_key.key.id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/variables.tf deleted file mode 100644 index ba671067..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bigquery/variables.tf +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} - -variable "dataset_id" { - description = "A unique ID for this dataset, without the project name. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). The maximum length is 1,024 characters." - type = string -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "friendly_name" { - description = "A descriptive name for the dataset" - type = string - default = "" -} - -variable "description" { - description = "A user-friendly description of the dataset" - type = string - default = "" -} - -variable "default_partition_expiration_ms" { - description = "The default partition expiration for all partitioned tables in the dataset, in milliseconds. Once this property is set, all newly-created partitioned tables in the dataset will have an expirationMs property in the timePartitioning settings set to this value, and changing the value will only affect new tables, not existing ones. The storage in a partition will have an expiration time of its partition time plus this value." - type = number - default = null -} - -variable "default_table_expiration_ms" { - description = "The default lifetime of all tables in the dataset, in milliseconds. The minimum value is 3600000 milliseconds (one hour). Once this property is set, all newly-created tables in the dataset will have an expirationTime property set to the creation time plus the value in this property, and changing the value will only affect new tables, not existing ones. When the expirationTime for a given table is reached, that table will be deleted automatically. If a table's expirationTime is modified or removed before the table expires, or if you provide an explicit expirationTime when creating a table, that value takes precedence over the default expiration time indicated by this property." - type = number - default = null -} - -variable "delete_contents_on_destroy" { - description = "If true, delete all the tables in the dataset when destroying the dataset; otherwise, destroying the dataset does not affect the tables in the dataset. If you try to delete a dataset that contains tables, and you set delete_contents_on_destroy to false when you created the dataset, the request will fail. Always use this flag with caution. A missing value is treated as false." - type = bool - default = false -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/README.md deleted file mode 100644 index e351f625..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/README.md +++ /dev/null @@ -1,149 +0,0 @@ -## IAM Permission Requirements - -To execute the provided Terraform configuration the following IAM permissions are required: - -- `cloudkms.cryptoKeys.get` -- `cloudkms.cryptoKeys.setIamPolicy` -- `iam.serviceAccounts.create` -- `iam.serviceAccounts.update` -- `storage.hmacKeys.create` -- `storage.hmacKeys.get` -- `storage.buckets.create` -- `storage.buckets.get` -- `storage.buckets.update` -- `storage.buckets.setIamPolicy` -- `storage.buckets.setLifecycle` -- `storage.objects.create` -- `storage.objects.delete` -- `resourcemanager.projects.get` - -## Notes: -- Additional permissions may be required based on specific use cases and actions within these resources. -- It's recommended to adhere to the principle of least privilege and grant only the permissions necessary for the tasks. -- Assign these permissions via predefined roles or create a custom IAM role encompassing all necessary permissions. -- Always review and adjust permissions according to organizational security policies. - - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | -| [google_storage_bucket_object.root_folder](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_projects.log](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_random\_suffix](#input\_add\_random\_suffix) | whether to add a random suffix to the bucket name | `bool` | `false` | no | -| [dual\_region\_locations](#input\_dual\_region\_locations) | dual region description | `list(string)` | `[]` | no | -| [force\_destroy](#input\_force\_destroy) | (Optional, Default: true) When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | `bool` | `true` | no | -| [gcs\_bucket\_prefix](#input\_gcs\_bucket\_prefix) | Name prefix to be used for GCS Bucket | `string` | `"bkt"` | no | -| [labels](#input\_labels) | Labels to be attached to the buckets | `map(string)` | 
{
  "classification": "dataclassification",
  "label": "samplelabel",
  "owner": "testowner"
}
| no | -| [lifecycle\_rules](#input\_lifecycle\_rules) | List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches\_storage\_class should be a comma delimited string. | 
set(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = map(string)

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = map(string)
  }))
| 
[
  {
    "action": {
      "storage_class": "NEARLINE",
      "type": "SetStorageClass"
    },
    "condition": {
      "age": "30",
      "matches_storage_class": "REGIONAL"
    }
  },
  {
    "action": {
      "type": "Delete"
    },
    "condition": {
      "with_state": "ARCHIVED"
    }
  }
]
| no | -| [name](#input\_name) | name of bucket | `string` | n/a | yes | -| [object\_folder\_temporary\_hold](#input\_object\_folder\_temporary\_hold) | Set root folder temporary hold according to security control GCS-CO-6.16, toggle off to allow for object deletion. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| [requester\_pays](#input\_requester\_pays) | Enables Requester Pays on a storage bucket. | `bool` | `false` | no | -| [retention\_policy](#input\_retention\_policy) | Map of retention policy values. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket#retention_policy | `any` | `{}` | no | -| [storage\_class](#input\_storage\_class) | Storage class to create the bucket | `string` | `"STANDARD"` | no | -| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Whether to have uniform access levels or not | `bool` | `true` | no | -| [versioning\_enabled](#input\_versioning\_enabled) | Whether to enable versioning or not | `bool` | `true` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [storage\_bucket](#output\_storage\_bucket) | Storage Bucket. | - - -## Security Controls - -The following table outlines which of the suggested controls for Vertex Generative AI are enabled in this module. -| Name | Control ID | NIST 800-53 | CRI Profile | Category | Source Blueprint -|------|------------|-------------|-------------|----------| ----------------| -|Customer Managed Encryption Keys| COM-CO-2.3| SC-12
SC-13| PR.DS-1.1
PR.DS-2.1
PR.DS-2.2
PR.DS-5.1 | Recommended | Secure Foundation v4 -|Regional Storage Class Lifecycle Rule | GCS-CO-6.11 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Regional Storage Class Lifecycle Rule | GCS-CO-6.12 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Ensure Lifecycle management is enabled 1 of 2 | GCS-CO-6.13 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Ensure Lifecycle management is enabled 2 of 2 | GCS-CO-6.14 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Ensure Retention policy is using the bucket lock| GCS-CO-6.15 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Object contains a temporary hold and should be evaluated| GCS-CO-6.16 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Retention Policy| GCS-CO-6.17 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Classification Tag| GCS-CO-6.18 | SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Versioning is Enabled| GCS-CO-6.2| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Log Bucket Exists| GCS-CO-6.3| AU-2
AU-3
AU-8
AU-9| DM.ED-7.1
DM.ED-7.2
DM.ED-7.3
DM.ED-7.4
PR.IP-1.4 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Labeling Tag| GCS-CO-6.4| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Deletion Rules| GCS-CO-6.5| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Deletion Rules For Deleted Objects| GCS-CO-6.6| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Ensure that versioning is enabled on all Cloud Storage instances| GCS-CO-6.7| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Owner Tag| GCS-CO-6.8| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Ensure HMAC keys for service accounts are handled correctly| GCS-CO-6.9| SI-12
SC-13 | PR.IP-1.1
PR.IP-1.2
PR.IP-2.1
PR.DS-2.2
PR.DS-5.1 | Required | ML Foundation v0.1.0-alpha.1 -|Owner Tag| GCS-CO-7.1| SI-12 | PR.IP-2.1
PR.IP-2.2
PR.IP-2.3 | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 - - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_random\_suffix | whether to add a random suffix to the bucket name | `bool` | `false` | no | -| dual\_region\_locations | dual region description | `list(string)` | `[]` | no | -| force\_destroy | (Optional, Default: true) When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | `bool` | `true` | no | -| gcs\_bucket\_prefix | Name prefix to be used for GCS Bucket | `string` | `"bkt"` | no | -| labels | Labels to be attached to the buckets | `map(string)` | 
{
  "classification": "dataclassification",
  "label": "samplelabel",
  "owner": "testowner"
}
| no | -| lifecycle\_rules | List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches\_storage\_class should be a comma delimited string. | 
set(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = map(string)

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = map(string)
  }))
| 
[
  {
    "action": {
      "storage_class": "NEARLINE",
      "type": "SetStorageClass"
    },
    "condition": {
      "age": "30",
      "matches_storage_class": "REGIONAL"
    }
  },
  {
    "action": {
      "type": "Delete"
    },
    "condition": {
      "with_state": "ARCHIVED"
    }
  }
]
| no | -| name | name of bucket | `string` | n/a | yes | -| object\_folder\_temporary\_hold | Set root folder temporary hold according to security control GCS-CO-6.16, toggle off to allow for object deletion. | `bool` | `false` | no | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| requester\_pays | Enables Requester Pays on a storage bucket. | `bool` | `false` | no | -| retention\_policy | Map of retention policy values. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket#retention_policy | `any` | `{}` | no | -| storage\_class | Storage class to create the bucket | `string` | `"STANDARD"` | no | -| uniform\_bucket\_level\_access | Whether to have uniform access levels or not | `bool` | `true` | no | -| versioning\_enabled | Whether to enable versioning or not | `bool` | `true` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| storage\_bucket | Storage Bucket. | - - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/data.tf deleted file mode 100644 index 61a44b27..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/data.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_projects" "log" { - filter = "labels.application_name:env-logging labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/locals.tf deleted file mode 100644 index c640c38e..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/locals.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - log_bucket_prefix = "bkt" - keyring_name = "sample-keyring" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/main.tf deleted file mode 100644 index 1ab54cbf..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/main.tf +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_storage_bucket" "bucket" { - provider = google-beta - name = join("-", [var.gcs_bucket_prefix, data.google_projects.log.projects.0.labels.env_code, var.name]) - project = data.google_project.project.project_id - location = upper(var.region) - - dynamic "custom_placement_config" { - for_each = length(var.dual_region_locations) != 0 ? [1] : [] - content { - data_locations = var.dual_region_locations - } - } - - force_destroy = var.force_destroy - uniform_bucket_level_access = var.uniform_bucket_level_access - storage_class = var.storage_class - public_access_prevention = "enforced" - - #Versioning is Enabled - #Control ID: GCS-CO-6.2 and GCS-CO-6.7 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - versioning { - enabled = var.versioning_enabled - } - - #Labeling Tag - #Control ID: GCS-CO-6.4 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - labels = var.labels - - #Retention Policy - #Control ID: GCS-CO-6.17 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - dynamic "retention_policy" { - for_each = var.retention_policy != {} ? [var.retention_policy] : [] - content { - - #Ensure Retention policy is using the bucket lock - #Control ID: GCS-CO-6.13 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - is_locked = lookup(retention_policy.value, "is_locked", null) - retention_period = lookup(retention_policy.value, "retention_period", null) - } - } - - #Ensure Lifecycle management is enabled 1 of 2 - #Control ID: GCS-CO-6.13 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - #Ensure Lifecycle management is enabled 2 of 2 - #Control ID: GCS-CO-6.14 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules - content { - action { - type = lifecycle_rule.value.action.type - - #Regional Storage Class Lifecycle Rule - #Control ID: GCS-CO-6.11 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) - } - condition { - age = lookup(lifecycle_rule.value.condition, "age", null) - created_before = lookup(lifecycle_rule.value.condition, "created_before", null) - with_state = lookup(lifecycle_rule.value.condition, "with_state", lookup(lifecycle_rule.value.condition, "is_live", false) ? "LIVE" : null) - - #Regional Storage Class Lifecycle Rule - #Control ID: GCS-CO-6.12 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - matches_storage_class = contains(keys(lifecycle_rule.value.condition), "matches_storage_class") ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null - num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) - custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) - days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) - days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) - noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) - } - } - } - - #Customer Managed Encryption Keys - #Control ID: COM-CO-2.3 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - - encryption { - default_kms_key_name = data.google_kms_crypto_key.key.id - } - - #Log Bucket Exists - #Control ID: GCS-CO-6.3 and GCS-CO-7.1 - #NIST 800-53: AU-2 AU-3 AU-8 AU-9 - #CRI Profile: DM.ED-7.1 DM.ED-7.2 DM.ED-7.3 DM.ED-7.4 PR.IP-1.4 - - logging { - log_bucket = join("-", [local.log_bucket_prefix, data.google_projects.log.projects.0.project_id]) - } -} - -resource "google_storage_bucket_object" "root_folder" { - name = "root/" - content = " " - bucket = google_storage_bucket.bucket.name - - #Object contains a temporary hold and should be evaluated - #Control ID: GCS-CO-6.16 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - temporary_hold = var.object_folder_temporary_hold - - #Customer Managed Encryption Keys - #Control ID: COM-CO-2.3 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/outputs.tf deleted file mode 100644 index 248be99c..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "storage_bucket" { - description = "Storage Bucket." - value = google_storage_bucket.bucket -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/variables.tf deleted file mode 100644 index 9abddef7..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/bucket/variables.tf +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - type = string - description = "name of bucket" -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "dual_region_locations" { - type = list(string) - default = [] - description = "dual region description" - validation { - condition = length(var.dual_region_locations) == 0 || length(var.dual_region_locations) == 2 - error_message = "Exactly 0 or 2 regions expected." - } -} - -variable "force_destroy" { - type = bool - description = "(Optional, Default: true) When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run." - default = true -} - -variable "versioning_enabled" { - type = bool - description = "Whether to enable versioning or not" - default = true -} - -variable "lifecycle_rules" { - type = set(object({ - # Object with keys: - # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. - # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. - action = map(string) - - # Object with keys: - # - age - (Optional) Minimum age of an object in days to satisfy this condition. - # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. - # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". - # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL. - # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. - # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. - # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. - # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. - # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. - condition = map(string) - })) - description = "List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches_storage_class should be a comma delimited string." - default = [ - { - #Deletion Rules - #Control ID: GCS-CO-6.5 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - action = { - type = "SetStorageClass" - storage_class = "NEARLINE" - } - condition = { - age = "30" - matches_storage_class = "REGIONAL" - } - }, - { - #Deletion Rules - #Control ID: GCS-CO-6.6 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - action = { - type = "Delete" - } - condition = { - with_state = "ARCHIVED" - } - } - ] -} - -variable "retention_policy" { - type = any - default = {} - description = "Map of retention policy values. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket#retention_policy" -} - -variable "object_folder_temporary_hold" { - type = bool - default = false - description = "Set root folder temporary hold according to security control GCS-CO-6.16, toggle off to allow for object deletion." -} - -#Labeling Tag -#Control ID: GCS-CO-6.4 -#NIST 800-53: SC-12 -#CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - -variable "labels" { - description = "Labels to be attached to the buckets" - type = map(string) - default = { - #Labelling tag - #Control ID: GCS-CO-6.4 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - label = "samplelabel" - - #Owner Tag - #Control ID: GCS-CO-6.8 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - owner = "testowner" - - #Classification Tag - #Control ID: GCS-CO-6.18 - #NIST 800-53: SC-12 - #CRI Profile: PR.IP-2.1 PR.IP-2.2 PR.IP-2.3 - - classification = "dataclassification" - } -} - -variable "add_random_suffix" { - description = "whether to add a random suffix to the bucket name" - type = bool - default = false -} - -variable "uniform_bucket_level_access" { - description = "Whether to have uniform access levels or not" - type = bool - default = true -} - -variable "storage_class" { - type = string - description = "Storage class to create the bucket" - default = "STANDARD" - validation { - condition = contains(["STANDARD", "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "ARCHIVE"], var.storage_class) - error_message = "Storage class can be one of STANDARD, MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE." - } -} - -variable "requester_pays" { - description = "Enables Requester Pays on a storage bucket." - type = bool - default = false -} - -variable "gcs_bucket_prefix" { - description = "Name prefix to be used for GCS Bucket" - type = string - default = "bkt" -} - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/README.md deleted file mode 100644 index 87e59ee6..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/README.md +++ /dev/null @@ -1,129 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | -| [random](#provider\_random) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 8.1 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_cloudbuildv2_connection.repo_connect](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_cloudbuildv2_connection) | resource | -| [google-beta_google_cloudbuildv2_repository.repo](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_cloudbuildv2_repository) | resource | -| [google-beta_google_composer_environment.cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_composer_environment) | resource | -| [google_cloudbuild_trigger.zip_files](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | -| [google_secret_manager_secret_iam_policy.policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_policy) | resource | -| [google_service_account.trigger_sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | -| [google_service_account_iam_member.trigger_sa_impersonate](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_iam_policy.serviceagent_secretAccessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_netblock_ip_ranges.health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.iap_forwarders](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.legacy_health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_pubsub_topic.secret_rotations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/pubsub_topic) | data source | -| [google_secret_manager_secret.github_api_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/secret_manager_secret) | data source | -| [google_secret_manager_secret_version.github_api](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/secret_manager_secret_version) | data source | -| [google_service_account.composer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/service_account) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [airflow\_config\_overrides](#input\_airflow\_config\_overrides) | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | -| [env\_variables](#input\_env\_variables) | Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z\_][a-zA-Z0-9\_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW\_\_[A-Z0-9\_]+\_\_[A-Z0-9\_]+), and they cannot match any of the following reserved names: [AIRFLOW\_HOME,C\_FORCE\_ROOT,CONTAINER\_NAME,DAGS\_FOLDER,GCP\_PROJECT,GCS\_BUCKET,GKE\_CLUSTER\_NAME,SQL\_DATABASE,SQL\_INSTANCE,SQL\_PASSWORD,SQL\_PROJECT,SQL\_REGION,SQL\_USER] | `map(any)` | `{}` | no | -| [github\_app\_installation\_id](#input\_github\_app\_installation\_id) | The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build | `number` | n/a | yes | -| [github\_name\_prefix](#input\_github\_name\_prefix) | A name for your github connection to cloubuild | `string` | `"github-modules"` | no | -| [github\_remote\_uri](#input\_github\_remote\_uri) | Url of your github repo | `string` | n/a | yes | -| [github\_secret\_name](#input\_github\_secret\_name) | Name of the github secret to extract github token info | `string` | `"github-api-token"` | no | -| [image\_version](#input\_image\_version) | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.5.2-airflow-2.6.3"` | no | -| [labels](#input\_labels) | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | -| [maintenance\_window](#input\_maintenance\_window) | The configuration settings for Cloud Composer maintenance window. | 
object({
    start_time = string
    end_time   = string
    recurrence = string
  })
| 
{
  "end_time": "2021-01-01T13:00:00Z",
  "recurrence": "FREQ=WEEKLY;BYDAY=SU",
  "start_time": "2021-01-01T01:00:00Z"
}
| no | -| [name](#input\_name) | name of the Composer environment | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Optional project ID where Cloud Composer Environment is created. | `string` | `null` | no | -| [pypi\_packages](#input\_pypi\_packages) | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | -| [python\_version](#input\_python\_version) | The default version of Python used to run the Airflow scheduler, worker, and webserver processes. | `string` | `"3"` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| [service\_account\_prefix](#input\_service\_account\_prefix) | Name prefix to use for service accounts. | `string` | `"sa"` | no | -| [web\_server\_allowed\_ip\_ranges](#input\_web\_server\_allowed\_ip\_ranges) | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied. | 
list(object({
    value       = string
    description = string
  }))
| `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [airflow\_uri](#output\_airflow\_uri) | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | -| [composer\_env\_id](#output\_composer\_env\_id) | ID of Cloud Composer Environment. | -| [composer\_env\_name](#output\_composer\_env\_name) | Name of the Cloud Composer Environment. | -| [gcs\_bucket](#output\_gcs\_bucket) | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | -| [gke\_cluster](#output\_gke\_cluster) | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | - - -## Security Controls - -The following table outlines which of the suggested controls for Vertex Generative AI are enabled in this module. -| Name | Control ID | NIST 800-53 | CRI Profile | Category | Source Blueprint -|------|------------|-------------|-------------|----------| ----------------| -|Customer Managed Encryption Keys| COM-CO-2.3| SC-12
SC-13| PR.DS-1.1
PR.DS-2.1
PR.DS-2.2
PR.DS-5.1 | Recommended | Secure Foundation v4 - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | -| env\_variables | Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z\_][a-zA-Z0-9\_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW\_\_[A-Z0-9\_]+\_\_[A-Z0-9\_]+), and they cannot match any of the following reserved names: [AIRFLOW\_HOME,C\_FORCE\_ROOT,CONTAINER\_NAME,DAGS\_FOLDER,GCP\_PROJECT,GCS\_BUCKET,GKE\_CLUSTER\_NAME,SQL\_DATABASE,SQL\_INSTANCE,SQL\_PASSWORD,SQL\_PROJECT,SQL\_REGION,SQL\_USER] | `map(any)` | `{}` | no | -| github\_app\_installation\_id | The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build | `number` | n/a | yes | -| github\_name\_prefix | A name for your github connection to cloubuild | `string` | `"github-modules"` | no | -| github\_remote\_uri | Url of your github repo | `string` | n/a | yes | -| github\_secret\_name | Name of the github secret to extract github token info | `string` | `"github-api-token"` | no | -| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.5.2-airflow-2.6.3"` | no | -| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | -| maintenance\_window | The configuration settings for Cloud Composer maintenance window. | 
object({
    start_time = string
    end_time   = string
    recurrence = string
  })
| 
{
  "end_time": "2021-01-01T13:00:00Z",
  "recurrence": "FREQ=WEEKLY;BYDAY=SU",
  "start_time": "2021-01-01T01:00:00Z"
}
| no | -| name | name of the Composer environment | `string` | n/a | yes | -| project\_id | Optional project ID where Cloud Composer Environment is created. | `string` | `null` | no | -| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | -| python\_version | The default version of Python used to run the Airflow scheduler, worker, and webserver processes. | `string` | `"3"` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| service\_account\_prefix | Name prefix to use for service accounts. | `string` | `"sa"` | no | -| web\_server\_allowed\_ip\_ranges | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied. | 
list(object({
    value       = string
    description = string
  }))
| `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | -| composer\_env\_id | ID of Cloud Composer Environment. | -| composer\_env\_name | Name of the Cloud Composer Environment. | -| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | -| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/data.tf deleted file mode 100644 index 5d66e4b3..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/data.tf +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_netblock_ip_ranges" "legacy_health_checkers" { - range_type = "legacy-health-checkers" -} - -data "google_netblock_ip_ranges" "health_checkers" { - range_type = "health-checkers" -} - -// Cloud IAP's TCP forwarding netblock -data "google_netblock_ip_ranges" "iap_forwarders" { - range_type = "iap-forwarders" -} - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_service_account" "composer" { - account_id = format("%s-%s-%s", var.service_account_prefix, data.google_project.project.labels.env_code, "composer") - project = data.google_project.project.project_id -} - -data "google_pubsub_topic" "secret_rotations" { - name = "secret-rotation-notifications" - project = data.google_project.project.project_id -} - -data "google_secret_manager_secret" "github_api_secret" { - secret_id = var.github_secret_name - project = data.google_project.project.project_id -} - -data "google_secret_manager_secret_version" "github_api" { - secret = data.google_secret_manager_secret.github_api_secret.id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/locals.tf deleted file mode 100644 index 795efcc1..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/locals.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - composer_node_use4 = "172.16.8.0/22" - composer_node_usc1 = "172.17.8.0/22" - - # secondary - pods_use4 = "172.18.0.0/16" - services_use4 = "172.16.12.0/22" - - pods_usc1 = "172.19.0.0/16" - services_usc1 = "172.17.12.0/22" - - # composer specific - composer_master_use4 = "192.168.0.0/28" - composer_master_usc1 = "192.168.1.0/28" - - composer_webserver_use4 = "192.168.2.0/29" - composer_webserver_usc1 = "192.168.3.0/29" - - private_service_connect_ip = "10.116.46.2" - - keyring_name = "sample-keyring" - - sa_name = format("%s-%s", data.google_project.project.labels.env_code, var.name) - - labels = merge( - var.labels, - { - "environment" = data.google_project.project.labels.environment - "env_code" = data.google_project.project.labels.env_code - } - ) - region_short_code = { - "us-central1" = "usc1" - "us-east4" = "use4" - } - zones = { - "us-central1" = ["a", "b", "c"] - "us-east4" = ["a", "b", "c"] - } - network_name = var.region == "us-central1" ? "composer-vpc-usc1" : "composer-vpc-use4" - subnetwork = var.region == "us-central1" ? "composer-primary-usc1" : "composer-primary-use4" - services_secondary_range_name = var.region == "us-central1" ? "composer-services-primary-usc1" : "composer-services-primary-use4" - cluster_secondary_range_name = var.region == "us-central1" ? "pods-primary-usc1" : "pods-primary-use4" - - service_agents = [ - "artifactregistry.googleapis.com", - "composer.googleapis.com", - "compute.googleapis.com", - "container.googleapis.com", - "pubsub.googleapis.com", - "storage.googleapis.com", - "secretmanager.googleapis.com" - ] - - tags = var.region == "us-central1" ? ["composer-usc1"] : ["composer-use4"] - - github_repository = replace(var.github_remote_uri, "https://", "") -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/main.tf deleted file mode 100644 index 193102f9..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/main.tf +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_shuffle" "zones" { - input = local.zones[var.region] - result_count = 1 -} - -resource "google_composer_environment" "cluster" { - provider = google-beta - - project = data.google_project.project.project_id - name = var.name - region = var.region - labels = local.labels - - config { - node_config { - network = "projects/${data.google_project.project.project_id}/global/networks/${local.network_name}" - subnetwork = "projects/${data.google_project.project.project_id}/regions/${var.region}/subnetworks/${local.subnetwork}" - service_account = data.google_service_account.composer.email - tags = local.tags - - ip_allocation_policy { - cluster_secondary_range_name = local.cluster_secondary_range_name - services_secondary_range_name = local.services_secondary_range_name - } - } - - private_environment_config { - enable_private_endpoint = true - master_ipv4_cidr_block = var.region == "us-central1" ? "192.168.1.0/28" : "192.168.0.0/28" - cloud_sql_ipv4_cidr_block = var.region == "us-central1" ? "192.168.5.0/24" : "192.168.4.0/24" - } - - maintenance_window { - start_time = var.maintenance_window.start_time - end_time = var.maintenance_window.end_time - recurrence = var.maintenance_window.recurrence - } - - dynamic "web_server_network_access_control" { - for_each = var.web_server_allowed_ip_ranges == null ? [] : [1] - content { - dynamic "allowed_ip_range" { - for_each = var.web_server_allowed_ip_ranges - content { - value = allowed_ip_range.value.value - description = allowed_ip_range.value.description - } - } - } - } - - # allow the capability to set software overrides - dynamic "software_config" { - for_each = var.python_version != "" ? [ - { - airflow_config_overrides = var.airflow_config_overrides - env_variables = var.env_variables - image_version = var.image_version - pypi_packages = var.pypi_packages - }] : [] - content { - airflow_config_overrides = software_config.value["airflow_config_overrides"] - env_variables = software_config.value["env_variables"] - image_version = software_config.value["image_version"] - pypi_packages = software_config.value["pypi_packages"] - } - } - - encryption_config { - kms_key_name = data.google_kms_crypto_key.key.id - } - } - - depends_on = [ - module.vpc, - ] -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/outputs.tf deleted file mode 100644 index 499525d6..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/outputs.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "composer_env_name" { - value = google_composer_environment.cluster.name - description = "Name of the Cloud Composer Environment." -} - -output "composer_env_id" { - value = google_composer_environment.cluster.id - description = "ID of Cloud Composer Environment." -} - -output "gke_cluster" { - value = google_composer_environment.cluster.config.0.gke_cluster - description = "Google Kubernetes Engine cluster used to run the Cloud Composer Environment." -} - -output "gcs_bucket" { - value = google_composer_environment.cluster.config.0.dag_gcs_prefix - description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." -} - -output "airflow_uri" { - value = google_composer_environment.cluster.config.0.airflow_uri - description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/pipeline.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/pipeline.tf deleted file mode 100644 index 8258abf4..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/pipeline.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_iam_policy" "serviceagent_secretAccessor" { - binding { - role = "roles/secretmanager.secretAccessor" - members = ["serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com"] - } -} - -resource "google_secret_manager_secret_iam_policy" "policy" { - project = data.google_secret_manager_secret.github_api_secret.project - secret_id = data.google_secret_manager_secret.github_api_secret.secret_id - policy_data = data.google_iam_policy.serviceagent_secretAccessor.policy_data -} - -resource "google_cloudbuildv2_connection" "repo_connect" { - provider = google-beta - project = data.google_project.project.project_id - location = var.region - name = "${var.github_name_prefix}-connection" - - github_config { - app_installation_id = var.github_app_installation_id - authorizer_credential { - oauth_token_secret_version = data.google_secret_manager_secret_version.github_api.id - } - } - depends_on = [google_secret_manager_secret_iam_policy.policy] -} - -resource "google_cloudbuildv2_repository" "repo" { - provider = google-beta - project = data.google_project.project.project_id - location = var.region - name = "${var.github_name_prefix}-repo" - parent_connection = google_cloudbuildv2_connection.repo_connect.id - remote_uri = var.github_remote_uri -} - -###### Added in but not used yet ######## -resource "google_service_account" "trigger_sa" { - account_id = "sa-apps-${local.sa_name}" - project = data.google_project.project.project_id - description = "Service account for Cloud Build in ${data.google_project.project.project_id}" -} -###### Added in but not used yet ######## -resource "google_service_account_iam_member" "trigger_sa_impersonate" { - service_account_id = google_service_account.trigger_sa.id - role = "roles/iam.serviceAccountTokenCreator" - member = "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com" -} - -resource "google_cloudbuild_trigger" "zip_files" { - name = "zip-tf-files-trigger" - project = data.google_project.project.project_id - location = var.region - - repository_event_config { - repository = google_cloudbuildv2_repository.repo.id - push { - branch = "^${local.labels.environment}$" - } - } - build { - step { - id = "unshallow" - name = "gcr.io/cloud-builders/git" - secret_env = ["token"] - entrypoint = "/bin/bash" - args = [ - "-c", - "git fetch --unshallow https://$token@${local.github_repository}" - ] - - } - available_secrets { - secret_manager { - env = "token" - version_name = data.google_secret_manager_secret.github_api_secret.name - } - } - step { - id = "find-folders-affected-in-push" - name = "gcr.io/cloud-builders/gsutil" - entrypoint = "/bin/bash" - args = [ - "-c", - <<-EOT - changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r) - dags=$(echo "$changed_files" | xargs basename | sort | uniq ) - - for dag in $dags; do - echo "Found change in DAG: $dag" - (cd dags && zip /workspace/$dag.zip $dag) - done - EOT - ] - } - step { - id = "push-to-bucket" - name = "gcr.io/cloud-builders/gsutil" - args = ["cp", "/workspace/*.zip", "${google_composer_environment.cluster.config.0.dag_gcs_prefix}/"] - } - } - - depends_on = [google_composer_environment.cluster, google_cloudbuildv2_repository.repo] -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/terraform.tfvars.example b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/terraform.tfvars.example deleted file mode 100644 index 3d4db2c1..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/terraform.tfvars.example +++ /dev/null @@ -1,29 +0,0 @@ -name = "isolated-composer-env10" -environment = "development" -project_id = "prj-c-composer" -cmek_project_id = "prj-c-bu3cmek-wo8w" -region = "us-central1" -maintenance_window = { - start_time = "2023-01-01T01:00:00Z" - end_time = "2023-01-01T13:00:00Z" - recurrence = "FREQ=WEEKLY;BYDAY=SU" -} -airflow_config_overrides = { "core-dags_are_paused_at_creation" = "True" } -env_variables = { "EXAMPLE_VAR" = "value" } -image_version = "composer-2.5.2-airflow-2.6.3" - -web_server_allowed_ip_ranges = [ - { - value = "192.168.100.0/24" - description = "Office network" - }, - { - value = "192.168.101.0/24" - description = "Home network" - } -] - -github_name_prefix = "github-composer-cloudbuild" -github_app_installation_id = "APP_INSTALATION_ID_HERE" -github_api_token = "GITHUB_API_TOKEN_HERE" -github_remote_uri = "LINK_TO_GITHUB_REPO_CONTAINING_DAGS" diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/variables.tf deleted file mode 100644 index ea77a199..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/variables.tf +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - type = string - description = "name of the Composer environment" -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "labels" { - type = map(string) - description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer." - default = {} -} - -variable "maintenance_window" { - type = object({ - start_time = string - end_time = string - recurrence = string - }) - - description = "The configuration settings for Cloud Composer maintenance window." - - # Set Start time, Timezone, Days, and Length, so that combined time for the - # specified schedule is at least 12 hours in a 7-day rolling window. For example, - # a period of 4 hours every Monday, Wednesday, and Friday provides the required amount of time. - - # 12-hour maintenance window between 01:00 and 13:00 (UTC) on Sundays - default = { - start_time = "2021-01-01T01:00:00Z" - end_time = "2021-01-01T13:00:00Z" - recurrence = "FREQ=WEEKLY;BYDAY=SU" - } -} - -################################################ -# software_config # -################################################ -variable "airflow_config_overrides" { - type = map(string) - description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"." - default = {} -} - -variable "env_variables" { - type = map(any) - description = "Additional environment variables to provide to the Apache Airflow scheduler, worker, and webserver processes. Environment variable names must match the regular expression [a-zA-Z_][a-zA-Z0-9_]*. They cannot specify Apache Airflow software configuration overrides (they cannot match the regular expression AIRFLOW__[A-Z0-9_]+__[A-Z0-9_]+), and they cannot match any of the following reserved names: [AIRFLOW_HOME,C_FORCE_ROOT,CONTAINER_NAME,DAGS_FOLDER,GCP_PROJECT,GCS_BUCKET,GKE_CLUSTER_NAME,SQL_DATABASE,SQL_INSTANCE,SQL_PASSWORD,SQL_PROJECT,SQL_REGION,SQL_USER]" - default = {} -} - -variable "image_version" { - type = string - description = "The version of the aiflow running in the cloud composer environment." - default = "composer-2.5.2-airflow-2.6.3" - validation { - condition = can(regex("^composer-([2-9]|[1-9][0-9]+)\\..*$", var.image_version)) - error_message = "The airflow_image_version must be GCP Composer version 2 or higher (e.g., composer-2.x.x-airflow-x.x.x)." - } -} - -variable "pypi_packages" { - type = map(string) - description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." - default = {} -} - -variable "python_version" { - description = "The default version of Python used to run the Airflow scheduler, worker, and webserver processes." - type = string - default = "3" -} - -variable "web_server_allowed_ip_ranges" { - description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions will be applied." - default = null - type = list(object({ - value = string - description = string - })) -} - -variable "github_remote_uri" { - description = "Url of your github repo" - type = string -} - -variable "github_name_prefix" { - description = "A name for your github connection to cloubuild" - type = string - default = "github-modules" -} - -variable "github_app_installation_id" { - description = "The app installation ID that was created when installing Google Cloud Build in Github: https://github.com/apps/google-cloud-build" - type = number - -} - -variable "service_account_prefix" { - description = "Name prefix to use for service accounts." - type = string - default = "sa" -} - -variable "project_id" { - description = "Optional project ID where Cloud Composer Environment is created." - type = string - default = null -} - -variable "github_secret_name" { - description = "Name of the github secret to extract github token info" - type = string - default = "github-api-token" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/vpc.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/vpc.tf deleted file mode 100644 index db633f28..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/composer/vpc.tf +++ /dev/null @@ -1,288 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 8.1" - - project_id = data.google_project.project.project_id - network_name = local.network_name - routing_mode = "REGIONAL" - - subnets = [ - { - subnet_name = "composer-primary-use4" - subnet_ip = local.composer_node_use4 - subnet_region = "us-east4" - subnet_private_access = true - subnet_flow_logs = "true" - }, - { - subnet_name = "composer-primary-usc1" - subnet_ip = local.composer_node_usc1 - subnet_region = "us-central1" - subnet_private_access = true - subnet_flow_logs = "true" - } - ] - - secondary_ranges = { - composer-primary-use4 = [ - { - range_name = "pods-primary-use4" - ip_cidr_range = local.pods_use4 - }, - { - range_name = "composer-services-primary-use4" - ip_cidr_range = local.services_use4 - }, - ] - - composer-primary-usc1 = [ - { - range_name = "pods-primary-usc1" - ip_cidr_range = local.pods_usc1 - }, - { - range_name = "composer-services-primary-usc1" - ip_cidr_range = local.services_usc1 - } - ] - } - - ingress_rules = [ - { - name = "allow-internal" - description = "Allow internal traffic within the VPC" - source_tags = ["internal"] - destination_ranges = ["10.0.0.0/8"] - allow = [ - { - protocol = "tcp" - ports = ["0-65535"] - }, - { - protocol = "udp" - ports = ["0-65535"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "iap-all-to-all" - description = "Allow support for IAP connections via google source ranges" - source_ranges = data.google_netblock_ip_ranges.iap_forwarders.cidr_blocks_ipv4 - allow = [ - { - protocol = "tcp" - ports = ["22"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "health-check-google-to-all" - description = "Allow support for Health Check connections via google source ranges" - source_ranges = concat(data.google_netblock_ip_ranges.health_checkers.cidr_blocks_ipv4, data.google_netblock_ip_ranges.legacy_health_checkers.cidr_blocks_ipv4) - allow = [ - { - protocol = "tcp" - ports = ["80", "443"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - } - ] - - egress_rules = [ - { - name = "egress-health-check-composer-to-google" - description = "Allow egress for Health Check connections from composer clusters" - target_tags = ["composer-use4", "composer-usc1"] - destination_ranges = concat(data.google_netblock_ip_ranges.health_checkers.cidr_blocks_ipv4, data.google_netblock_ip_ranges.legacy_health_checkers.cidr_blocks_ipv4) - allow = [ - { - protocol = "tcp" - ports = ["80", "443"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - - }, - { - name = "composer-to-dns" - description = "Composer DNS access" - destination_ranges = [local.composer_node_usc1, local.composer_node_use4] - allow = [ - { - protocol = "tcp" - ports = ["53"] - }, - { - protocol = "udp" - ports = ["53"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-use4-node-to-node" - description = "Composer node to node all comms in USE4" - target_tags = ["composer-use4"] - destination_ranges = [local.composer_node_use4] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-usc1-node-to-node" - description = "Composer node to node all comms in USC1" - target_tags = ["composer-usc1"] - destination_ranges = [local.composer_node_usc1] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-use4-node-to-master" - description = "Composer node to master all comms in USE4" - target_tags = ["composer-use4"] - destination_ranges = [local.composer_master_use4] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-usc1-node-to-master" - description = "Composer node to master all comms in USC1" - target_tags = ["composer-usc1"] - destination_ranges = [local.composer_master_usc1] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-use4-to-webserver" - description = "Composer Nodes to Web Server in USE4" - target_tags = ["composer-use4"] - destination_ranges = [local.composer_webserver_use4] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-usc1-to-webserver" - description = "Composer Nodes to Web Server in USC1" - target_tags = ["composer-usc1"] - destination_ranges = [local.composer_webserver_usc1] - allow = [ - { - protocol = "all" - ports = [] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "all-to-googleapi" - description = "Access for all resources in isolated VPC to Google APIs" - destination_ranges = ["199.36.153.8/30", "199.36.153.4/30"] - allow = [ - { - protocol = "tcp" - ports = ["443"] - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "composer-v2-internal-egress" - description = "Access for all resources in isolated VPC to Google APIs" - target_tags = ["composer-usc1", "composer-use4"] - destination_ranges = [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16" - ] - allow = [ - { - protocol = "all" - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - }, - { - name = "deny-all-egress" - description = "Default deny egress" - target_tags = ["composer-usc1", "composer-use4"] - destination_ranges = ["0.0.0.0/0"] - allow = [ - { - protocol = "all" - } - ] - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } - } - ] -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/README.md deleted file mode 100644 index 08ca1098..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/README.md +++ /dev/null @@ -1,70 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_vertex_ai_metadata_store.store](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_vertex_ai_metadata_store) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [name](#input\_name) | The name of the metadata store instance | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [vertex\_ai\_metadata\_store](#output\_vertex\_ai\_metadata\_store) | Vertex AI Metadata Store. | - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| name | The name of the metadata store instance | `string` | `null` | no | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| vertex\_ai\_metadata\_store | Vertex AI Metadata Store. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/data.tf deleted file mode 100644 index 035a39b9..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/data.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/locals.tf deleted file mode 100644 index 9a1fcf38..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/locals.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - keyring_name = "sample-keyring" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/main.tf deleted file mode 100644 index e9a77821..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_vertex_ai_metadata_store" "store" { - name = var.name - provider = google-beta - description = "Vertex Metadata store" - region = var.region - project = data.google_project.project.project_id - encryption_spec { - kms_key_name = data.google_kms_crypto_key.key.id - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/outputs.tf deleted file mode 100644 index 6c9cd94f..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "vertex_ai_metadata_store" { - description = "Vertex AI Metadata Store." - value = google_vertex_ai_metadata_store.store -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/variables.tf deleted file mode 100644 index 403910f3..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/metadata/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - type = string - description = "The name of the metadata store instance" - default = null -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/README.md deleted file mode 100644 index 4c445f0d..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/README.md +++ /dev/null @@ -1,137 +0,0 @@ -## Prerequisites - -#### IAM Permissions - -| Service Account | Scope | Role | -|-----------------|-------|------| -| PROJECT_NUMBER@cloudbuild.gserviceaccount.com | Project | Browser | -| | Project | Service Usage Consumer | -| | Project | Notebooks Admin | -| | Project | Compute Network Admin | -| | Project | Compute Security Admin | - -#### Organizational policies - -| Policy constraint | Scope | Value | -|-------------------|-------|-------| -| constraints/ainotebooks.requireAutoUpgradeSchedule | Project | Google-managed | -| constraints/ainotebooks.environmentOptions | Project | Google-managed | - - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | 5.14.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [google_compute_network.shared_vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_netblock_ip_ranges.health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.iap_forwarders](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.legacy_health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_projects.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_type](#input\_accelerator\_type) | The type of accelerator to use | `string` | `"NVIDIA_TESLA_K80"` | no | -| [boot\_disk\_size\_gb](#input\_boot\_disk\_size\_gb) | (Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"100"` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Possible disk types for notebook instances | `string` | `"PD_SSD"` | no | -| [boundry\_code](#input\_boundry\_code) | The boundry code for the tenant | `string` | `"001"` | no | -| [core\_count](#input\_core\_count) | number of accelerators to use | `number` | `1` | no | -| [data\_disk\_size\_gb](#input\_data\_disk\_size\_gb) | (Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"100"` | no | -| [data\_disk\_type](#input\_data\_disk\_type) | Optional. Input only. Indicates the type of the disk. Possible values are: PD\_STANDARD, PD\_SSD, PD\_BALANCED, PD\_EXTREME. | `string` | `"PD_SSD"` | no | -| [disable\_proxy\_access](#input\_disable\_proxy\_access) | (Optional) The notebook instance will not register with the proxy | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | Use this VM image family to find the image; the newest image in this family will be used. | `string` | `"workbench-instances"` | no | -| [image\_name](#input\_image\_name) | Use VM image name to find the image. | `string` | `""` | no | -| [image\_project](#input\_image\_project) | The name of the Google Cloud project that this VM image belongs to. Format: projects/{project\_id} | `string` | `"cloud-notebooks-managed"` | no | -| [install\_gpu\_driver](#input\_install\_gpu\_driver) | Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs. | `bool` | `false` | no | -| [instance\_owners](#input\_instance\_owners) | email of the owner of the instance, e.g. alias@example.com. Only one owner is supported! | `set(string)` | n/a | yes | -| [location](#input\_location) | Notebook instance location (zone). | `string` | `"us-central1-a"` | no | -| [machine\_type](#input\_machine\_type) | type of the machine to spin up for the notebook | `string` | `"e2-standard-4"` | no | -| [name](#input\_name) | name of the notebook instance | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [tags](#input\_tags) | The Compute Engine tags to add to instance. | `list(string)` | 
[
  "egress-internet"
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [create\_time](#output\_create\_time) | Instance creation time | -| [effective\_labels](#output\_effective\_labels) | All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services. | -| [id](#output\_id) | an identifier for the resource with format projects/{{project}}/locations/{{location}}/instances/{{name}} | -| [proxy\_uri](#output\_proxy\_uri) | The proxy endpoint that is used to access the Jupyter notebook. Only returned when the resource is in a PROVISIONED state. If needed you can utilize terraform apply -refresh-only to await the population of this value. | -| [state](#output\_state) | The state of this instance. | -| [terraform\_labels](#output\_terraform\_labels) | The combination of labels configured directly on the resource and default labels configured on the provider. | -| [update\_time](#output\_update\_time) | Instance update time. | - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| accelerator\_type | The type of accelerator to use | `string` | `"NVIDIA_TESLA_K80"` | no | -| boot\_disk\_size\_gb | (Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"150"` | no | -| boot\_disk\_type | Possible disk types for notebook instances | `string` | `"PD_SSD"` | no | -| boundry\_code | The boundry code for the tenant | `string` | `"001"` | no | -| core\_count | number of accelerators to use | `number` | `1` | no | -| data\_disk\_size\_gb | (Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"150"` | no | -| data\_disk\_type | Optional. Input only. Indicates the type of the disk. Possible values are: PD\_STANDARD, PD\_SSD, PD\_BALANCED, PD\_EXTREME. | `string` | `"PD_SSD"` | no | -| disable\_proxy\_access | (Optional) The notebook instance will not register with the proxy | `bool` | `false` | no | -| image\_family | Use this VM image family to find the image; the newest image in this family will be used. | `string` | `"workbench-instances"` | no | -| image\_name | Use VM image name to find the image. | `string` | `""` | no | -| image\_project | The name of the Google Cloud project that this VM image belongs to. Format: projects/{project\_id} | `string` | `"cloud-notebooks-managed"` | no | -| install\_gpu\_driver | Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs. | `bool` | `false` | no | -| instance\_owners | email of the owner of the instance, e.g. alias@example.com. Only one owner is supported! | `set(string)` | n/a | yes | -| location | Notebook instance location (zone). | `string` | `"us-central1-a"` | no | -| machine\_type | type of the machine to spin up for the notebook | `string` | `"e2-standard-4"` | no | -| name | name of the notebook instance | `string` | n/a | yes | -| project\_id | Optional Project ID. | `string` | `null` | no | -| tags | The Compute Engine tags to add to instance. | `list(string)` | 
[
  "egress-internet"
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| create\_time | Instance creation time | -| effective\_labels | All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services. | -| id | an identifier for the resource with format projects/{{project}}/locations/{{location}}/instances/{{name}} | -| proxy\_uri | The proxy endpoint that is used to access the Jupyter notebook. Only returned when the resource is in a PROVISIONED state. If needed you can utilize terraform apply -refresh-only to await the population of this value. | -| state | The state of this instance. | -| terraform\_labels | The combination of labels configured directly on the resource and default labels configured on the provider. | -| update\_time | Instance update time. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/data.tf deleted file mode 100644 index 64654ba2..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/data.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_projects" "vpc" { - filter = "labels.application_name:restricted-shared-vpc-host labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" - # filter = "labels.application_name:base-shared-vpc-host labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_compute_network" "shared_vpc" { - name = "vpc-${data.google_project.project.labels.env_code}-shared-restricted" - # name = "vpc-${data.google_project.project.labels.env_code}-shared-base" - project = data.google_projects.vpc.projects.0.project_id -} - -data "google_compute_subnetwork" "subnet" { - name = "sb-${data.google_project.project.labels.env_code}-shared-restricted-${local.region}" - # name = "sb-${data.google_project.project.labels.env_code}-shared-base-${local.region}" - project = data.google_projects.vpc.projects.0.project_id - region = local.region -} - -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = local.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_netblock_ip_ranges" "legacy_health_checkers" { - range_type = "legacy-health-checkers" -} - -data "google_netblock_ip_ranges" "health_checkers" { - range_type = "health-checkers" -} - -// Cloud IAP's TCP forwarding netblock -data "google_netblock_ip_ranges" "iap_forwarders" { - range_type = "iap-forwarders" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/locals.tf deleted file mode 100644 index cdfed661..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/locals.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - name_var = format("%s-%s", data.google_project.project.labels.env_code, var.name) - region = substr(var.location, 0, length(var.location) - 2) -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/main.tf deleted file mode 100644 index 6686081e..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/main.tf +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_workbench_instance" "instance" { - name = var.name - location = var.location - - gce_setup { - machine_type = var.machine_type - - dynamic "accelerator_configs" { - for_each = var.install_gpu_driver == true ? [1] : [] - content { - type = var.accelerator_type - core_count = var.core_count - } - } - disable_public_ip = true - - - dynamic "vm_image" { - for_each = var.image_family != "" ? [1] : [] - content { - project = var.image_project - family = var.image_family - } - } - - dynamic "vm_image" { - for_each = var.image_name != "" ? [1] : [] - content { - project = var.image_project - name = var.image_name - } - } - - boot_disk { - disk_type = var.boot_disk_type - disk_size_gb = var.boot_disk_size_gb - disk_encryption = "CMEK" - kms_key = data.google_kms_crypto_key.key.id - } - - data_disks { - disk_size_gb = var.data_disk_size_gb - disk_type = var.data_disk_type - disk_encryption = "CMEK" - kms_key = data.google_kms_crypto_key.key.id - } - - enable_ip_forwarding = false - - tags = var.tags - - network_interfaces { - network = data.google_compute_network.shared_vpc.id - subnet = data.google_compute_subnetwork.subnet.id - nic_type = "GVNIC" - } - - metadata = { - notebook-disable-downloads = "true" - notebook-disable-root = "true" - notebook-disable-terminal = "true" - notebook-upgrade-schedule = "00 19 * * MON" - # disable-mixer = "${var.dataproc_kernel_access ? false : true}" - disable-mixer = "false" // Enable access to Dataproc kernels - report-dns-resolution = "true" - report-event-health = "true" - terraform = "true" - } - } - - instance_owners = var.instance_owners - - disable_proxy_access = var.disable_proxy_access - - labels = { - environment = data.google_project.project.labels.environment - boundry_code = var.boundry_code - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/outputs.tf deleted file mode 100644 index 77ab019a..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "id" { - description = "an identifier for the resource with format projects/{{project}}/locations/{{location}}/instances/{{name}}" - value = google_workbench_instance.instance.id -} - -output "proxy_uri" { - description = "The proxy endpoint that is used to access the Jupyter notebook. Only returned when the resource is in a PROVISIONED state. If needed you can utilize terraform apply -refresh-only to await the population of this value." - value = google_workbench_instance.instance.proxy_uri -} - -output "state" { - description = "The state of this instance." - value = google_workbench_instance.instance.state -} - -output "create_time" { - description = "Instance creation time" - value = google_workbench_instance.instance.create_time -} - -output "update_time" { - description = "Instance update time." - value = google_workbench_instance.instance.update_time -} - -output "terraform_labels" { - description = "The combination of labels configured directly on the resource and default labels configured on the provider." - value = google_workbench_instance.instance.terraform_labels -} - -output "effective_labels" { - description = "All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services." - value = google_workbench_instance.instance.effective_labels -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/variables.tf deleted file mode 100644 index ab4c2c28..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/notebook/variables.tf +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - description = "name of the notebook instance" - type = string -} - -variable "location" { - type = string - description = "Notebook instance location (zone)." - default = "us-central1-a" - validation { - condition = contains(["us-central1", "us-east4"], substr(var.location, 0, length(var.location) - 2)) - error_message = "Location must be one within of [us-central1, us-east4] regions." - } -} - -variable "machine_type" { - description = "type of the machine to spin up for the notebook" - type = string - default = "e2-standard-4" -} - -variable "instance_owners" { - description = "email of the owner of the instance, e.g. alias@example.com. Only one owner is supported!" - type = set(string) -} - - -variable "accelerator_type" { - description = "The type of accelerator to use" - type = string - default = "NVIDIA_TESLA_K80" - validation { - condition = contains(["ACCELERATOR_TYPE_UNSPECIFIED", "NVIDIA_TESLA_K80", - "NVIDIA_TESLA_P100", "NVIDIA_TESLA_V100", "NVIDIA_TESLA_P4", - "NVIDIA_TESLA_T4", "NVIDIA_TESLA_T4_VWS", "NVIDIA_TESLA_P100_VWS", - "NVIDIA_TESLA_P4_VWS", "NVIDIA_TESLA_A100", "TPU_V2", "TPU_V3"], var.accelerator_type) - error_message = "Accelerator type can be one of the following: " - } -} -variable "core_count" { - type = number - default = 1 - description = "number of accelerators to use" -} - -variable "image_project" { - description = "The name of the Google Cloud project that this VM image belongs to. Format: projects/{project_id}" - type = string - # default = "deeplearning-platform-release" - default = "cloud-notebooks-managed" -} - -variable "image_family" { - description = "Use this VM image family to find the image; the newest image in this family will be used." - type = string - # default = "common-cpu-notebooks" - default = "workbench-instances" -} - -variable "image_name" { - description = "Use VM image name to find the image." - type = string - default = "" -} - -variable "install_gpu_driver" { - description = "Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs." - type = bool - default = false -} - -variable "boot_disk_type" { - description = "Possible disk types for notebook instances" - type = string - default = "PD_SSD" - validation { - condition = contains(["DISK_TYPE_UNSPECIFIED", "PD_STANDARD", "PD_SSD", "PD_BALANCED", "PD_EXTREME"], var.boot_disk_type) - error_message = "Illegal value for boot disk type" - } -} - -variable "boot_disk_size_gb" { - description = "(Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)" - type = string - default = "150" -} - -variable "data_disk_type" { - description = "Optional. Input only. Indicates the type of the disk. Possible values are: PD_STANDARD, PD_SSD, PD_BALANCED, PD_EXTREME." - type = string - default = "PD_SSD" - validation { - condition = contains(["PD_STANDARD", "PD_SSD", "PD_BALANCED", "PD_EXTREME"], var.data_disk_type) - error_message = "Illegal value for data disk type" - } -} - -variable "data_disk_size_gb" { - description = "(Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)" - type = string - default = "150" -} - - -variable "disable_proxy_access" { - description = "(Optional) The notebook instance will not register with the proxy" - type = bool - default = false -} - -# variable "dataproc_kernel_access" { -# description = "(Optional) Enables access to Dataproc kernels." -# type = bool -# default = true -# } - -variable "boundry_code" { - description = "The boundry code for the tenant" - type = string - default = "001" -} - - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} - -variable "tags" { - type = list(string) - description = "The Compute Engine tags to add to instance." - default = ["egress-internet"] -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/README.md deleted file mode 100644 index 1aa923b9..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/README.md +++ /dev/null @@ -1,82 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | -| [google-beta](#provider\_google-beta) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_pubsub_topic.pubsub_topic](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_pubsub_topic) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [locked\_regions](#input\_locked\_regions) | Regions that pubsub presistence is locked to | `list(any)` | 
[
  "us-central1",
  "us-east4"
]
| no | -| [message\_retention\_duration](#input\_message\_retention\_duration) | Message retention duration. | `string` | `"86400s"` | no | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| [topic\_name](#input\_topic\_name) | Topic name | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [pubsub\_topic](#output\_pubsub\_topic) | Pub/Sub Topic. | - - -## Security Controls - -The following table outlines which of the suggested controls for Vertex Generative AI are enabled in this module. -| Name | Control ID | NIST 800-53 | CRI Profile | Category | Source Blueprint -|------|------------|-------------|-------------|----------| ----------------| -|Customer Managed Encryption Keys for Pub/Sub Messages| PS-CO-6.1| SC-12
SC-13| PR.DS-1.1
PR.DS-1.2
PR.DS-2.1
PR.DS-2.2
PR.DS-5.1 | Recommended | Secure Foundation v4 -|Configure Message Storage Policies | PS-CO-4.1 | AC-3
AC-17
AC-20
| PR.AC-3.1
PR.AC-3.2
PR.AC-4.1
PR.AC-4.2
PR.AC-4.3
PR.AC-6.1
PR.PT-3.1
PR.PT-4.1 | Optional | ML Foundation v0.1.0-alpha.1 - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| locked\_regions | Regions that pubsub presistence is locked to | `list(any)` | 
[
  "us-central1",
  "us-east4"
]
| no | -| message\_retention\_duration | Message retention duration. | `string` | `"86400s"` | no | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| topic\_name | Topic name | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| pubsub\_topic | Pub/Sub Topic. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/data.tf deleted file mode 100644 index 5ec06b5d..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/data.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/locals.tf deleted file mode 100644 index f92bce3f..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/locals.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - keyring_name = "sample-keyring" -} - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/main.tf deleted file mode 100644 index 329f0ca4..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/main.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_pubsub_topic" "pubsub_topic" { - provider = google-beta - - project = data.google_project.project.project_id - name = var.topic_name - message_retention_duration = var.message_retention_duration - - #Customer Managed Encryption Keys - #Control ID: PS-CO-6.1 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - - kms_key_name = data.google_kms_crypto_key.key.id - - #Configure Message Storage Policies - #Control ID: PS-CO-4.1 - #NIST 800-53: AC-3 AC-17 AC-20 - #CRI Profile: PR.AC-3.1 PR.AC-3.2 PR.AC-4.1 PR.AC-4.2 PR.AC-4.3 PR.AC-6.1 PR.PT-3.1 PR.PT-4.1 - - message_storage_policy { - allowed_persistence_regions = var.locked_regions - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/outputs.tf deleted file mode 100644 index 5606922c..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "pubsub_topic" { - description = "Pub/Sub Topic." - value = google_pubsub_topic.pubsub_topic -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/variables.tf deleted file mode 100644 index 8101d407..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/pubsub/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} - -variable "message_retention_duration" { - description = "Message retention duration." - default = "86400s" - type = string -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "topic_name" { - description = "Topic name" - type = string -} - -variable "locked_regions" { - description = "Regions that pubsub presistence is locked to" - type = list(any) - default = [ - "us-central1", - "us-east4" - ] -} - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/README.md deleted file mode 100644 index d003b9b8..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/README.md +++ /dev/null @@ -1,79 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_secret_manager_secret.secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_pubsub_topic.secret_rotations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/pubsub_topic) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| [secret\_names](#input\_secret\_names) | Names of the secrets to be created. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [secret\_manager](#output\_secret\_manager) | Secret Manager resource. | - - -## Security Controls - -The following table outlines which of the suggested controls for Vertex Generative AI are enabled in this module. -| Name | Control ID | NIST 800-53 | CRI Profile | Category | Source Blueprint -|------|------------|-------------|-------------|----------| ----------------| -|Customer Managed Encryption Keys| COM-CO-2.3| SC-12
SC-13 | PR.DS-1.1
PR.DS-2.1
PR.DS-2.2
PR.DS-5.1 | Recommended | Secure Foundation v4 -|Automatic Secret Replication| SM-CO-6.1| SC-12
SC-13| None | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 -|Set up Automatic Rotation of Secrets| SM-CO-6.2| SC-12
SC-13| None | Recommended based on customer use case | ML Foundation v0.1.0-alpha.1 - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | -| secret\_names | Names of the secrets to be created. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| secret\_manager | Secret Manager resource. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/data.tf deleted file mode 100644 index b243d811..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/data.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - -data "google_pubsub_topic" "secret_rotations" { - name = local.pubsub_topic_name - project = data.google_project.project.project_id -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/locals.tf deleted file mode 100644 index 5defd111..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/locals.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - keyring_name = "sample-keyring" - pubsub_topic_name = "secret-rotation-notifications" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/main.tf deleted file mode 100644 index c20104c1..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/main.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_secret_manager_secret" "secret" { - for_each = toset(var.secret_names) - secret_id = each.key - - project = data.google_project.project.project_id - - #Set up Automatic Rotation of Secrets - #Control ID: SM-CO-6.2 - #NIST 800-53: SC-12 SC-13 - - rotation { - next_rotation_time = formatdate("YYYY-MM-DD'T'hh:mm:ss'Z'", timeadd(timestamp(), "720h")) - rotation_period = "43200s" - } - - topics { - name = data.google_pubsub_topic.secret_rotations.id - } - - #Automatic Secret Replication - #Control ID: SM-CO-6.1 - #NIST 800-53: SC-12 SC-13 - - replication { - user_managed { - replicas { - location = data.google_kms_key_ring.kms.location - - #Customer Managed Encryption Keys - #Control ID: COM-CO-2.3 - #NIST 800-53: SC-12 SC-13 - #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1 - - customer_managed_encryption { - kms_key_name = data.google_kms_crypto_key.key.id - } - } - } - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/outputs.tf deleted file mode 100644 index 6ad27497..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "secret_manager" { - description = "Secret Manager resource." - value = google_secret_manager_secret.secret -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/variables.tf deleted file mode 100644 index 517c8158..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/secrets/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "secret_names" { - type = list(string) - description = "Names of the secrets to be created." -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/README.md b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/README.md deleted file mode 100644 index 9716b12b..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/README.md +++ /dev/null @@ -1,69 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_vertex_ai_tensorboard.tensorboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/vertex_ai_tensorboard) | resource | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [name](#input\_name) | The name of the metadata store instance | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [region](#input\_region) | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [vertex\_ai\_tensorboard](#output\_vertex\_ai\_tensorboard) | TensorBoard resource. | - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| name | The name of the metadata store instance | `string` | `null` | no | -| project\_id | Optional Project ID. | `string` | `null` | no | -| region | The resource region, one of [us-central1, us-east4]. | `string` | `"us-central1"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| vertex\_ai\_tensorboard | TensorBoard resource. | - - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/data.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/data.tf deleted file mode 100644 index 881d3bbe..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/data.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_kms_key_ring" "kms" { - name = local.keyring_name - location = var.region - project = data.google_projects.kms.projects.0.project_id -} - -data "google_kms_crypto_key" "key" { - name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id -} - diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/locals.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/locals.tf deleted file mode 100644 index 9a1fcf38..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/locals.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - keyring_name = "sample-keyring" -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/main.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/main.tf deleted file mode 100644 index 9b4ace46..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_vertex_ai_tensorboard" "tensorboard" { - display_name = var.name - description = "Vertex tensorboard instance with cmek" - labels = {} - region = var.region - project = data.google_project.project.project_id - encryption_spec { - kms_key_name = data.google_kms_crypto_key.key.id - } -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/outputs.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/outputs.tf deleted file mode 100644 index 36691c0d..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "vertex_ai_tensorboard" { - description = "TensorBoard resource." - value = google_vertex_ai_tensorboard.tensorboard -} diff --git a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/variables.tf b/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/variables.tf deleted file mode 100644 index 403910f3..00000000 --- a/docs/assets/terraform/5-appinfra/service-catalog-pipeline/modules/tensorboard/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "name" { - type = string - description = "The name of the metadata store instance" - default = null -} - -variable "region" { - type = string - description = "The resource region, one of [us-central1, us-east4]." - default = "us-central1" - validation { - condition = contains(["us-central1", "us-east4"], var.region) - error_message = "Region must be one of [us-central1, us-east4]." - } -} - -variable "project_id" { - type = string - description = "Optional Project ID." - default = null -} diff --git a/docs/deploy_on_foundation_v4.0.0.md b/docs/deploy_on_foundation_v4.0.0.md index fbca50b0..79bd8450 100644 --- a/docs/deploy_on_foundation_v4.0.0.md +++ b/docs/deploy_on_foundation_v4.0.0.md @@ -2250,9 +2250,9 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa ```bash cd ml-artifact-publish/ git checkout -b plan - - cp -RT ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/artifact-publish-infra-repo . - cp -R ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/modules . + + cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh @@ -2294,7 +2294,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa git push --set-upstream origin plan ``` -- Merge changes to shared. Because this is a [named environment branch](../docs/FAQ.md#what-is-a-named-branch), pushing to this branch triggers both _terraform plan_ and _terraform apply_. Review the apply output in your Cloud Build project `https://console.cloud.google.com/cloud-build/builds;region=DEFAULT_REGION?project=YOUR_INFRA_PIPELINE_PROJECT_ID`. +- Merge changes to shared. Because this is a [named environment branch](../docs/FAQ.md#what-is-a-named-branch), pushing to this branch triggers both _terraform plan_ and _terraform apply_. Review the apply output in your Cloud Build project `https://console.cloud.google.com/cloud-build/builds;region=DEFAULT_REGION?project=YOUR_INFRA_PIPELINE_PROJECT_ID`. Before proceeding further, make sure that the build applied successfully. ```bash git checkout -b production @@ -2331,7 +2331,7 @@ The series of steps below will trigger the custom artifacts pipeline. git checkout -b main git commit -m "Initialize Repository" --allow-empty - cp -RT ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/artifacts-pipeline . + cp -RT ../terraform-google-enterprise-genai/5-app-infra/source_repos/artifact-publish/ . ``` - Commit changes and push your main branch to the new repo. @@ -2418,8 +2418,8 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc cd ml-service-catalog git checkout -b plan - cp -RT ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/service-catalog-infra-repo . - cp -R ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/modules . + cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ . + cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* . cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh @@ -2469,7 +2469,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc git push --set-upstream origin plan ``` -- Merge changes to production. Because this is a [named environment branch](../docs/FAQ.md#what-is-a-named-branch), pushing to this branch triggers both _terraform plan_ and _terraform apply_. Review the apply output in your Cloud Build project `https://console.cloud.google.com/cloud-build/builds;region=DEFAULT_REGION?project=YOUR_INFRA_PIPELINE_PROJECT_ID`. +- Merge changes to production. Because this is a [named environment branch](../docs/FAQ.md#what-is-a-named-branch), pushing to this branch triggers both _terraform plan_ and _terraform apply_. Review the apply output in your Cloud Build project `https://console.cloud.google.com/cloud-build/builds;region=DEFAULT_REGION?project=YOUR_INFRA_PIPELINE_PROJECT_ID`. Before proceeding further, make sure that the build applied successfully. ```bash git checkout -b production @@ -2503,7 +2503,7 @@ The series of steps below will trigger the custom Service Catalog Pipeline. ```bash cd service-catalog/ - cp -RT ../terraform-google-enterprise-genai/docs/assets/terraform/5-appinfra/service-catalog-pipeline . + cp -RT ../terraform-google-enterprise-genai/5-app-infra/source_repos/service-catalog/ . git add img git commit -m "Add img directory" ```