updating docs

.github/workflows/lint.yaml

@@ -48,10 +48,10 @@ jobs:
         env:
           DISABLE_TFLINT: 1
           ENABLE_PARALLEL: 0
-          EXCLUDE_LINT_DIRS: \./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning
+          EXCLUDE_LINT_DIRS: \./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning|\./docs/assets/terraform
 
       - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh
         env:
           DISABLE_TFLINT: 1
           ENABLE_PARALLEL: 0
-          EXCLUDE_LINT_DIRS: \./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning
+          EXCLUDE_LINT_DIRS: \./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning|\./docs/assets/terraform

Makefile

@@ -28,7 +28,7 @@ docker_test_lint:
 	docker run --rm -it \
 		-e ENABLE_PARALLEL=0 \
 		-e DISABLE_TFLINT=1 \
-		-e EXCLUDE_LINT_DIRS="\./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning" \
+		-e EXCLUDE_LINT_DIRS="\./5-app-infra/6-machine-learning|\./5-app-infra/projects/machine-learning|\./docs/assets/terraform" \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_lint.sh

docs/assets/terraform/1-org/README.md

@@ -0,0 +1 @@
+# Terraform code in this directory is used on docs as reference/assets.

docs/assets/terraform/1-org/ml_key_rings.tf

@@ -0,0 +1,10 @@
+module "kms_keyring" {
+  source = "../../modules/ml_kms_keyring"
+
+  keyring_admins = [
+    "serviceAccount:${local.projects_step_terraform_service_account_email}"
+  ]
+  project_id      = module.org_kms.project_id
+  keyring_regions = var.keyring_regions
+  keyring_name    = var.keyring_name
+}

docs/assets/terraform/1-org/ml_ops_org_policy.tf

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "ml_organization_policies" {
+  source = "../../modules/ml-org-policies"
+
+  org_id    = local.organization_id
+  folder_id = local.folder_id
+
+  allowed_locations = [
+    "in:us-locations"
+  ]
+
+  allowed_vertex_vpc_networks = {
+    parent_type = "project"
+    ids         = [for instance in module.base_restricted_environment_network : instance.restricted_shared_vpc_project_id],
+  }
+
+  allowed_vertex_images = [
+    "ainotebooks-vm/deeplearning-platform-release/image-family/pytorch-1-13-cu113-notebooks",
+    "ainotebooks-vm/deeplearning-platform-release/image-family/pytorch-1-13-cu113-notebooks",
+    "ainotebooks-vm/deeplearning-platform-release/image-family/common-cu113-notebooks",
+    "ainotebooks-vm/deeplearning-platform-release/image-family/common-cpu-notebooks",
+    "ainotebooks-container/us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu113.py310",
+    "ainotebooks-container/us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu113.py37",
+    "ainotebooks-container/us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu110.py310",
+    "ainotebooks-container/us-docker.pkg.dev/deeplearning-platform-release/gcr.io/tf2-cpu.2-12.py310",
+    "ainotebooks-container/us-docker.pkg.dev/deeplearning-platform-release/gcr.io/tf2-gpu.2-12.py310"
+  ]
+
+  restricted_services = [
+    "alloydb.googleapis.com"
+  ]
+
+  allowed_integrations = [
+    "github.com",
+    "source.developers.google.com"
+  ]
+
+  restricted_tls_versions = [
+    "TLS_VERSION_1",
+    "TLS_VERSION_1_1"
+  ]
+
+  restricted_non_cmek_services = [
+    "bigquery.googleapis.com",
+    "aiplatform.googleapis.com"
+  ]
+
+  allowed_vertex_access_modes = [
+    "single-user",
+    "service-account"
+  ]
+}

docs/assets/terraform/2-environments/README.md

@@ -0,0 +1 @@
+# Terraform code in this directory is used on docs as reference/assets.

docs/assets/terraform/2-environments/ml_key_rings.tf

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  logging_key_name = module.env_logs.project_id
+}
+
+// Creates a keyring with logging key for each region (us-central1, us-east4)
+module "kms_keyring" {
+  source = "../ml_kms_keyring"
+
+  keyring_admins = [
+    "serviceAccount:${local.projects_step_terraform_service_account_email}"
+  ]
+  project_id          = module.env_kms.project_id
+  keyring_regions     = var.keyring_regions
+  keyring_name        = var.keyring_name
+  keys                = [local.logging_key_name]
+  kms_prevent_destroy = var.kms_prevent_destroy
+}

docs/assets/terraform/2-environments/ml_logging.tf

@@ -0,0 +1,87 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+data "google_storage_project_service_account" "gcs_logging_account" {
+  project = module.env_logs.project_id
+}
+
+/******************************************
+  Project for Environment Logging
+*****************************************/
+
+module "env_logs" {
+  source  = "terraform-google-modules/project-factory/google"
+  version = "~> 14.0"
+
+  random_project_id        = true
+  random_project_id_length = 4
+  default_service_account  = "deprivilege"
+  name                     = "${local.project_prefix}-${var.environment_code}-logging"
+  org_id                   = local.org_id
+  billing_account          = local.billing_account
+  folder_id                = google_folder.env.id
+  activate_apis            = ["logging.googleapis.com", "billingbudgets.googleapis.com", "storage.googleapis.com"]
+
+  labels = {
+    environment       = var.env
+    application_name  = "env-logging"
+    billing_code      = "1234"
+    primary_contact   = "example1"
+    secondary_contact = "example2"
+    business_code     = "abcd"
+    env_code          = var.environment_code
+  }
+  budget_alert_pubsub_topic   = var.project_budget.logging_alert_pubsub_topic
+  budget_alert_spent_percents = var.project_budget.logging_alert_spent_percents
+  budget_amount               = var.project_budget.logging_budget_amount
+  budget_alert_spend_basis    = var.project_budget.logging_budget_alert_spend_basis
+
+}
+
+// Create Bucket for this project
+resource "google_storage_bucket" "log_bucket" {
+  name                        = "${var.gcs_bucket_prefix}-${module.env_logs.project_id}"
+  location                    = var.gcs_logging_bucket_location
+  project                     = module.env_logs.project_id
+  uniform_bucket_level_access = true
+
+  dynamic "retention_policy" {
+    for_each = var.gcs_logging_retention_period != null ? [var.gcs_logging_retention_period] : []
+    content {
+      is_locked        = var.gcs_logging_retention_period.is_locked
+      retention_period = var.gcs_logging_retention_period.retention_period_days * 24 * 60 * 60
+    }
+  }
+
+  encryption {
+    default_kms_key_name = google_kms_crypto_key_iam_member.gcs_logging_key.crypto_key_id #module.kms_keyring.keys_by_region[var.gcs_logging_bucket_location][local.logging_key_name]
+  }
+}
+
+/******************************************
+  Logging Bucket - IAM
+*****************************************/
+# resource "google_storage_bucket_iam_member" "bucket_logging" {
+#   bucket = google_storage_bucket.log_bucket.name
+#   role   = "roles/storage.objectCreator"
+#   member = "group:[email protected]"
+# }
+
+resource "google_kms_crypto_key_iam_member" "gcs_logging_key" {
+  crypto_key_id = module.kms_keyring.keys_by_region[var.gcs_logging_bucket_location][local.logging_key_name]
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member        = "serviceAccount:${data.google_storage_project_service_account.gcs_logging_account.email_address}"
+}

docs/assets/terraform/3-networks-dual-svpc/ml_dns_notebooks.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+module "ml_dns_vertex_ai" {
+  source = "../ml_dns_notebooks"
+
+  project_id                         = local.restricted_project_id
+  private_service_connect_ip         = var.restricted_private_service_connect_ip
+  private_visibility_config_networks = [module.restricted_shared_vpc.network_self_link]
+  zone_names = {
+    kernels_googleusercontent_zone   = "dz-${var.environment_code}-shared-restricted-kernels-googleusercontent"
+    notebooks_googleusercontent_zone = "dz-${var.environment_code}-shared-restricted-notebooks-googleusercontent"
+    notebooks_cloudgoogle_zone       = "dz-${var.environment_code}-shared-restricted-notebooks"
+  }
+}

docs/assets/terraform/4-projects/ml_business_unit/development/README.md

@@ -0,0 +1,25 @@
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| default\_region | Default region to create resources where applicable. | `string` | `"us-central1"` | no |
+| env | The environment this deployment belongs to (ie. development) | `string` | n/a | yes |
+| key\_rotation\_period | Rotation period in seconds to be used for KMS Key | `string` | `"7776000s"` | no |
+| location\_gcs | Case-Sensitive Location for GCS Bucket (Should be same region as the KMS Keyring) | `string` | `"US"` | no |
+| location\_kms | Case-Sensitive Location for KMS Keyring (Should be same region as the GCS Bucket) | `string` | `"us"` | no |
+| peering\_module\_depends\_on | List of modules or resources peering module depends on. | `list(any)` | `[]` | no |
+| project\_budget | Budget configuration.<br>  budget\_amount: The amount to use as the budget.<br>  alert\_spent\_percents: A list of percentages of the budget to alert on when threshold is exceeded.<br>  alert\_pubsub\_topic: The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}`.<br>  alert\_spend\_basis: The type of basis used to determine if spend has passed the threshold. Possible choices are `CURRENT_SPEND` or `FORECASTED_SPEND` (default). | <pre>object({<br>    budget_amount        = optional(number, 1000)<br>    alert_spent_percents = optional(list(number), [1.2])<br>    alert_pubsub_topic   = optional(string, null)<br>    alert_spend_basis    = optional(string, "FORECASTED_SPEND")<br>  })</pre> | `{}` | no |
+| remote\_state\_bucket | Backend bucket to load Terraform Remote State Data from previous steps. | `string` | n/a | yes |
+| tfc\_org\_name | Name of the TFC organization. | `string` | `""` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| enable\_cloudbuild\_deploy | Enable infra deployment using Cloud Build. |
+| machine\_learning\_kms\_keys | Key ID for the machine learning project. |
+| machine\_learning\_project\_id | Project machine learning project. |
+| machine\_learning\_project\_number | Project number of machine learning project. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

docs/assets/terraform/4-projects/ml_business_unit/development/backend.tf

@@ -0,0 +1,22 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  backend "gcs" {
+    bucket = "UPDATE_PROJECTS_BACKEND"
+    prefix = "terraform/projects/ml_business_unit/development"
+  }
+}

docs/assets/terraform/4-projects/ml_business_unit/development/backend.tf.cloud.example

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  cloud {
+    workspaces {
+      name = "4-ml-development"
+    }
+  }
+}

docs/assets/terraform/4-projects/ml_business_unit/development/common.auto.tfvars

@@ -0,0 +1 @@
+../../common.auto.tfvars

docs/assets/terraform/4-projects/ml_business_unit/development/development.auto.tfvars

@@ -0,0 +1 @@
+../../development.auto.tfvars

policy-library/storage_logging.yaml → docs/assets/terraform/4-projects/ml_business_unit/development/locals.tf

@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,16 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-apiVersion: constraints.gatekeeper.sh/v1alpha1
-kind: GCPStorageLoggingConstraintV1
-metadata:
-  name: storage_logging
-  annotations:
-    description: Ensure storage logs are delivered to a separate bucket
-spec:
-  severity: high
-  match:
-    ancestries:
-    - "organizations/**"
-    excludedAncestries: [] # optional, default is no exclusions
-  parameters: {}
+locals {
+  repo_name     = "ml-composer"
+  business_code = "ml"
+  business_unit = "ml_business_unit"
+}