feat: Infrastructure Manager workspace blueprint (#271)

build/int.cloudbuild.yaml

@@ -26,16 +26,19 @@ steps:
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
   - 'TF_VAR_group_org_admins=test-gcp-org-admins@test.blueprints.joonix.net'
   - 'TF_VAR_group_billing_admins=test-gcp-billing-admins@test.blueprints.joonix.net'
+  secretEnv: ['IM_GITHUB_PAT']
 - id: init-all
   waitFor:
     - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
 - id: create-all
   waitFor:
     - init-all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
+  secretEnv: ['IM_GITHUB_PAT']
 - id: converge-simple
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-default']
@@ -132,6 +135,29 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage teardown --verbose']
 
+- id: apply-imworkspace-github
+  waitFor:
+      - create-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestIMCloudBuildWorkspaceGitHub --stage apply --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: verify-imworkspace-github
+  waitFor:
+      - apply-imworkspace-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestIMCloudBuildWorkspaceGitHub --stage verify --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+- id: teardown-imworkspace-github
+  waitFor:
+      - verify-imworkspace-github
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestIMCloudBuildWorkspaceGitHub --stage teardown --verbose']
+  secretEnv: ['IM_GITHUB_PAT']
+
+availableSecrets:
+  secretManager:
+  - versionName: $_IM_GITHUB_PAT_SECRET_ID/versions/latest
+    env: 'IM_GITHUB_PAT'
 tags:
 - 'ci'
 - 'integration'

examples/im_cloudbuild_workspace_github/README.md

@@ -0,0 +1,25 @@
+## Overview
+
+This example demonstrates the simplest usage of the [im_cloudbuild_workspace](../../modules/im_cloudbuild_workspace/) module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| im\_github\_pat | GitHub personal access token. | `string` | n/a | yes |
+| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes |
+| repository\_url | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_apply\_trigger\_id | Trigger used for running IM apply |
+| cloudbuild\_preview\_trigger\_id | Trigger used for creating IM previews |
+| cloudbuild\_sa | Service account used by the Cloud Build triggers |
+| github\_secret\_id | The secret ID for the GitHub secret containing the personal access token. |
+| infra\_manager\_sa | Service account used by Infrastructure Manager |
+| project\_id | n/a |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/im_cloudbuild_workspace_github/apis.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "enabled_google_apis" {
+  source  = "terraform-google-modules/project-factory/google//modules/project_services"
+  version = "~> 14.0"
+
+  project_id                  = var.project_id
+  disable_services_on_destroy = false
+
+  activate_apis = [
+    "iam.googleapis.com",
+    "secretmanager.googleapis.com",
+    "compute.googleapis.com",
+    "cloudbuild.googleapis.com",
+    "config.googleapis.com",
+  ]
+}

examples/im_cloudbuild_workspace_github/main.tf

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "im_workspace" {
+  source = "../../modules/im_cloudbuild_workspace"
+
+  project_id    = var.project_id
+  deployment_id = "im-example-github-deployment"
+
+  tf_repo_type           = "GITHUB"
+  im_deployment_repo_uri = var.repository_url
+  im_deployment_ref      = "main"
+  im_tf_variables        = "project_id=${var.project_id}"
+  infra_manager_sa_roles = ["roles/compute.networkAdmin"]
+  tf_cloudbuilder        = "hashicorp/terraform:1.2.3"
+
+  // Found in the URL of your Cloud Build GitHub app configuration settings
+  // https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen#connecting_a_github_host_programmatically
+  github_app_installation_id = "47590865"
+
+  github_personal_access_token = var.im_github_pat
+}

examples/im_cloudbuild_workspace_github/outputs.tf

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  value = var.project_id
+}
+
+output "cloudbuild_preview_trigger_id" {
+  description = "Trigger used for creating IM previews"
+  value       = module.im_workspace.cloudbuild_preview_trigger_id
+}
+
+output "cloudbuild_apply_trigger_id" {
+  description = "Trigger used for running IM apply"
+  value       = module.im_workspace.cloudbuild_apply_trigger_id
+}
+
+output "cloudbuild_sa" {
+  description = "Service account used by the Cloud Build triggers"
+  value       = module.im_workspace.cloudbuild_sa
+}
+
+output "infra_manager_sa" {
+  description = "Service account used by Infrastructure Manager"
+  value       = module.im_workspace.infra_manager_sa
+}
+
+output "github_secret_id" {
+  description = "The secret ID for the GitHub secret containing the personal access token."
+  value       = module.im_workspace.github_secret_id
+  sensitive   = true
+}

examples/im_cloudbuild_workspace_github/variables.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The ID of the project in which to provision resources."
+  type        = string
+}
+
+variable "repository_url" {
+  description = "The URI of the repo where the Terraform configs are stored."
+  type        = string
+}
+
+variable "im_github_pat" {
+  description = "GitHub personal access token."
+  type        = string
+  sensitive   = true
+}

modules/im_cloudbuild_workspace/README.md

@@ -0,0 +1,113 @@
+## Overview
+
+This IM Cloud Build Workspace blueprint creates an opinionated workflow for actuating Terraform
+resources on Cloud Build using Infrastructure Manager. A set of Cloud Build triggers manage
+preview and apply operations on a configuration stored in a GitHub repository.
+The Cloud Build triggers use a per-workspace Service Account which can be configured with a
+minimal set of permissions for calling Infrastructure Manager. Infrastructure Manager uses a separate
+service account with a set of permissions required by the given Terraform configuration.
+
+## Usage
+
+Basic usage of this module is as follows:
+
+```hcl
+module "im-workspace" {
+  source = "terraform-google-modules/bootstrap/google//modules/im_cloudbuild_workspace"
+  version = "~> 7.0"
+
+  project_id = var.project_id
+  deployment_id = var.deployment_id
+  im_deployment_repo_uri = var.im_deployment_repo_uri
+  im_deployment_ref = var.im_deployment_ref
+
+  github_app_installation_id = var.github_app_installation_id
+  github_personal_access_token = var.github_personal_access_token
+}
+```
+
+## Resources Created
+
+This module creates:
+- Two Cloud Build triggers with an inline build configuration for planning and applying Terraform configurations
+using Infrastructure Manger. Additional optional build configurations can be specified.
+- Optional custom Service Accounts and roles to be used for invoking Cloud Build and used in Infrastructure Manager
+for actuating resources.
+- Connections to GitHub or GitLab repositories.
+
+![](./assets/arch.png)
+
+## Notes
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| cloudbuild\_apply\_filename | Optional Cloud Build YAML definition used for Cloud Build triggers of Infra Manager apply. Defaults to using inline definition. | `string` | `""` | no |
+| cloudbuild\_ignored\_files | Optional list. Changes only affecting ignored files will not invoke a build. | `list(string)` | `[]` | no |
+| cloudbuild\_included\_files | Optional list. Changes affecting at least one of these files will invoke a build. | `list(string)` | `[]` | no |
+| cloudbuild\_preview\_filename | Optional Cloud Build YAML definition used for Cloud Build triggers of Infra Manager preview. Defaults to using inline definition. | `string` | `""` | no |
+| cloudbuild\_sa | Custom SA ID of form projects/{{project}}/serviceAccounts/{{email}} to be used for creating Cloud Build triggers. Creates one if not given. | `string` | `""` | no |
+| custom\_cloudbuild\_sa\_name | Custom name to be used if creating a Cloud Build service account. Defaults to generated name if empty. | `string` | `""` | no |
+| custom\_infra\_manager\_sa\_name | Custom name to be used if creating an Infrastructure Manager service account. Defaults to generated name if empty. | `string` | `""` | no |
+| deployment\_id | Custom ID to be used for the Infrastructure Manager deployment. | `string` | n/a | yes |
+| github\_app\_installation\_id | Installation ID of the Cloud Build GitHub app used for pull and push request triggers. | `string` | `""` | no |
+| github\_pat\_secret | The secret ID within Secret Manager for an existing personal access token for GitHub. | `string` | `""` | no |
+| github\_pat\_secret\_version | The secret version ID or alias for the GitHub PAT secret. Uses the latest if not provided. | `string` | `""` | no |
+| github\_personal\_access\_token | Personal access token for a GitHub repository. If provided, creates a secret within Secret Manager. | `string` | `""` | no |
+| host\_connection\_name | Name for the VCS connection. Generated if not given. | `string` | `""` | no |
+| im\_deployment\_ref | Git branch or ref configured to run infra-manager apply. All other refs will run plan by default. | `string` | n/a | yes |
+| im\_deployment\_repo\_dir | The directory inside the repo where the Terraform root config is located. If empty defaults to repo root. | `string` | `""` | no |
+| im\_deployment\_repo\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes |
+| im\_tf\_variables | Optional list of Terraform variables to pass to Infrastructure Manager, if the configuration exists in a different repo. List of strings of form KEY=VALUE expected. | `string` | `""` | no |
+| infra\_manager\_sa | Custom SA id of form projects/{{project}}/serviceAccounts/{{email}} to be used by Infra Manager. Defaults to generated name if empty. | `string` | `""` | no |
+| infra\_manager\_sa\_roles | List of roles to grant to Infrastructure Manager SA for actuating resources defined in the Terraform configuration. | `list(string)` | `[]` | no |
+| location | Location for Infrastructure Manager deployment. | `string` | `"us-central1"` | no |
+| project\_id | GCP project for Infrastructure Manager deployments and Cloud Build triggers. | `string` | n/a | yes |
+| pull\_request\_comment\_control | Configure builds to run whether a repository owner or collaborator needs to comment /gcbrun. | `string` | `"COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"` | no |
+| repo\_connection\_name | Connection name for linked repository. Generated if not given. | `string` | `""` | no |
+| substitutions | Optional map of substitutions to use in builds if using a custom Cloud Build YAML definition. | `map(string)` | `{}` | no |
+| tf\_cloudbuilder | Name of the Cloud Builder image used for running build steps. | `string` | `"hashicorp/terraform:1.5.7"` | no |
+| tf\_repo\_type | Type of repo | `string` | `"GITHUB"` | no |
+| trigger\_location | Location of for Cloud Build triggers created in the workspace. Matches `location` if not given. | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_apply\_trigger\_id | Trigger used for running infra-manager apply |
+| cloudbuild\_preview\_trigger\_id | Trigger used for running infra-manager preview |
+| cloudbuild\_sa | Service account used by the Cloud Build triggers |
+| github\_secret\_id | The secret ID for the GitHub secret containing the personal access token. |
+| infra\_manager\_sa | Service account used by Infrastructure Manager |
+| repo\_connection\_id | The Cloud Build repository connection ID |
+| vcs\_connection\_id | The Cloud Build VCS host connection ID |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Requirements
+
+### Software
+
+-   [Terraform](https://www.terraform.io/downloads.html) ~> 1.2.3
+-   [terraform-provider-google] plugin >= 3.50.x
+
+### Permissions
+
+### APIs
+
+A project with the following APIs enabled must be used to host the
+resources of this module:
+
+```hcl
+"config.googleapis.com",
+"iam.googleapis.com",
+"cloudbuild.googleapis.com",
+"storage.googleapis.com",
+```
+
+## Contributing
+
+Refer to the [contribution guidelines](../../CONTRIBUTING.md) for
+information on contributing to this module.