feat: add github and gitlab support on 1-bootstrap (#288)

Co-authored-by: Andrew Peabody <[email protected]>

1-bootstrap/README.md

@@ -22,6 +22,166 @@ Each pipeline has the following associated resources:
 
 ## Usage
 
+### Pre-requisites
+
+#### Secrets Project
+
+You will need a Google Cloud project with [Secret Manager](https://cloud.google.com/security/products/secret-manager) to store your git credentials, throughout the documentation this will be referenced as `$GIT_SECRET_PROJECT`.
+
+#### Cloud Build with Github Pre-requisites
+
+To proceed with GitHub as your git provider you will need:
+
+- An authenticated GitHub account. The steps in this documentation assumes you have a configured SSH key for cloning and modifying repositories.
+- A **private** [GitHub repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-new-repository) for each one of the repositories below:
+  - Multitenant (`eab-multitenant`)
+  - Fleetscope (`eab-fleetscope`)
+  - Application Factory (`eab-applicationfactory`)
+
+   > Note: Default names for the repositories are, in sequence: `eab-multitenant`, `eab-fleetscope` and `eab-applicationfactory`; If you choose other names for your repository make sure you update `terraform.tfvars` the repository names under `cloudbuildv2_repository_config` variable.
+
+- [Install Cloud Build App on Github](https://github.com/apps/google-cloud-build). After the installation, take note of the application id, it will be used later.
+- [Create Personal Access Token on Github with `repo` and `read:user` (or if app is installed in org use `read:org`)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) - After creating the token in Secret Manager, you will use the secret id in the `terraform.tfvars` file.
+- Create a secret for the Github Cloud Build App ID:
+
+   ```bash
+   APP_ID_VALUE=<replace_with_app_id>
+   printf $APP_ID_VALUE | gcloud secrets create github-app-id --project=$GIT_SECRET_PROJECT --data-file=-
+   ```
+
+- Take note of the secret id, it will be used in `terraform.tfvars` later on:
+
+   ```bash
+   gcloud secrets describe github-app-id --project=$GIT_SECRET_PROJECT --format="value(name)"
+   ```
+
+- Create a secret for the Github Personal Access Token:
+
+   ```bash
+   GITHUB_TOKEN=<replace_with_token>
+   printf $GITHUB_TOKEN | gcloud secrets create github-pat --project=$GIT_SECRET_PROJECT --data-file=-
+   ```
+
+- Take note of the secret id, it will be used in `terraform.tfvars` later on:
+
+   ```bash
+   gcloud secrets describe github-pat --project=$GIT_SECRET_PROJECT --format="value(name)"
+   ```
+
+- Populate your `terraform.tfvars` file in `1-bootstrap` with the Cloud Build 2nd Gen configuration variable, here is an example:
+
+   ```hcl
+   cloudbuildv2_repository_config = {
+      repo_type = "GITHUBv2"
+
+      repositories = {
+         multitenant = {
+            repository_name = "eab-multitenant"
+            repository_url  = "https://github.com/your-org/eab-multitenant.git"
+         }
+
+         applicationfactory = {
+            repository_name = "eab-applicationfactory"
+            repository_url  = "https://github.com/your-org/eab-applicationfactory.git"
+         }
+
+         fleetscope = {
+            repository_name = "eab-fleetscope"
+            repository_url  = "https://github.com/your-org/eab-fleetscope.git"
+         }
+      }
+
+      github_secret_id                            = "projects/REPLACE_WITH_PRJ_NUMBER/secrets/github-pat" # Personal Access Token Secret
+      github_app_id_secret_id                     = "projects/REPLACE_WITH_PRJ_NUMBER/secrets/github-app-id" # App ID value secret
+   }
+   ```
+
+#### Cloud Build with Gitlab Pre-requisites
+
+To proceed with Gitlab as your git provider you will need:
+
+- An authenticated Gitlab account. The steps in this documentation assumes you have a configured SSH key for cloning and modifying repositories.
+- A **private** GitLab repository for each one of the repositories below:
+  - Multitenant (`eab-multitenant`)
+  - Fleetscope (`eab-fleetscope`)
+  - Application Factory (`eab-applicationfactory`)
+
+   > Note: Default names for the repositories are, in sequence: `eab-multitenant`, `eab-fleetscope` and `eab-applicationfactory`; If you choose other names for your repository make sure you update `terraform.tfvars` the repository names under `cloudbuildv2_repository_config` variable.
+
+- An access token with the `api` scope to use for connecting and disconnecting repositories.
+
+- An access token with the `read_api` scope to ensure Cloud Build repositories can access source code in repositories.
+
+- Create a secret for the Gitlab API Access Token:
+
+   ```bash
+   GITLAB_API_TOKEN=<replace_with_app_id>
+   printf $GITLAB_API_TOKEN | gcloud secrets create gitlab-api-token --project=$GIT_SECRET_PROJECT --data-file=-
+   ```
+
+- Take note of the secret id, it will be used in `terraform.tfvars` later on:
+
+   ```bash
+   gcloud secrets describe gitlab-api-token --project=$GIT_SECRET_PROJECT --format="value(name)"
+   ```
+
+- Create a secret for the Gitlab Read API Access Token:
+
+   ```bash
+   GITLAB_READ_API_TOKEN=<replace_with_token>
+   printf $GITLAB_READ_API_TOKEN | gcloud secrets create gitlab-read-api-token --project=$GIT_SECRET_PROJECT --data-file=-
+   ```
+
+- Take note of the secret id, it will be used in `terraform.tfvars` later on:
+
+   ```bash
+   gcloud secrets describe gitlab-read-api-token --project=$GIT_SECRET_PROJECT --format="value(name)"
+   ```
+
+- Generate a random 36 character string that will be used as the Webhook Secret:
+
+   ```bash
+   GITLAB_WEBHOOK=<replace_with_webhook>
+   printf $GITLAB_WEBHOOK | gcloud secrets create gitlab-webhook --project=$GIT_SECRET_PROJECT --data-file=-
+   ```
+
+   > NOTE: for testing purposes, you may use the following command to generate the webhook in bash: `GITLAB_WEBHOOK=$(cat /dev/urandom | tr -dc "[:alnum:]" | head -c 36)`
+
+- Take note of the secret id, it will be used in `terraform.tfvars` later on:
+
+   ```bash
+   gcloud secrets describe gitlab-webhook --project=$GIT_SECRET_PROJECT --format="value(name)"
+   ```
+
+- Populate your `terraform.tfvars` file in `1-bootstrap` with the Cloud Build 2nd Gen configuration variable, here is an example:
+
+   ```hcl
+   cloudbuildv2_repository_config = {
+      repo_type = "GITLABv2"
+
+      repositories = {
+         multitenant = {
+            repository_name = "eab-multitenant"
+            repository_url  = "https://gitlab.com/your-group/eab-multitenant.git"
+         }
+
+         applicationfactory = {
+            repository_name = "eab-applicationfactory"
+            repository_url  = "https://gitlab.com/your-group/eab-applicationfactory.git"
+         }
+
+         fleetscope = {
+            repository_name = "eab-fleetscope"
+            repository_url  = "https://gitlab.com/your-group/eab-fleetscope.git"
+         }
+      }
+
+      gitlab_authorizer_credential_secret_id         = "projects/REPLACE_WITH_PRJ_NUMBER/secrets/gitlab-api-token"
+      gitlab_read_authorizer_credential_secret_id    = "projects/REPLACE_WITH_PRJ_NUMBER/secrets/gitlab-read-api-token"
+      gitlab_webhook_secret_id                       = "projects/REPLACE_WITH_PRJ_NUMBER/secrets/gitlab-webhook"
+   }
+   ```
+
 ### Deploying with Cloud Build
 
 #### Deploying on Enterprise Foundation blueprint
@@ -56,7 +216,9 @@ example-organization
    mv terraform.example.tfvars terraform.tfvars
    ```
 
-1. Update the `terraform.tfvars` file with your project id.
+1. Update the `terraform.tfvars` file with your project id. If you are using Github or Gitlab as your Git provider for Cloud Build, you will need to configure the `cloudbuildv2_repository_config` variable as described in the following sections:
+   - [Cloud Build with Github Pre-requisites](#cloud-build-with-github-pre-requisites)
+   - [Cloud Build with Gitlab Pre-requisites](#cloud-build-with-gitlab-pre-requisites)
 
 You can now deploy the common environment for these pipelines.
 
@@ -106,12 +268,13 @@ Within the repository, you'll find `backend.tf` files that define the GCS bucket
 |------|-------------|------|---------|:--------:|
 | bucket\_force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no |
 | bucket\_prefix | Name prefix to use for buckets created. | `string` | `"bkt"` | no |
+| cloudbuildv2\_repository\_config | Configuration for integrating repositories with Cloud Build v2:<br>  - repo\_type: Specifies the type of repository. Supported types are 'GITHUBv2', 'GITLABv2', and 'CSR'.<br>  - repositories: A map of repositories to be created. The key must match the exact name of the repository. Each repository is defined by:<br>      - repository\_name: The name of the repository.<br>      - repository\_url: The URL of the repository.<br>  - github\_secret\_id: (Optional) The personal access token for GitHub authentication.<br>  - github\_app\_id\_secret\_id: (Optional) The application ID for a GitHub App used for authentication.<br>  - gitlab\_read\_authorizer\_credential\_secret\_id: (Optional) The read authorizer credential for GitLab access.<br>  - gitlab\_authorizer\_credential\_secret\_id: (Optional) The authorizer credential for GitLab access.<br>  - gitlab\_webhook\_secret\_id: (Optional) The secret ID for the GitLab WebHook..<br>Note: When using GITLABv2, specify `gitlab_read_authorizer_credential` and `gitlab_authorizer_credential` and `gitlab_webhook_secret_id`.<br>Note: When using GITHUBv2, specify `github_pat` and `github_app_id`.<br>Note: If 'cloudbuildv2\_repository\_config' variable is not configured, CSR (Cloud Source Repositories) will be used by default. | <pre>object({<br>    repo_type = string # Supported values are: GITHUBv2, GITLABv2 and CSR<br>    # repositories to be created<br>    repositories = object({<br>      multitenant = object({<br>        repository_name = optional(string, "eab-multitenant")<br>        repository_url  = string<br>      }),<br>      applicationfactory = object({<br>        repository_name = optional(string, "eab-applicationfactory")<br>        repository_url  = string<br>      }),<br>      fleetscope = object({<br>        repository_name = optional(string, "eab-fleetscope")<br>        repository_url  = string<br>      }),<br>    })<br>    # Credential Config for each repository type<br>    github_secret_id                            = optional(string)<br>    github_app_id_secret_id                     = optional(string)<br>    gitlab_read_authorizer_credential_secret_id = optional(string)<br>    gitlab_authorizer_credential_secret_id      = optional(string)<br>    gitlab_webhook_secret_id                    = optional(string)<br>  })</pre> | <pre>{<br>  "repo_type": "CSR",<br>  "repositories": {<br>    "applicationfactory": {<br>      "repository_url": ""<br>    },<br>    "fleetscope": {<br>      "repository_url": ""<br>    },<br>    "multitenant": {<br>      "repository_url": ""<br>    }<br>  }<br>}</pre> | no |
 | common\_folder\_id | Folder ID in which to create all application admin projects, must be prefixed with 'folders/' | `string` | n/a | yes |
 | envs | Environments | <pre>map(object({<br>    billing_account    = string<br>    folder_id          = string<br>    network_project_id = string<br>    network_self_link  = string<br>    org_id             = string<br>    subnets_self_links = list(string)<br>  }))</pre> | n/a | yes |
 | location | Location for build buckets. | `string` | `"us-central1"` | no |
 | project\_id | Project ID for initial resources | `string` | n/a | yes |
 | tf\_apply\_branches | List of git branches configured to run terraform apply Cloud Build trigger. All other branches will run plan by default. | `list(string)` | <pre>[<br>  "development",<br>  "nonproduction",<br>  "production"<br>]</pre> | no |
-| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"global"` | no |
+| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"us-central1"` | no |
 
 ## Outputs
 

1-bootstrap/main.tf

@@ -17,34 +17,52 @@
 locals {
   cb_config = {
     "multitenant" = {
-      repo_name    = "eab-multitenant",
       bucket_infix = "mt"
       roles = [
         "roles/container.admin"
       ]
     }
     "applicationfactory" = {
-      repo_name    = "eab-applicationfactory",
       bucket_infix = "af"
       roles        = ["roles/resourcemanager.projectIamAdmin"]
     }
     "fleetscope" = {
-      repo_name    = "eab-fleetscope",
       bucket_infix = "fs"
       roles        = []
     }
   }
+  use_csr                    = var.cloudbuildv2_repository_config.repo_type == "CSR"
+  csr_repos                  = local.use_csr ? { for k, v in var.cloudbuildv2_repository_config.repositories : k => v.repository_name } : {}
   cb_service_accounts_emails = { for k, v in module.tf_cloudbuild_workspace : k => reverse(split("/", v.cloudbuild_sa))[0] }
 }
 
 resource "google_sourcerepo_repository" "gcp_repo" {
-  for_each = local.cb_config
+  for_each = local.csr_repos
 
   project                      = var.project_id
-  name                         = each.value.repo_name
+  name                         = each.value
   create_ignore_already_exists = true
 }
 
+module "cloudbuild_repositories" {
+  count = local.use_csr ? 0 : 1
+
+  source  = "terraform-google-modules/bootstrap/google//modules/cloudbuild_repo_connection"
+  version = "~> 10.0"
+
+  project_id = var.project_id
+
+  connection_config = {
+    connection_type                             = var.cloudbuildv2_repository_config.repo_type
+    github_secret_id                            = var.cloudbuildv2_repository_config.github_secret_id
+    github_app_id_secret_id                     = var.cloudbuildv2_repository_config.github_app_id_secret_id
+    gitlab_read_authorizer_credential_secret_id = var.cloudbuildv2_repository_config.gitlab_read_authorizer_credential_secret_id
+    gitlab_authorizer_credential_secret_id      = var.cloudbuildv2_repository_config.gitlab_authorizer_credential_secret_id
+    gitlab_webhook_secret_id                    = var.cloudbuildv2_repository_config.gitlab_webhook_secret_id
+  }
+  cloud_build_repositories = var.cloudbuildv2_repository_config.repositories
+}
+
 module "tfstate_bucket" {
   source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
   version = "~> 8.0"
@@ -56,19 +74,19 @@ module "tfstate_bucket" {
 }
 
 module "tf_cloudbuild_workspace" {
+  for_each = var.cloudbuildv2_repository_config.repositories
+
   source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace"
   version = "~> 10.0"
 
-  for_each = local.cb_config
-
   project_id = var.project_id
   location   = var.location
 
-  tf_repo_uri           = google_sourcerepo_repository.gcp_repo[each.key].url
-  tf_repo_type          = "CLOUD_SOURCE_REPOSITORIES"
+  tf_repo_uri           = local.use_csr ? google_sourcerepo_repository.gcp_repo[each.key].url : module.cloudbuild_repositories[0].cloud_build_repositories_2nd_gen_repositories[each.key].id
+  tf_repo_type          = local.use_csr ? "CLOUD_SOURCE_REPOSITORIES" : "CLOUDBUILD_V2_REPOSITORY"
   trigger_location      = var.trigger_location
-  artifacts_bucket_name = "${var.bucket_prefix}-${var.project_id}-${each.value.bucket_infix}-build"
-  log_bucket_name       = "${var.bucket_prefix}-${var.project_id}-${each.value.bucket_infix}-logs"
+  artifacts_bucket_name = "${var.bucket_prefix}-${var.project_id}-${local.cb_config[each.key].bucket_infix}-build"
+  log_bucket_name       = "${var.bucket_prefix}-${var.project_id}-${local.cb_config[each.key].bucket_infix}-logs"
 
   create_state_bucket    = false
   state_bucket_self_link = module.tfstate_bucket.bucket.self_link
@@ -78,7 +96,7 @@ module "tf_cloudbuild_workspace" {
   cloudbuild_sa_roles = {
     "roles" = {
       project_id = var.project_id
-    roles = each.value.roles }
+    roles = local.cb_config[each.key].roles }
   }
 
   substitutions = {