fix: docs, jenkins GCE missing scopes & roles (terraform-google-modul…

…es#109)

0-bootstrap/README.md

@@ -1,6 +1,6 @@
 # 0-bootstrap
 
-The purpose of this step is to bootstrap a GCP organization, creating all the required resources & permissions to start using the Cloud Foundation Toolkit (CFT). This step also configures Cloud Build & Cloud Source Repos for foundations code in subsequent stages.
+The purpose of this step is to bootstrap a GCP organization, creating all the required resources & permissions to start using the Cloud Foundation Toolkit (CFT). This step also configures a CICD pipeline for foundations code in subsequent stages. The CICD pipeline can use either Cloud Build & Cloud Source Repos or Jenkins & your own Git repos (which might live on-prem).
 
 ## Prerequisites
 
@@ -12,7 +12,11 @@ The purpose of this step is to bootstrap a GCP organization, creating all the re
 
 Further details of permissions required and resources created, can be found in the bootstrap module [documentation.](https://github.com/terraform-google-modules/terraform-google-bootstrap)
 
-## Usage
+## 0-bootstrap usage to deploy Jenkins
+
+If you are using the `jenkins_bootstrap` sub-module, please see [README-Jenkins](./README-Jenkins.md) for requirements and instructions on how to run the 0-bootstrap step. Using Jenkins requires a few manual steps, including configuring connectivity with your current Jenkins Master environment.
+
+## 0-bootstrap usage to deploy Cloud Build
 
 1. Change into 0-bootstrap folder
 1. Copy tfvars by running `cp terraform.example.tfvars terraform.tfvars` and update `terraform.tfvars` with values from your environment.
@@ -62,13 +66,8 @@ Currently, the bucket information is replaced in the state backends as a part of
 
 ## Requirements
 
-If you are using `jenkins_bootstrap`, please see the [README](./modules/jenkins-agent/README.md) for the requirements.
-
-## Instructions
-
-If you are using `jenkins_bootstrap`, please follow the instructions on how to run the bootstrap step with the `jenkins_bootstrap` sub-module described in the [README](./modules/jenkins-agent/README.md), which include implementing VPN, configuring your Jenkins Master among other steps.
-
 ### Software
 
--   [gcloud sdk](https://cloud.google.com/sdk/install) >= 206.0.0
--   [Terraform](https://www.terraform.io/downloads.html) >= 0.12.6
+- [gcloud sdk](https://cloud.google.com/sdk/install) >= 206.0.0
+- [Terraform](https://www.terraform.io/downloads.html) >= 0.12.6
+    - You should use the same version in the manual steps during 0-bootstrap to avoid possible  [Terraform State Snapshot Lock](https://github.com/hashicorp/terraform/issues/23290) errors caused by differences in terraform versions. This can usually be resolved with a version upgrade.

0-bootstrap/backend.tf.example

@@ -1,6 +1,6 @@
 terraform {
   backend "gcs" {
-    bucket = "STATE_BUCKET"
+    bucket = "UPDATE_ME"
     prefix = "terraform/bootstrap/state"
   }
 }

0-bootstrap/main.tf

@@ -83,6 +83,7 @@ module "seed_bootstrap" {
     "roles/iam.serviceAccountAdmin",
     "roles/logging.configWriter",
     "roles/orgpolicy.policyAdmin",
+    "roles/resourcemanager.projectCreator",
     "roles/resourcemanager.folderAdmin",
     "roles/securitycenter.notificationConfigEditor",
     "roles/resourcemanager.organizationViewer"

0-bootstrap/modules/jenkins-agent/main.tf

@@ -95,10 +95,7 @@ resource "google_compute_instance" "jenkins_agent_gce_instance" {
   service_account {
     email = google_service_account.jenkins_agent_gce_sa.email
     scopes = [
-      // TODO(caleonardo): These scopes will need to change.
-      "https://www.googleapis.com/auth/compute.readonly",
-      "https://www.googleapis.com/auth/devstorage.read_only",
-      "https://www.googleapis.com/auth/userinfo.email",
+      "https://www.googleapis.com/auth/cloud-platform",
     ]
   }
 

0-bootstrap/modules/jenkins-agent/outputs.tf

@@ -27,6 +27,11 @@ output "jenkins_agent_gce_instance_id" {
   value       = google_compute_instance.jenkins_agent_gce_instance.id
 }
 
+output "jenkins_agent_vpc_id" {
+  description = "Jenkins Agent VPC name."
+  value       = google_compute_network.jenkins_agents.id
+}
+
 output "jenkins_agent_sa_email" {
   description = "Email for privileged custom service account for Jenkins Agent GCE instance."
   value       = google_service_account.jenkins_agent_gce_sa.email

0-bootstrap/outputs.tf

@@ -77,6 +77,11 @@ output "kms_crypto_key" {
 //  value       = module.jenkins_bootstrap.jenkins_agent_gce_instance_id
 //}
 //
+//output "jenkins_agent_vpc_id" {
+//  description = "Jenkins Agent VPC name."
+//  value       = module.jenkins_bootstrap.jenkins_agent_vpc_id
+//}
+//
 //output "jenkins_agent_sa_email" {
 //  description = "Email for privileged custom service account for Jenkins Agent GCE instance."
 //  value       = module.jenkins_bootstrap.jenkins_agent_sa_email

0-bootstrap/terraform.example.tfvars

@@ -26,3 +26,19 @@ default_region = "australia-southeast1"
 
 //Optional - for development.  Will place all resources under a specific folder instead of org root
 //parent_folder = "01234567890"
+
+/* ----------------------------------------
+    Specific to jenkins_bootstrap module
+   ---------------------------------------- */
+// Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build
+//jenkins_agent_gce_subnetwork_cidr_range = "10.2.0.0/24"
+//
+//jenkins_agent_gce_private_ip_address = "10.2.0.6"
+//
+//jenkins_agent_gce_ssh_pub_key = "ssh-rsa [KEY_VALUE] [USERNAME]"
+//
+//jenkins_agent_sa_email = "jenkins-agent-gce" # service_account_prefix will be added
+//
+//jenkins_master_ip_addresses = ["10.1.0.6/32"]
+//
+//nat_bgp_asn = "64514"

0-bootstrap/variables.tf

@@ -58,9 +58,9 @@ variable "skip_gcloud_download" {
   default     = true
 }
 
-/////* ----------------------------------------
-////    Specific to jenkins_bootstrap module
-////   ---------------------------------------- */
+/* ----------------------------------------
+    Specific to jenkins_bootstrap module
+   ---------------------------------------- */
 //// Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build
 //variable "jenkins_agent_gce_subnetwork_cidr_range" {
 //  description = "The subnetwork to which the Jenkins Agent will be connected to (in CIDR range 0.0.0.0/0)"

1-org/envs/shared/README.md

@@ -5,7 +5,7 @@ The purpose of this step is to setup top level shared folders, monitoring & netw
 ## Prerequisites
 
 1. 0-bootstrap executed successfully.
-2. Cloud Identity / Gsuite group for security admins
+2. Cloud Identity / G Suite group for security admins
 3. Membership in the security admins group for user running terraform
 
 ## Usage
@@ -29,7 +29,42 @@ You can choose not to enable the Data Access logs by setting variable `data_acce
 1. Merge changes to prod branch with `git checkout -b prod` and `git push origin prod`
     1. Review the apply output in your cloud build project https://console.cloud.google.com/cloud-build/builds?project=YOUR_CLOUD_BUILD_PROJECT_ID
 
-### Run terraform locally
+#### Setup to run via Jenkins
+1. Clone repo you created manually in Bootstrap: `git clone <YOUR_NEW_REPO-1-org>`
+1. Navigate into the repo `cd YOUR_NEW_REPO_CLONE-1-org` and change to a non prod branch `git checkout -b plan`
+1. Copy contents of foundation to new repo `cp -R ../terraform-example-foundation/1-org/* .` (modify accordingly based on your current directory).
+1. Copy the `Jenkinsfile` to the root of your new repository and replace the `_TF_SA_EMAIL` with the name of your Terraform Service Account in the `seed` project. (modify accordingly based on your current directory):
+   ```
+   cp ../terraform-example-foundation/build/Jenkinsfile .
+   sed -i 's/_TF_SA_EMAIL/TF_SERVICE_ACCOUNT_EMAIL/' Jenkinsfile
+   ```
+   **If using MacOS:**
+    ```
+    cp ../terraform-example-foundation/build/Jenkinsfile .
+    sed -i '.bak' 's/_TF_SA_EMAIL/TF_SERVICE_ACCOUNT_EMAIL/' Jenkinsfile
+    rm Jenkinsfile.bak
+    ```
+1. Copy the `tf-wrapper.sh` configuration file to the root of your new repository (modify accordingly based on your current directory):
+    ```
+    cp ../terraform-example-foundation/build/tf-wrapper.sh .
+    ```
+1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
+1. Rename `terraform.example.tfvars` to `terraform.tfvars` and update the file with values from your environment. (you can re-run `terraform output` in the 0-bootstrap directory to find these values). Make sure that `default_region` is set to a valid [BigQuery dataset region](https://cloud.google.com/bigquery/docs/locations).
+    ```
+    # Rename file
+    mv envs/shared/terraform.example.tfvars envs/shared/terraform.tfvars
+    # Edit the file to provide the necessary values
+    vi envs/shared/terraform.tfvars
+    ```
+
+1. Commit changes with `git add .` and `git commit -m 'Your message'`
+1. Push your plan branch `git push --set-upstream origin plan`. The branch `plan` is not a special one. Any branch which name is different from `dev`, `nonprod` or `prod` will trigger a terraform plan.
+    - Assuming you configured an automatic trigger in your Jenkins Master (see [Jenkins sub-module README](../../../0-bootstrap/modules/jenkins-agent)), this will trigger a plan. You can also trigger a Jenkins job manually. Given the many options to do this in Jenkins, it is out of the scope of this document see [Jenkins website](www.jenkins.io) for more details.
+    1. Review the plan output in your Master's web UI.
+1. Merge changes to prod branch with `git checkout -b prod` and `git push origin prod`
+    1. Review the apply output in your Master's web UI (You might want to use the option to "Scan Multibranch Pipeline Now" in your Jenkins Master UI).
+
+### (Optional) If you want to run  terraform locally
 1. Change into 1-org/envs/shared/ folder.
 1. Rename terraform.example.tfvars to terraform.tfvars and update the file with values from your environment and bootstrap.
 1. Update backend.tf with your bucket from bootstrap.
@@ -42,10 +77,10 @@ You can choose not to enable the Data Access logs by setting variable `data_acce
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| audit\_data\_users | Gsuite or Cloud Identity group that have access to audit logs. | string | n/a | yes |
+| audit\_data\_users | G Suite or Cloud Identity group that have access to audit logs. | string | n/a | yes |
 | audit\_logs\_table\_expiration\_ms | Period before tables expire for all audit logs in milliseconds. Default is 30 days. | number | `"2592000000"` | no |
 | billing\_account | The ID of the billing account to associate this project with | string | n/a | yes |
-| billing\_data\_users | Gsuite or Cloud Identity group that have access to billing data set. | string | n/a | yes |
+| billing\_data\_users | G Suite or Cloud Identity group that have access to billing data set. | string | n/a | yes |
 | create\_access\_context\_manager\_access\_policy | Whether to create access context manager access policy | bool | `"true"` | no |
 | data\_access\_logs\_enabled | Enable Data Access logs of types DATA_READ, DATA_WRITE and ADMIN_READ for all GCP services. Enabling Data Access logs might result in your organization being charged for the additional logs usage. See https://cloud.google.com/logging/docs/audit#data-access | bool | `"true"` | no |
 | default\_region | Default region for BigQuery resources. | string | n/a | yes |

1-org/envs/shared/variables.tf

@@ -35,12 +35,12 @@ variable "default_region" {
 }
 
 variable "billing_data_users" {
-  description = "Gsuite or Cloud Identity group that have access to billing data set."
+  description = "G Suite or Cloud Identity group that have access to billing data set."
   type        = string
 }
 
 variable "audit_data_users" {
-  description = "Gsuite or Cloud Identity group that have access to audit logs."
+  description = "G Suite or Cloud Identity group that have access to audit logs."
   type        = string
 }
 

2-environments/README.md

@@ -6,11 +6,11 @@ The purpose of this step is to set up dev, nonprod, and prod environments within
 
 1. 0-bootstrap executed successfully.
 1. 1-org executed successfully.
-1. Cloud Identity / Gsuite group for monitoring admins.
+1. Cloud Identity / G Suite group for monitoring admins.
 1. Membership in the monitoring admins group for user running terraform
 
 ## Usage
-### Setup to run via Cloud Build
+### Setup to run the automated pipelines
 1. Clone repo `gcloud source repos clone gcp-environments --project=YOUR_CLOUD_BUILD_PROJECT_ID`
 1. Change freshly cloned repo and change to non master branch `git checkout -b plan`
 1. Copy contents of foundation to new repo `cp -R ../terraform-example-foundation/2-environments/* .` (modify accordingly based on your current directory)
@@ -19,6 +19,8 @@ The purpose of this step is to set up dev, nonprod, and prod environments within
 1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
 1. Rename terraform.example.tfvars to terraform.tfvars and update the file with values from your environment and bootstrap.
 1. Commit changes with `git add .` and `git commit -m 'Your message'`
+
+#### If using Cloud Build
 1. Push your plan branch to trigger a plan for all environments `git push --set-upstream origin plan` (the branch `plan` is not a special one. Any branch which name is different from `dev`, `nonprod` or `prod` will trigger a terraform plan).
     1. Review the plan output in your cloud build project https://console.cloud.google.com/cloud-build/builds?project=YOUR_CLOUD_BUILD_PROJECT_ID
 1. Merge changes to dev with `git checkout -b dev` and `git push origin dev`
@@ -28,8 +30,18 @@ The purpose of this step is to set up dev, nonprod, and prod environments within
 1. Merge changes to prod with `git checkout -b prod` and `git push origin prod`
     1. Review the apply output in your cloud build project https://console.cloud.google.com/cloud-build/builds?project=YOUR_CLOUD_BUILD_PROJECT_ID
 
+#### If using Jenkins
+1. Push your plan branch to trigger a plan for all environments `git push --set-upstream origin plan` (the branch `plan` is not a special one. Any branch which name is different from `dev`, `nonprod` or `prod` will trigger a terraform plan).
+    1. Review the plan output in your Master's web UI.
+1. Merge changes to dev with `git checkout -b dev` and `git push origin dev`
+    1. Review the apply output in your Master's web UI.
+1. Merge changes to nonprod with `git checkout -b nonprod` and `git push origin nonprod`
+    1. Review the apply output in your Master's web UI.
+1. Merge changes to prod with `git checkout -b prod` and `git push origin prod`
+    1. Review the apply output in your Master's web UI.
+
 
-### Run terraform locally
+### (Optional) If running terraform locally
 1. Change into 2-environments folder
 1. Rename terraform.example.tfvars to terraform.tfvars and update the file with values from your environment and bootstrap. Copy terraform.tfvars into each of the folders within the envs/ folder.
 1. Within each envs/ folder, update backend.tf with your bucket name from the bootstrap step.

build/Jenkinsfile

@@ -0,0 +1,45 @@
+pipeline {
+  agent {
+      label "gcp-agent"
+  }
+  stages {
+    stage('setup') {
+      steps {
+          sh '''
+          echo "Setting up gcloud for impersonation"
+          gcloud config set auth/impersonate_service_account ${_TF_SA_EMAIL}
+          echo "Adding bucket information to backends"
+          for i in `find -name 'backend.tf'`; do sed -i 's/UPDATE_ME/${_STATE_BUCKET_NAME}/' $i; done
+          '''
+        }
+    }
+    stage('TF init') {
+      steps {
+          sh '''
+          ./tf-wrapper.sh init $BRANCH_NAME
+          '''
+        }
+    }
+    stage('TF plan') {
+      steps {
+          sh '''
+          ./tf-wrapper.sh plan $BRANCH_NAME
+          '''
+        }
+    }
+    stage('TF validate') {
+      steps {
+          sh '''
+          ./tf-wrapper.sh validate $BRANCH_NAME $POLICY_REPO
+          '''
+        }
+    }
+    stage('TF apply') {
+      steps {
+          sh '''
+          ./tf-wrapper.sh apply $BRANCH_NAME
+          '''
+        }
+    }
+  }
+}