Merge pull request #1 from rjerrems/inital-prototype

Initial prototype of org module

.gitignore

@@ -43,3 +43,11 @@ crash.log
 **/*.tfvars
 
 credentials.json
+
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+.idea/
+
+setup-tests.sh

test/integration/simple_example/controls/gsutil.rb → .pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,12 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-control "gsutil" do
-  title "gsutil"
-
-  describe command("gsutil ls -p #{attribute("project_id")}") do
-    its(:exit_status) { should eq 0 }
-    its(:stderr) { should eq "" }
-    its(:stdout) { should match "gs://#{attribute("bucket_name")}" }
-  end
-end
+repos:
+- repo: git://github.com/antonbabenko/pre-commit-terraform
+  rev: v1.21.0
+  hooks:
+    - id: terraform_fmt
+    # - id: terraform_docs
+- repo: git://github.com/pre-commit/pre-commit-hooks
+  rev: v2.2.3
+  hooks:
+    - id: check-executables-have-shebangs
+    - id: check-json
+    - id: check-merge-conflict
+    - id: check-xml
+    - id: check-yaml
+    - id: detect-private-key
+    - id: pretty-format-json
+      args: [--autofix]
+    - id: sort-simple-yaml
+    - id: trailing-whitespace

CONTRIBUTING.md

@@ -49,6 +49,8 @@ You will also need to set a few environment variables:
 export TF_VAR_org_id="your_org_id"
 export TF_VAR_folder_id="your_folder_id"
 export TF_VAR_billing_account="your_billing_account_id"
+export TF_VAR_group_org_admins="[email protected]"
+export TF_VAR_group_billing_admins="[email protected]"
 ```
 
 With these settings in place, you can prepare a test project using Docker:

Makefile

@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.6.0
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
@@ -39,6 +39,8 @@ docker_test_prepare:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
+		-e TF_VAR_group_org_admins \
+		-e TF_VAR_group_billing_admins \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh prepare_environment
@@ -51,6 +53,8 @@ docker_test_cleanup:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
+		-e TF_VAR_group_org_admins \
+		-e TF_VAR_group_billing_admins \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh cleanup_environment

README.md

@@ -1,10 +1,6 @@
 # terraform-google-bootstrap
 
-This module was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template/), which by default generates a module that simply creates a GCS bucket. As the module develops, this README should be updated.
-
-The resources/services/activations/deletions that this module will create/trigger are:
-
-- Create a GCS bucket with the provided name
+The purpose of this module is to help bootstrap a GCP organization, creating all the required GCP resources & permissions to start using the Cloud Foundation Toolkit (CFT). For users who want to use Cloud Build & Cloud Source Repos for foundations code, there is also a submodule to help bootstrap all the required resources to do this.
 
 ## Usage
 
@@ -15,68 +11,99 @@ module "bootstrap" {
   source  = "terraform-google-modules/bootstrap/google"
   version = "~> 0.1"
 
-  project_id  = "<PROJECT ID>"
-  bucket_name = "gcs-test-bucket"
+  org_id      = "<ORGANIZATION_ID>"
+  billing_account      = "<BILLING_ACCOUNT_ID>"
+  group_org_admins     = "[email protected]"
+  group_billing_admins = "[email protected]"
+  default_region       = "australia-southeast1"
 }
 ```
 
 Functional examples are included in the
 [examples](./examples/) directory.
 
+## Features
+
+The Organization Bootstrap module will take the following actions:
+
+1. Create a new GCP seed project using `project_prefix`.
+1. Enable APIs in the seed project using `activate_apis`
+1. Create a new service account for terraform in seed project
+1. Create GCS bucket for Terraform state and grant access to service account
+1. Grant IAM permissions required for CFT modules & Organization setup
+    1. Overwrite organization wide project creator and billing account creator roles
+    1. Grant Organization permissions to service account using `sa_org_iam_permissions`
+    1. Grant access to billing account for service account
+    1. Grant Organization permissions to `group_org_admins` using `org_admins_org_iam_permissions`
+    1. Grant billing permissions to `group_billing_admins`
+    1. (optional) Permissions required for service account impersonation using `sa_enable_impersonation`
+
+For the cloudbuild submodule, see the README [cloudbuild](./modules/cloudbuild).
+
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| bucket\_name | The name of the bucket to create | string | n/a | yes |
-| project\_id | The project ID to deploy to | string | n/a | yes |
+| activate\_apis | List of APIs to enable in the seed project. | list(string) | `<list>` | no |
+| billing\_account | The ID of the billing account to associate projects with. | string | n/a | yes |
+| default\_region | Default region to create resources where applicable. | string | `"us-central1"` | no |
+| folder\_id | The ID of a folder to host this project | string | `""` | no |
+| group\_billing\_admins | Google Group for GCP Billing Administrators | string | n/a | yes |
+| group\_org\_admins | Google Group for GCP Organization Administrators | string | n/a | yes |
+| org\_admins\_org\_iam\_permissions | List of permissions granted to the group supplied in group_org_admins variable across the GCP organization. | list(string) | `<list>` | no |
+| org\_id | GCP Organization ID | string | n/a | yes |
+| org\_project\_creators | Additional list of members to have project creator role accross the organization. Prefix of group: user: or serviceAccount: is required. | list(string) | `<list>` | no |
+| project\_prefix | Name prefix to use for projects created. | string | `"cft"` | no |
+| sa\_enable\_impersonation | Allow org_admins group to impersonate service account & enable APIs required. | bool | `"false"` | no |
+| sa\_org\_iam\_permissions | List of permissions granted to Terraform service account across the GCP organization. | list(string) | `<list>` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| bucket\_name |  |
+| gcs\_bucket\_tfstate | Bucket used for storing terraform state for foundations pipelines in seed project. |
+| seed\_project\_id | Project where service accounts and core APIs will be enabled. |
+| terraform\_sa\_email | Email for privileged service account for Terraform. |
+| terraform\_sa\_name | Fully qualified name for privileged service account for Terraform. |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
 
-These sections describe requirements for using this module.
-
 ### Software
 
-The following dependencies must be available:
+-   [gcloud sdk](https://cloud.google.com/sdk/install) >= 206.0.0
+-   [Terraform](https://www.terraform.io/downloads.html) >= 0.12.6
+-   [terraform-provider-google] plugin 2.1.x
+-   [terraform-provider-google-beta] plugin 2.1.x
 
-- [Terraform][terraform] v0.12
-- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0
+### Permissions
 
-### Service Account
+- `roles/resourcemanager.organizationAdmin` on GCP Organization
+- `roles/billing.admin` on supplied billing account
+- `roles/resourcemanager.projectCreator` on GCP Organization for `group_org_admins` group.
+- Account running terraform should be a member of group provided in `group_org_admins` variable, otherwise they will loose `roles/resourcemanager.projectCreator` access. Additional members can be added by using the `org_project_creators` variable.
 
-A service account with the following roles must be used to provision
-the resources of this module:
+### Credentials
 
-- Storage Admin: `roles/storage.admin`
-
-The [Project Factory module][project-factory-module] and the
-[IAM module][iam-module] may be used in combination to provision a
-service account with the necessary roles applied.
+For users interested in using service account impersonation which this module helps enable with `sa_enable_impersonation`, please see this [blog post](https://medium.com/google-cloud/terraform-assume-role-and-service-account-impersonation-on-google-cloud-ffc553863e72) which explains how it works.
 
 ### APIs
 
 A project with the following APIs enabled must be used to host the
 resources of this module:
 
-- Google Cloud Storage JSON API: `storage-api.googleapis.com`
+- Google Cloud Resource Manager API: `cloudresourcemanager.googleapis.com`
+- Google Cloud Billing API: `cloudbilling.googleapis.com`
+- Google Cloud IAM API: `iam.googleapis.com`
+- Google Cloud Storage API `storage-api.googleapis.com`
+- Google Cloud Service Usage API: `serviceusage.googleapis.com`
 
-The [Project Factory module][project-factory-module] can be used to
-provision a project with the necessary APIs enabled.
+This API can be enabled in the default project created during establishing an organization.
 
 ## Contributing
 
 Refer to the [contribution guidelines](./CONTRIBUTING.md) for
 information on contributing to this module.
-
-[iam-module]: https://registry.terraform.io/modules/terraform-google-modules/iam/google
-[project-factory-module]: https://registry.terraform.io/modules/terraform-google-modules/project-factory/google
-[terraform-provider-gcp]: https://www.terraform.io/docs/providers/google/index.html
-[terraform]: https://www.terraform.io/downloads.html

build/int.cloudbuild.yaml

@@ -21,15 +21,33 @@ steps:
   - 'TF_VAR_org_id=$_ORG_ID'
   - 'TF_VAR_folder_id=$_FOLDER_ID'
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
+  - '[email protected]'
+  - 'TF_VAR_group_billing_admins=test-gcp-billing-admins@test.infra.cft.tips'
 - id: create
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
-- id: converge
+- id: converge-simple
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge']
-- id: verify
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-default']
+- id: verify-simple
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-default']
+  # Required to rerun to reinstate ci-integration account as project creator as not member of group_org_admins.
+- id: prepare-rerun
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment']
+  env:
+  - 'TF_VAR_org_id=$_ORG_ID'
+  - 'TF_VAR_folder_id=$_FOLDER_ID'
+  - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
+  - '[email protected]'
+  - 'TF_VAR_group_billing_admins=test-gcp-billing-admins@test.infra.cft.tips'
+- id: converge-cloudbuild-enabled
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge cloudbuild-enabled-default']
+- id: verify-cloudbuild-enabled
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify cloudbuild-enabled-default']
 - id: destroy
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy']
@@ -38,4 +56,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0'

examples/cloudbuild_enabled/README.md

@@ -0,0 +1,31 @@
+## Overview
+
+This example combines the Organization bootstrap module with the Cloud Build submodule, to setup everything that is required to run subsequent infrastructure as code using cloud native tooling and limited external dependencies. For more details on what the Cloud Build module is doing, see the [readme](../../modules/cloudbuild).
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| billing\_account | The ID of the billing account to associate projects with. | string | n/a | yes |
+| default\_region | Default region to create resources where applicable. | string | `"us-central1"` | no |
+| group\_billing\_admins | Google Group for GCP Billing Administrators | string | n/a | yes |
+| group\_org\_admins | Google Group for GCP Organization Administrators | string | n/a | yes |
+| org\_id | GCP Organization ID | string | n/a | yes |
+| org\_project\_creators | Additional list of members to have project creator role accross the organization. Prefix of group: user: or serviceAccount: is required. | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_project\_id | Project where CloudBuild configuration and terraform container image will reside. |
+| csr\_repos | List of Cloud Source Repos created by the module, linked to Cloud Build triggers. |
+| gcs\_bucket\_cloudbuild\_artifacts | Bucket used to store Cloud/Build artefacts in CloudBuild project. |
+| gcs\_bucket\_tfstate | Bucket used for storing terraform state for foundations pipelines in seed project. |
+| kms\_crypto\_key | KMS key created by the module. |
+| kms\_keyring | KMS Keyring created by the module. |
+| seed\_project\_id | Project where service accounts and core APIs will be enabled. |
+| terraform\_sa\_email | Email for privileged service account for Terraform. |
+| terraform\_sa\_name | Fully qualified name for privileged service account for Terraform. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/cloudbuild_enabled/cloudbuild-tf-apply.yaml

@@ -0,0 +1,42 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 1200s
+steps:
+# terraform init -input=false
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - init
+  - -input=false
+# terraform validate
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - validate
+# terraform plan -input=false -out=tfplan
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - plan
+  - -input=false
+  - -out=tfplan
+# terraform apply -auto-approve -input=false tfplan
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - apply
+  - -auto-approve
+  - -input=false
+  - tfplan
+artifacts:
+  objects:
+    location: 'gs://${_ARTIFACT_BUCKET_NAME}/terraform/cloudbuild/apply/$BUILD_ID'
+    paths: ['cloudbuild-tf-apply.yaml', 'tfplan']

test/integration/simple_example/controls/gcp.rb → examples/cloudbuild_enabled/cloudbuild-tf-plan.yaml

@@ -1,4 +1,4 @@
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-control "gcp" do
-  title "GCP Resources"
-
-  describe google_storage_bucket(name: attribute("bucket_name")) do
-    it { should exist }
-  end
-end
+timeout: 1200s
+steps:
+# terraform init -input=false
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - init
+  - -input=false
+# terraform validate
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - validate
+# terraform plan -input=false -out=tfplan
+- name: gcr.io/$PROJECT_ID/terraform
+  args:
+  - plan
+  - -input=false
+  - -out=tfplan
+artifacts:
+  objects:
+    location: 'gs://${_ARTIFACT_BUCKET_NAME}/terraform/cloudbuild/plan/$BUILD_ID'
+    paths: ['cloudbuild-tf-plan.yaml', 'tfplan']