From ec3501cd80ab22e5967fa062a65462f2780e9c99 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Mon, 22 Oct 2018 16:42:31 +0900 Subject: [PATCH 01/17] implement terraform-google-pubsub module --- .gitignore | 2 + Gemfile | 19 ++ Gemfile.lock | 216 +++++++++++++++ Makefile | 70 +++++ README.md | 148 ++++++++++- examples/example.tf | 46 ++++ helpers/combine_docfiles.py | 47 ++++ kitchen.yml | 33 +++ main.tf | 48 ++++ outputs.tf | 29 ++ test/boilerplate/boilerplate.Dockerfile.txt | 13 + test/boilerplate/boilerplate.Makefile.txt | 13 + test/boilerplate/boilerplate.go.txt | 15 ++ test/boilerplate/boilerplate.py.txt | 13 + test/boilerplate/boilerplate.rb.txt | 13 + test/boilerplate/boilerplate.sh.txt | 13 + test/boilerplate/boilerplate.tf.txt | 15 ++ test/boilerplate/boilerplate.xml.txt | 15 ++ test/boilerplate/boilerplate.yaml.txt | 13 + test/boilerplate/boilerplate.yml.txt | 13 + test/fixtures/main.tf | 39 +++ test/fixtures/outputs.tf | 19 ++ test/fixtures/variables.tf | 27 ++ test/integration/default/controls/pubsub.rb | 38 +++ test/integration/default/inspec.yml | 21 ++ test/integration/default/sample.sh | 24 ++ test/make.sh | 98 +++++++ test/test.sh | 22 ++ test/test_verify_boilerplate.py | 132 +++++++++ test/verify_boilerplate.py | 279 ++++++++++++++++++++ variables.tf | 35 +++ 31 files changed, 1527 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 Makefile create mode 100644 examples/example.tf create mode 100644 helpers/combine_docfiles.py create mode 100644 kitchen.yml create mode 100644 main.tf create mode 100644 outputs.tf create mode 100644 test/boilerplate/boilerplate.Dockerfile.txt create mode 100644 test/boilerplate/boilerplate.Makefile.txt create mode 100644 test/boilerplate/boilerplate.go.txt create mode 100644 test/boilerplate/boilerplate.py.txt create mode 100644 test/boilerplate/boilerplate.rb.txt create mode 100644 test/boilerplate/boilerplate.sh.txt create mode 100644 test/boilerplate/boilerplate.tf.txt create mode 100644 test/boilerplate/boilerplate.xml.txt create mode 100644 test/boilerplate/boilerplate.yaml.txt create mode 100644 test/boilerplate/boilerplate.yml.txt create mode 100644 test/fixtures/main.tf create mode 100644 test/fixtures/outputs.tf create mode 100644 test/fixtures/variables.tf create mode 100644 test/integration/default/controls/pubsub.rb create mode 100644 test/integration/default/inspec.yml create mode 100644 test/integration/default/sample.sh create mode 100755 test/make.sh create mode 100755 test/test.sh create mode 100755 test/test_verify_boilerplate.py create mode 100755 test/verify_boilerplate.py create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82d1b5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.kitchen/ +.kitchen.local.yml diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..e27ae50 --- /dev/null +++ b/Gemfile @@ -0,0 +1,19 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source 'https://rubygems.org/' + +gem 'test-kitchen' +gem 'kitchen-terraform', '~> 3.3' +gem 'kitchen-inspec', git: 'https://github.com/inspec/kitchen-inspec.git', ref: '0590f1b' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..8bdeff3 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,216 @@ +GIT + remote: https://github.com/inspec/kitchen-inspec.git + revision: 0590f1bbc99d3a28c31ca8841c5d6cb4139ab773 + ref: 0590f1b + specs: + kitchen-inspec (0.23.1) + hashie (~> 3.4) + inspec (>= 0.34.0, < 3.0.0) + test-kitchen (~> 1.6) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + aws-sdk (2.11.154) + aws-sdk-resources (= 2.11.154) + aws-sdk-core (2.11.154) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.11.154) + aws-sdk-core (= 2.11.154) + aws-sigv4 (1.0.3) + azure_mgmt_resources (0.17.2) + ms_rest_azure (~> 0.11.0) + builder (3.2.3) + coderay (1.1.2) + concurrent-ruby (1.0.5) + diff-lcs (1.3) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.7.0) + concurrent-ruby (~> 1.0) + dry-container (0.6.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.1) + dry-inflector (0.1.2) + dry-logic (0.4.2) + dry-container (~> 0.2, >= 0.2.6) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-types (0.13.2) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) + dry-validation (0.12.2) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (~> 0.2, >= 0.2.1) + dry-equalizer (~> 0.2) + dry-logic (~> 0.4, >= 0.4.0) + dry-types (~> 0.13.1) + erubis (2.7.0) + excon (0.62.0) + faraday (0.15.3) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + ffi (1.9.25) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + hashie (3.6.0) + htmlentities (4.3.4) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + inifile (3.0.0) + inspec (2.1.72) + addressable (~> 2.4) + faraday (>= 0.9.0) + hashie (~> 3.4) + htmlentities + json (>= 1.8, < 3.0) + method_source (~> 0.8) + mixlib-log + parallel (~> 1.9) + parslet (~> 1.5) + pry (~> 0) + rspec (~> 3) + rspec-its (~> 1.2) + rubyzip (~> 1.1) + semverse + sslshake (~> 1.2) + thor (~> 0.20) + tomlrb (~> 1.2) + train (~> 1.4) + jmespath (1.4.0) + json (2.1.0) + kitchen-terraform (3.3.1) + dry-types (~> 0.9) + dry-validation (~> 0.10) + kitchen-inspec (~> 0.18) + mixlib-shellout (~> 2.2) + test-kitchen (~> 1.16) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + method_source (0.9.0) + mixlib-install (3.11.5) + mixlib-shellout + mixlib-versioning + thor + mixlib-log (2.0.4) + mixlib-shellout (2.4.0) + mixlib-versioning (1.2.2) + ms_rest (0.7.3) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.11.0) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.2) + multi_json (1.13.1) + multipart-post (2.0.0) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + nori (2.6.0) + parallel (1.12.1) + parslet (1.8.2) + pry (0.11.3) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.0.3) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + semverse (2.0.0) + sslshake (1.2.0) + test-kitchen (1.23.2) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.20.0) + timeliness (0.3.8) + tomlrb (1.2.7) + train (1.4.4) + aws-sdk (~> 2) + azure_mgmt_resources (~> 0.15) + docker-api (~> 1.26) + inifile + json (>= 1.8, < 3.0) + mixlib-shellout (~> 2.0) + net-scp (~> 1.2) + net-ssh (>= 2.9, < 5.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + winrm (2.3.0) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.3.1) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + kitchen-inspec! + kitchen-terraform (~> 3.3) + test-kitchen + +BUNDLED WITH + 1.16.6 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..df5f9cf --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make will use bash instead of sh +SHELL := /usr/bin/env bash + +# All is the first target in the file so it will get picked up when you just run 'make' on its own +all: check_shell check_python check_golang check_terraform check_docker check_base_files check_trailing_whitespace generate_docs + +# The .PHONY directive tells make that this isn't a real target and so +# the presence of a file named 'check_shell' won't cause this target to stop +# working +.PHONY: check_shell +check_shell: + @source test/make.sh && check_shell + +.PHONY: check_python +check_python: + @source test/make.sh && check_python + +.PHONY: check_golang +check_golang: + @source test/make.sh && golang + +.PHONY: check_terraform +check_terraform: + @source test/make.sh && check_terraform + +.PHONY: check_docker +check_docker: + @source test/make.sh && docker + +.PHONY: check_base_files +check_base_files: + @source test/make.sh && basefiles + +.PHONY: check_shebangs +check_shebangs: + @source test/make.sh && check_bash + +.PHONY: check_trailing_whitespace +check_trailing_whitespace: + @source test/make.sh && check_trailing_whitespace + +.PHONY: test_check_headers +test_check_headers: + @python test/test_verify_boilerplate.py + +.PHONY: check_headers +check_headers: + @python test/verify_boilerplate.py + +.PHONY: generate_docs +generate_docs: + @source test/make.sh && generate_docs + +.PHONY: test +test: + @source test/test.sh diff --git a/README.md b/README.md index cb7274b..3f4218b 100644 --- a/README.md +++ b/README.md @@ -1 +1,147 @@ -# terraform-google-pubsub \ No newline at end of file +# terraform-google-pubsub + +This module makes it easy to create Google Cloud Pub/Sub topic and subscriptions associated with the topic. + +## Usage + +This is a simple usage of the module. Please see also a simple setup provided in the example directory. + +```hcl +module "pubsub" { + source = "github.com/terraform-google-modules/terraform-google-pubsub" + topic = "tf-topic" + project_id = "my-pubsub-project" + push_subscriptions = [ + { + name = "push" // required + ack_deadline_seconds = 20 // optonal + push_endpoint = "https://example.com" + } + ] + pull_subscriptions = [ + { + name = "pull" // required + ack_deadline_seconds = 20 // optional + } + ] +} +``` + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| project_id | The project ID to manage the Pub/Sub resources | string | - | yes | +| pull_subscriptions | The list of the pull subscriptions | list | `` | no | +| push_subscriptions | The list of the push subscriptions | list | `` | no | +| topic | The Pub/Sub topic name | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| subscriptions | The name list of Pub/Sub subscriptions | +| topic | The name of the Pub/Sub topic | + +[^]: (autogen_docs_end) + +## Requirements + +### Installation Dependencies + +- [terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.12.x + +### Configure a Service Account + +In order to execute this module you must have a Service Account with the following: + +#### Roles + +- `roles/pubsub.editor` + +### Enable APIs + +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Cloud Pub/Sub API + +## Testing + +### Requirements + +- [bundler](https://bundler.io/) +- [ruby](https://www.ruby-lang.org/) 2.5.x +- [python](https://www.python.org/getit/) 2.7.x +- [terraform-docs](https://github.com/segmentio/terraform-docs) 0.4.5 +- [google-cloud-sdk](https://cloud.google.com/sdk/) + +### Generate docs automatically + +```sh +$ make generate_docs +``` + +### Integration Test + +The integration tests for this module leverage kitchen-terraform and kitchen-inspec. + +You must set up by manually before running the integration test: + +- Copy from `test/integration/default/sample.sh` to `mine.sh`. +- Modify values having the `TF_VAR` prefix to match your environment. +- Perform `source ./mine.sh` to configure required environment variables. + +The tests will do the following: + +- Perform `bundle install` command + - Installs `test-kitchen`, `kitchen-terraform` and `kitchen-inspec` +- Perform `bundle exec kitchen create` command + - Performs `terraform init` +- Perform `bundle exec kitchen converge` command + - Performs `terraform apply -auto-approve` +- Perform `bundle exec kitchen verify` command + - Performs inspec tests +- Perform `bundle exec kitchen destroy` command + - Performs `terraform destroy -force` + +You can use the following command to run the integration test in the root directory. + +```sh +$ make test +``` + +## Linting + +The makefile in this project will lint or sometimes just format any shell, Python, golang, Terraform, or Dockerfiles. The linters will only be run if the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to run + +```sh +$ make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running terraform fmt +Running hadolint on Dockerfiles +Checking for required files +The following lines have trailing whitespace +Generating markdown docs with terraform-docs +``` + +The linters +are as follows: +- Shell - shellcheck. Can be found in homebrew +- Python - flake8. Can be installed with `pip install flake8` +- Golang - gofmt. gofmt comes with the standard golang installation. golang +-s a compiled language so there is no standard linter. +- Terraform - terraform has a built-in linter in the `terraform validate` command. +- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/examples/example.tf b/examples/example.tf new file mode 100644 index 0000000..223d633 --- /dev/null +++ b/examples/example.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_file_path" { + description = "Service account json auth path" +} + +variable "project_id" { + description = "The project id to run tests against" +} + +variable "topic_name" { + description = "The name for the topic" +} + +provider "google" { + version = "~> 1.19" + credentials = "${file("${var.credentials_file_path}")}" + region = "us-central1" +} + +module "terraform_google_cloud_pubsub" { + source = "../" + topic = "${var.topic_name}" + project_id = "${var.project_id}" + + pull_subscriptions = [ + { + name = "pull" + ack_deadline_seconds = 20 + }, + ] +} diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py new file mode 100644 index 0000000..b01af7c --- /dev/null +++ b/helpers/combine_docfiles.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Combine file from: + * script argument 1 + with content of file from: + * script argument 2 + using the beginning of line separators + hardcoded using regexes in this file: + + We exclude any text using the separate + regex specified here +''' + +import re +import sys + +insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' +exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' + +if len(sys.argv) != 3: + sys.exit(1) + +input = open(sys.argv[1], "r").read() +replace_content = open(sys.argv[2], "r").read() + +# Exclude the specified content from the replacement content +groups = re.match(exclude_separator_regex, replace_content, re.DOTALL).groups(0) +replace_content = groups[0] + groups[1] + +# Find where to put the replacement content, overwrite the input file +groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0) +output = groups[0] + replace_content + groups[2] +open(sys.argv[1], "w").write(output) diff --git a/kitchen.yml b/kitchen.yml new file mode 100644 index 0000000..e8b79ec --- /dev/null +++ b/kitchen.yml @@ -0,0 +1,33 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +driver: + name: terraform + root_module_directory: test/fixtures + +provisioner: + name: terraform + +transport: + name: exec + +verifier: + name: inspec + +platforms: + - name: local + +suites: + - name: default diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..ae33160 --- /dev/null +++ b/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + default_ack_deadline_seconds = 10 +} + +resource "google_pubsub_topic" "topic" { + name = "${var.topic}" + project = "${var.project_id}" +} + +resource "google_pubsub_subscription" "push_subscriptions" { + count = "${length(var.push_subscriptions)}" + name = "${lookup(var.push_subscriptions[count.index], "name")}" + topic = "${var.topic}" + project = "${var.project_id}" + ack_deadline_seconds = "${lookup(var.push_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" + + push_config { + push_endpoint = "${lookup(var.push_subscriptions[count.index], "push_endpoint")}" + } + + depends_on = ["google_pubsub_topic.topic"] +} + +resource "google_pubsub_subscription" "pull_subscriptions" { + count = "${length(var.pull_subscriptions)}" + name = "${lookup(var.pull_subscriptions[count.index], "name")}" + topic = "${var.topic}" + project = "${var.project_id}" + ack_deadline_seconds = "${lookup(var.pull_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" + + depends_on = ["google_pubsub_topic.topic"] +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..a8b3a2c --- /dev/null +++ b/outputs.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "topic" { + value = ["${google_pubsub_topic.topic.name}"] + description = "The name of the Pub/Sub topic" +} + +output "subscriptions" { + value = [ + "${google_pubsub_subscription.push_subscriptions.*.name}", + "${google_pubsub_subscription.pull_subscriptions.*.name}", + ] + + description = "The name list of Pub/Sub subscriptions" +} diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000..557e16f --- /dev/null +++ b/test/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.rb.txt b/test/boilerplate/boilerplate.rb.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.rb.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.tf.txt b/test/boilerplate/boilerplate.tf.txt new file mode 100644 index 0000000..cfccff8 --- /dev/null +++ b/test/boilerplate/boilerplate.tf.txt @@ -0,0 +1,15 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt new file mode 100644 index 0000000..3d98cdc --- /dev/null +++ b/test/boilerplate/boilerplate.xml.txt @@ -0,0 +1,15 @@ + diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.yaml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.yml.txt b/test/boilerplate/boilerplate.yml.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.yml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/fixtures/main.tf b/test/fixtures/main.tf new file mode 100644 index 0000000..db65123 --- /dev/null +++ b/test/fixtures/main.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + credentials = "${file(var.credentials_file_path)}" +} + +module "pubsub" { + source = "../../" + project_id = "${var.project}" + topic = "${var.topic_name}" + + push_subscriptions = [ + { + name = "push" + push_endpoint = "https://${var.project}.appspot.com/" + ack_deadline_seconds = 20 + }, + ] + + pull_subscriptions = [ + { + name = "pull" + }, + ] +} diff --git a/test/fixtures/outputs.tf b/test/fixtures/outputs.tf new file mode 100644 index 0000000..f93a6ba --- /dev/null +++ b/test/fixtures/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${var.project}" +} diff --git a/test/fixtures/variables.tf b/test/fixtures/variables.tf new file mode 100644 index 0000000..52172b0 --- /dev/null +++ b/test/fixtures/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_file_path" { + description = "Service account json auth path" +} + +variable "project" { + description = "The project to run tests against" +} + +variable "topic_name" { + description = "The project to run tests against" +} diff --git a/test/integration/default/controls/pubsub.rb b/test/integration/default/controls/pubsub.rb new file mode 100644 index 0000000..92f96d5 --- /dev/null +++ b/test/integration/default/controls/pubsub.rb @@ -0,0 +1,38 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id', default: ENV['TF_VAR_project']) +topic = attribute('topic_name', default: ENV['TF_VAR_topic_name']) + +describe command("gcloud --project='#{project_id}' pubsub topics describe #{topic}") do + its(:exit_status) { should be_zero } + it { expect(subject.stdout).to match(%r{name: projects/#{project_id}/topics/#{topic}}) } +end + +describe command("gcloud --project='#{project_id}' pubsub subscriptions describe pull --format=json") do + let(:stdout) { JSON.parse(subject.stdout, symbolize_names: true) } + its(:exit_status) { should be_zero } + it { expect(stdout).to include(name: "projects/#{project_id}/subscriptions/pull") } + it { expect(stdout).to include(topic: "projects/#{project_id}/topics/#{topic}") } + it { expect(stdout).to include(ackDeadlineSeconds: 10) } +end + +describe command("gcloud --project='#{project_id}' pubsub subscriptions describe push --format=json") do + let(:stdout) { JSON.parse(subject.stdout, symbolize_names: true) } + its(:exit_status) { should be_zero } + it { expect(stdout).to include(name: "projects/#{project_id}/subscriptions/push") } + it { expect(stdout).to include(topic: "projects/#{project_id}/topics/#{topic}") } + it { expect(stdout).to include(pushConfig: { pushEndpoint: "https://#{project_id}.appspot.com/" }) } + it { expect(stdout).to include(ackDeadlineSeconds: 20) } +end diff --git a/test/integration/default/inspec.yml b/test/integration/default/inspec.yml new file mode 100644 index 0000000..92aac35 --- /dev/null +++ b/test/integration/default/inspec.yml @@ -0,0 +1,21 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: cloud-pubsub +title: Google Cloud Pub/Sub +version: 0.1.0 +inspec_version: '~> 2.1.0' +depends: + - name: inspec-terraform + git: https://github.com/lagrange-automation/inspec-terraform diff --git a/test/integration/default/sample.sh b/test/integration/default/sample.sh new file mode 100644 index 0000000..8f385d3 --- /dev/null +++ b/test/integration/default/sample.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# PLEASE FILL THE VARIABLES WITH VALID VALUES FOR TESTING # +# DO NOT REMOVE ANY OF THE VARIABLES # +################################################################# + +## These values you *MUST* modify to match your environment +export TF_VAR_credentials_file_path="test/integration/default/sa-key.json" +export TF_VAR_project="tf-proj" +export TF_VAR_topic_name="$TF_VAR_project-topic" diff --git a/test/make.sh b/test/make.sh new file mode 100755 index 0000000..8bdbfe4 --- /dev/null +++ b/test/make.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function checks to make sure that every +# shebang has a '- e' flag, which causes it +# to exit on error +function check_bash() { +find . -name "*.sh" | while IFS= read -d '' -r file; +do + if [[ "$file" != *"bash -e"* ]]; + then + echo "$file is missing shebang with -e"; + exit 1; + fi; +done; +} + +# This function makes sure that the required files for +# releasing to OSS are present +function basefiles() { + echo "Checking for required files" + test -f LICENSE || echo "Missing LICENSE" + test -f README.md || echo "Missing README.md" +} + +# This function runs the hadolint linter on +# every file named 'Dockerfile' +function docker() { + echo "Running hadolint on Dockerfiles" + find . -name "Dockerfile" -exec hadolint {} \; +} + +# This function runs 'terraform validate' against all +# files ending in '.tf' +function check_terraform() { + echo "Running terraform validate" + #shellcheck disable=SC2156 + find . -name "*.tf" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; + echo "Running terraform fmt" + terraform fmt -check=true -write=false +} + +# This function runs 'go fmt' and 'go vet' on every file +# that ends in '.go' +function golang() { + echo "Running go fmt and go vet" + find . -name "*.go" -exec go fmt {} \; + find . -name "*.go" -exec go vet {} \; +} + +# This function runs the flake8 linter on every file +# ending in '.py' +function check_python() { + echo "Running flake8" + find . -name "*.py" -exec flake8 {} \; +} + +# This function runs the shellcheck linter on every +# file ending in '.sh' +function check_shell() { + echo "Running shellcheck" + find . -name "*.sh" -exec shellcheck -x {} \; +} + +# This function makes sure that there is no trailing whitespace +# in any files in the project. +# There are some exclusions +function check_trailing_whitespace() { + echo "The following lines have trailing whitespace" + grep -r '[[:blank:]]$' --exclude-dir=".kitchen" --exclude-dir=".terraform" --exclude="*.png" --exclude="*.pyc" --exclude-dir=".git" . + rc=$? + if [ $rc = 0 ]; then + exit 1 + fi +} + +function generate_docs() { + echo "Generating markdown docs with terraform-docs" + TMPFILE=$(mktemp) + for j in $(for i in $(find . -type f | grep \.tf$ | grep -v "^./\(test/fixtures\|examples\)") ; do dirname "$i" ; done | sort -u) ; do + terraform-docs markdown "$j" > "$TMPFILE" + python helpers/combine_docfiles.py "$j"/README.md "$TMPFILE" + done + rm -f "$TMPFILE" +} diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 0000000..5af38d0 --- /dev/null +++ b/test/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +bundle exec kitchen create +bundle exec kitchen converge +bundle exec kitchen verify +bundle exec kitchen destroy diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py new file mode 100755 index 0000000..60eca23 --- /dev/null +++ b/test/test_verify_boilerplate.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' A simple test for the verify_boilerplate python script. +This will create a set of test files, both valid and invalid, +and confirm that the has_valid_header call returns the correct +value. +It also checks the number of files that are found by the +get_files call. +''' +from copy import deepcopy +from tempfile import mkdtemp +from shutil import rmtree +import unittest +from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ + get_args, get_files + + +class AllTestCase(unittest.TestCase): + """ + All of the setup, teardown, and tests are contained in this + class. + """ + + def write_file(self, filename, content, expected): + """ + A utility method that creates test files, and adds them to + the cases that will be tested. + Args: + filename: (string) the file name (path) to be created. + content: (list of strings) the contents of the file. + expected: (boolean) True if the header is expected to be valid, + false if not. + """ + + file = open(filename, 'w+') + for line in content: + file.write(line + "\n") + file.close() + self.cases[filename] = expected + + def create_test_files(self, tmp_path, extension, header): + """ + Creates 2 test files for .tf, .xml, .go, etc and one for + Dockerfile, and Makefile. + The reason for the difference is that Makefile and Dockerfile + don't have an extension. These would be substantially more + difficult to create negative test cases, unless the files + were written, deleted, and re-written. + Args: + tmp_path: (string) the path in which to create the files + extension: (string) the file extension + header: (list of strings) the header/boilerplate content + """ + + content = "\n...blah \ncould be code or could be garbage\n" + special_cases = ["Dockerfile", "Makefile"] + header_template = deepcopy(header) + valid_filename = tmp_path + extension + valid_content = header_template.append(content) + if extension not in special_cases: + # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) + invalid_header = [] + for line in header_template: + if "2018" in line: + invalid_header.append(line.replace('2018', 'YEAR')) + else: + invalid_header.append(line) + invalid_header.append(content) + invalid_content = invalid_header + invalid_filename = tmp_path + "invalid." + extension + self.write_file(invalid_filename, invalid_content, False) + valid_filename = tmp_path + "testfile." + extension + + valid_content = header_template + self.write_file(valid_filename, valid_content, True) + + def setUp(self): + """ + Set initial counts and values, and initializes the setup of the + test files. + """ + self.cases = {} + self.tmp_path = mkdtemp() + "/" + self.my_args = get_args() + self.my_refs = get_refs(self.my_args) + self.my_regex = get_regexs() + self.prexisting_file_count = len( + get_files(self.my_refs.keys(), self.my_args)) + for key in self.my_refs: + self.create_test_files(self.tmp_path, key, + self.my_refs.get(key)) + + def tearDown(self): + """ Delete the test directory. """ + rmtree(self.tmp_path) + + def test_files_headers(self): + """ + Confirms that the expected output of has_valid_header is correct. + """ + for case in self.cases: + if self.cases[case]: + self.assertTrue(has_valid_header(case, self.my_refs, + self.my_regex)) + else: + self.assertFalse(has_valid_header(case, self.my_refs, + self.my_regex)) + + def test_invalid_count(self): + """ + Test that the initial files found isn't zero, indicating + a problem with the code. + """ + self.assertFalse(self.prexisting_file_count == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py new file mode 100755 index 0000000..a632fde --- /dev/null +++ b/test/verify_boilerplate.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Verifies that all source files contain the necessary copyright boilerplate +# snippet. +# This is based on existing work +# https://github.com/kubernetes/test-infra/blob/master/hack +# /verify_boilerplate.py +from __future__ import print_function +import argparse +import glob +import os +import re +import sys + + +def get_args(): + """Parses command line arguments. + + Configures and runs argparse.ArgumentParser to extract command line + arguments. + + Returns: + An argparse.Namespace containing the arguments parsed from the + command line + """ + parser = argparse.ArgumentParser() + parser.add_argument("filenames", + help="list of files to check, " + "all files if unspecified", + nargs='*') + rootdir = os.path.dirname(__file__) + "/../" + rootdir = os.path.abspath(rootdir) + parser.add_argument( + "--rootdir", + default=rootdir, + help="root directory to examine") + + default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") + parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) + return parser.parse_args() + + +def get_refs(ARGS): + """Converts the directory of boilerplate files into a map keyed by file + extension. + + Reads each boilerplate file's contents into an array, then adds that array + to a map keyed by the file extension. + + Returns: + A map of boilerplate lines, keyed by file extension. For example, + boilerplate.py.txt would result in the k,v pair {".py": py_lines} where + py_lines is an array containing each line of the file. + """ + refs = {} + + # Find and iterate over the absolute path for each boilerplate template + for path in glob.glob(os.path.join( + ARGS.boilerplate_dir, + "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + return refs + + +# pylint: disable=too-many-locals +def has_valid_header(filename, refs, regexs): + """Test whether a file has the correct boilerplate header. + + Tests each file against the boilerplate stored in refs for that file type + (based on extension), or by the entire filename (eg Dockerfile, Makefile). + Some heuristics are applied to remove build tags and shebangs, but little + variance in header formatting is tolerated. + + Args: + filename: A string containing the name of the file to test + refs: A map of boilerplate headers, keyed by file extension + regexs: a map of compiled regex objects used in verifying boilerplate + + Returns: + True if the file has the correct boilerplate header, otherwise returns + False. + """ + try: + with open(filename, 'r') as fp: # pylint: disable=invalid-name + data = fp.read() + except IOError: + return False + basename = os.path.basename(filename) + extension = get_file_extension(filename) + if extension: + ref = refs[extension] + else: + ref = refs[basename] + # remove build tags from the top of Go files + if extension == "go": + con = regexs["go_build_constraints"] + (data, found) = con.subn("", data, 1) + # remove shebang + elif extension == "sh" or extension == "py": + she = regexs["shebang"] + (data, found) = she.subn("", data, 1) + data = data.splitlines() + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + return False + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + year = regexs["year"] + for datum in data: + if year.search(datum): + return False + + # if we don't match the reference at this point, fail + if ref != data: + return False + return True + + +def get_file_extension(filename): + """Extracts the extension part of a filename. + + Identifies the extension as everything after the last period in filename. + + Args: + filename: string containing the filename + + Returns: + A string containing the extension in lowercase + """ + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# These directories will be omitted from header checks +SKIPPED_DIRS = [ + 'Godeps', 'third_party', '_gopath', '_output', + '.git', 'vendor', '__init__.py', 'node_modules' +] + + +def normalize_files(files): + """Extracts the files that require boilerplate checking from the files + argument. + + A new list will be built. Each path from the original files argument will + be added unless it is within one of SKIPPED_DIRS. All relative paths will + be converted to absolute paths by prepending the root_dir path parsed from + the command line, or its default value. + + Args: + files: a list of file path strings + + Returns: + A modified copy of the files list where any any path in a skipped + directory is removed, and all paths have been made absolute. + """ + newfiles = [] + for pathname in files: + if any(x in pathname for x in SKIPPED_DIRS): + continue + newfiles.append(pathname) + for idx, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[idx] = os.path.join(ARGS.rootdir, pathname) + return newfiles + + +def get_files(extensions, ARGS): + """Generates a list of paths whose boilerplate should be verified. + + If a list of file names has been provided on the command line, it will be + treated as the initial set to search. Otherwise, all paths within rootdir + will be discovered and used as the initial set. + + Once the initial set of files is identified, it is normalized via + normalize_files() and further stripped of any file name whose extension is + not in extensions. + + Args: + extensions: a list of file extensions indicating which file types + should have their boilerplate verified + + Returns: + A list of absolute file paths + """ + files = [] + if ARGS.filenames: + files = ARGS.filenames + else: + for root, dirs, walkfiles in os.walk(ARGS.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for dpath in SKIPPED_DIRS: + if dpath in dirs: + dirs.remove(dpath) + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = get_file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + """Builds a map of regular expressions used in boilerplate validation. + + There are two scenarios where these regexes are used. The first is in + validating the date referenced is the boilerplate, by ensuring it is an + acceptable year. The second is in identifying non-boilerplate elements, + like shebangs and compiler hints that should be ignored when validating + headers. + + Returns: + A map of compiled regular expression objects, keyed by mnemonic. + """ + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the + # real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016 or 2017, company holder names can be + # anything + regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", + re.MULTILINE) + # strip #!.* from shell/python scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(args): + """Identifies and verifies files that should have the desired boilerplate. + + Retrieves the lists of files to be validated and tests each one in turn. + If all files contain correct boilerplate, this function terminates + normally. Otherwise it prints the name of each non-conforming file and + exists with a non-zero status code. + """ + regexs = get_regexs() + refs = get_refs(args) + filenames = get_files(refs.keys(), args) + nonconforming_files = [] + for filename in filenames: + if not has_valid_header(filename, refs, regexs): + nonconforming_files.append(filename) + if nonconforming_files: + print('%d files have incorrect boilerplate headers:' % len( + nonconforming_files)) + for filename in sorted(nonconforming_files): + print(os.path.relpath(filename, args.rootdir)) + sys.exit(1) + + +if __name__ == "__main__": + ARGS = get_args() + main(ARGS) diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..86caf7c --- /dev/null +++ b/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to manage the Pub/Sub resources" +} + +variable "topic" { + description = "The Pub/Sub topic name" +} + +variable "push_subscriptions" { + type = "list" + description = "The list of the push subscriptions" + default = [] +} + +variable "pull_subscriptions" { + type = "list" + description = "The list of the pull subscriptions" + default = [] +} From 697d09fa409981101455ad866571dec5826eaceb Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 00:46:33 +0900 Subject: [PATCH 02/17] bump kitchen-terraform and inspec versions to newer --- Gemfile | 4 +- Gemfile.lock | 94 +++++++++++++++++++++-------- test/integration/default/inspec.yml | 2 +- 3 files changed, 72 insertions(+), 28 deletions(-) diff --git a/Gemfile b/Gemfile index e27ae50..81f83c6 100644 --- a/Gemfile +++ b/Gemfile @@ -15,5 +15,5 @@ source 'https://rubygems.org/' gem 'test-kitchen' -gem 'kitchen-terraform', '~> 3.3' -gem 'kitchen-inspec', git: 'https://github.com/inspec/kitchen-inspec.git', ref: '0590f1b' +gem 'kitchen-terraform', '~> 4.0.3' +gem 'kitchen-inspec' diff --git a/Gemfile.lock b/Gemfile.lock index 8bdeff3..6cd1f0d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,31 +1,27 @@ -GIT - remote: https://github.com/inspec/kitchen-inspec.git - revision: 0590f1bbc99d3a28c31ca8841c5d6cb4139ab773 - ref: 0590f1b - specs: - kitchen-inspec (0.23.1) - hashie (~> 3.4) - inspec (>= 0.34.0, < 3.0.0) - test-kitchen (~> 1.6) - GEM remote: https://rubygems.org/ specs: addressable (2.5.2) public_suffix (>= 2.0.2, < 4.0) - aws-sdk (2.11.154) - aws-sdk-resources (= 2.11.154) - aws-sdk-core (2.11.154) + aws-sdk (2.11.156) + aws-sdk-resources (= 2.11.156) + aws-sdk-core (2.11.156) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sdk-resources (2.11.154) - aws-sdk-core (= 2.11.154) + aws-sdk-resources (2.11.156) + aws-sdk-core (= 2.11.156) aws-sigv4 (1.0.3) + azure_graph_rbac (0.17.0) + ms_rest_azure (~> 0.11.0) + azure_mgmt_key_vault (0.17.2) + ms_rest_azure (~> 0.11.0) azure_mgmt_resources (0.17.2) ms_rest_azure (~> 0.11.0) builder (3.2.3) coderay (1.1.2) concurrent-ruby (1.0.5) + declarative (0.0.10) + declarative-option (0.1.0) diff-lcs (1.3) docker-api (1.34.2) excon (>= 0.47.0) @@ -66,7 +62,24 @@ GEM faraday-cookie_jar (0.0.6) faraday (>= 0.7.4) http-cookie (~> 1.0.0) + faraday_middleware (0.12.2) + faraday (>= 0.7.4, < 1.0) ffi (1.9.25) + google-api-client (0.23.9) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.5, < 0.7.0) + httpclient (>= 2.8.1, < 3.0) + mime-types (~> 3.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.9) + googleauth (0.6.7) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) gssapi (1.2.0) ffi (>= 1.0.1) gyoku (1.3.1) @@ -77,38 +90,50 @@ GEM domain_name (~> 0.5) httpclient (2.8.3) inifile (3.0.0) - inspec (2.1.72) + inspec (2.3.24) addressable (~> 2.4) faraday (>= 0.9.0) + faraday_middleware (~> 0.12.2) hashie (~> 3.4) htmlentities json (>= 1.8, < 3.0) method_source (~> 0.8) mixlib-log + multipart-post parallel (~> 1.9) parslet (~> 1.5) pry (~> 0) rspec (~> 3) rspec-its (~> 1.2) - rubyzip (~> 1.1) + rubyzip (~> 1.2, >= 1.2.2) semverse sslshake (~> 1.2) + term-ansicolor thor (~> 0.20) tomlrb (~> 1.2) - train (~> 1.4) + train (~> 1.5) jmespath (1.4.0) json (2.1.0) - kitchen-terraform (3.3.1) + jwt (2.1.0) + kitchen-inspec (0.25.0) + hashie (~> 3.4) + inspec (>= 0.34.0, < 4.0.0) + test-kitchen (~> 1.6) + kitchen-terraform (4.0.3) dry-types (~> 0.9) dry-validation (~> 0.10) - kitchen-inspec (~> 0.18) + inspec (~> 2.2, != 2.3.5, != 2.3.4, != 2.2.112, != 2.2.102, != 2.2.101) mixlib-shellout (~> 2.2) - test-kitchen (~> 1.16) + test-kitchen (~> 1.23) little-plugger (1.1.4) logging (2.2.2) little-plugger (~> 1.1) multi_json (~> 1.10) + memoist (0.16.0) method_source (0.9.0) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) mixlib-install (3.11.5) mixlib-shellout mixlib-versioning @@ -133,12 +158,18 @@ GEM net-ssh-gateway (1.3.0) net-ssh (>= 2.6.5) nori (2.6.0) + os (1.0.0) parallel (1.12.1) parslet (1.8.2) pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) public_suffix (3.0.3) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) + retriable (3.1.2) rspec (3.8.0) rspec-core (~> 3.8.0) rspec-expectations (~> 3.8.0) @@ -158,7 +189,14 @@ GEM rubyntlm (0.6.2) rubyzip (1.2.2) semverse (2.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) sslshake (1.2.0) + term-ansicolor (1.6.0) + tins (~> 1.0) test-kitchen (1.23.2) mixlib-install (~> 3.6) mixlib-shellout (>= 1.2, < 3.0) @@ -171,18 +209,24 @@ GEM winrm-fs (~> 1.1) thor (0.20.0) timeliness (0.3.8) + tins (1.17.0) tomlrb (1.2.7) - train (1.4.4) + train (1.5.4) aws-sdk (~> 2) + azure_graph_rbac (~> 0.16) + azure_mgmt_key_vault (~> 0.17) azure_mgmt_resources (~> 0.15) docker-api (~> 1.26) + google-api-client (~> 0.23.9) + googleauth (~> 0.6.6) inifile json (>= 1.8, < 3.0) mixlib-shellout (~> 2.0) net-scp (~> 1.2) - net-ssh (>= 2.9, < 5.0) + net-ssh (>= 2.9, < 6.0) winrm (~> 2.0) winrm-fs (~> 1.0) + uber (0.1.0) unf (0.1.4) unf_ext unf_ext (0.0.7.5) @@ -208,8 +252,8 @@ PLATFORMS ruby DEPENDENCIES - kitchen-inspec! - kitchen-terraform (~> 3.3) + kitchen-inspec + kitchen-terraform (~> 4.0.3) test-kitchen BUNDLED WITH diff --git a/test/integration/default/inspec.yml b/test/integration/default/inspec.yml index 92aac35..9c457f8 100644 --- a/test/integration/default/inspec.yml +++ b/test/integration/default/inspec.yml @@ -15,7 +15,7 @@ name: cloud-pubsub title: Google Cloud Pub/Sub version: 0.1.0 -inspec_version: '~> 2.1.0' +inspec_version: '~> 2.3.24' depends: - name: inspec-terraform git: https://github.com/lagrange-automation/inspec-terraform From 6aa8337c35e5769c42255ba0a3a50268f1baa57a Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 00:47:04 +0900 Subject: [PATCH 03/17] add bundle installation in test integration --- test/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.sh b/test/test.sh index 5af38d0..0fcb160 100755 --- a/test/test.sh +++ b/test/test.sh @@ -16,6 +16,7 @@ set -e +bundle install bundle exec kitchen create bundle exec kitchen converge bundle exec kitchen verify From a7048e86b7929456fefcfea875670bfa9146adfa Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 00:48:21 +0900 Subject: [PATCH 04/17] rename testing task from test to test_integration --- Makefile | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index df5f9cf..b9b1fda 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,6 @@ check_headers: generate_docs: @source test/make.sh && generate_docs -.PHONY: test -test: +.PHONY: test_integration +test_integration: @source test/test.sh diff --git a/README.md b/README.md index 3f4218b..1717749 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The tests will do the following: You can use the following command to run the integration test in the root directory. ```sh -$ make test +$ make test_integration ``` ## Linting @@ -144,4 +144,4 @@ are as follows: - Golang - gofmt. gofmt comes with the standard golang installation. golang -s a compiled language so there is no standard linter. - Terraform - terraform has a built-in linter in the `terraform validate` command. -- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file +- Dockerfiles - hadolint. Can be found in homebrew From c808032ba6ad5046407c771113555c9feec95007 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 02:02:46 +0900 Subject: [PATCH 05/17] use string instead of list --- outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs.tf b/outputs.tf index a8b3a2c..ecd5ac9 100644 --- a/outputs.tf +++ b/outputs.tf @@ -15,7 +15,7 @@ */ output "topic" { - value = ["${google_pubsub_topic.topic.name}"] + value = "${google_pubsub_topic.topic.name}" description = "The name of the Pub/Sub topic" } From 51b9a5dc130e4d6287cf1bcaae5b09020f386fd6 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 02:03:12 +0900 Subject: [PATCH 06/17] remove unnecessary inspec-terraform --- test/integration/default/inspec.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/default/inspec.yml b/test/integration/default/inspec.yml index 9c457f8..7c9278c 100644 --- a/test/integration/default/inspec.yml +++ b/test/integration/default/inspec.yml @@ -16,6 +16,3 @@ name: cloud-pubsub title: Google Cloud Pub/Sub version: 0.1.0 inspec_version: '~> 2.3.24' -depends: - - name: inspec-terraform - git: https://github.com/lagrange-automation/inspec-terraform From f92b9f85edc6b7b123e8954df5e8d4c285502968 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 02:03:53 +0900 Subject: [PATCH 07/17] update .kitchen.yml to use terraform verifier --- README.md | 2 +- kitchen.yml | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1717749..1de90df 100644 --- a/README.md +++ b/README.md @@ -144,4 +144,4 @@ are as follows: - Golang - gofmt. gofmt comes with the standard golang installation. golang -s a compiled language so there is no standard linter. - Terraform - terraform has a built-in linter in the `terraform validate` command. -- Dockerfiles - hadolint. Can be found in homebrew +- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/kitchen.yml b/kitchen.yml index e8b79ec..17d9bd2 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -13,21 +13,25 @@ # limitations under the License. --- -driver: - name: terraform - root_module_directory: test/fixtures provisioner: name: terraform transport: name: exec + connection_timeout: 60 verifier: - name: inspec + name: terraform + systems: + - name: system + backend: local platforms: - name: local suites: - name: default + driver: + name: terraform + root_module_directory: test/fixtures From a50785d14370fb1bb2576edb65040647f6c772b2 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Thu, 25 Oct 2018 16:12:13 +0900 Subject: [PATCH 08/17] switch configuration values from environment variables to tfvars --- README.md | 7 +++---- test/fixtures/outputs.tf | 4 ++++ .../sample.sh => fixtures/terraform.tfvars.sample} | 7 +++---- test/integration/default/controls/pubsub.rb | 4 ++-- test/integration/default/inspec.yml | 10 ++++++++++ 5 files changed, 22 insertions(+), 10 deletions(-) rename test/{integration/default/sample.sh => fixtures/terraform.tfvars.sample} (84%) diff --git a/README.md b/README.md index 1de90df..7a3ff54 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,8 @@ The integration tests for this module leverage kitchen-terraform and kitchen-ins You must set up by manually before running the integration test: -- Copy from `test/integration/default/sample.sh` to `mine.sh`. -- Modify values having the `TF_VAR` prefix to match your environment. -- Perform `source ./mine.sh` to configure required environment variables. +- Copy from `test/fixtures/terraform.tfvars.sample` to `test/fixtures/terraform.tfvars`. +- Modify values to match your environment. The tests will do the following: @@ -144,4 +143,4 @@ are as follows: - Golang - gofmt. gofmt comes with the standard golang installation. golang -s a compiled language so there is no standard linter. - Terraform - terraform has a built-in linter in the `terraform validate` command. -- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file +- Dockerfiles - hadolint. Can be found in homebrew diff --git a/test/fixtures/outputs.tf b/test/fixtures/outputs.tf index f93a6ba..b11382b 100644 --- a/test/fixtures/outputs.tf +++ b/test/fixtures/outputs.tf @@ -17,3 +17,7 @@ output "project_id" { value = "${var.project}" } + +output "topic_name" { + value = "${var.topic_name}" +} diff --git a/test/integration/default/sample.sh b/test/fixtures/terraform.tfvars.sample similarity index 84% rename from test/integration/default/sample.sh rename to test/fixtures/terraform.tfvars.sample index 8f385d3..540c233 100644 --- a/test/integration/default/sample.sh +++ b/test/fixtures/terraform.tfvars.sample @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +18,6 @@ ################################################################# ## These values you *MUST* modify to match your environment -export TF_VAR_credentials_file_path="test/integration/default/sa-key.json" -export TF_VAR_project="tf-proj" -export TF_VAR_topic_name="$TF_VAR_project-topic" +credentials_file_path="../integration/default/sa-key.json" +project="tf-proj" +topic_name="tf-proj-topic" diff --git a/test/integration/default/controls/pubsub.rb b/test/integration/default/controls/pubsub.rb index 92f96d5..a675e1e 100644 --- a/test/integration/default/controls/pubsub.rb +++ b/test/integration/default/controls/pubsub.rb @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -project_id = attribute('project_id', default: ENV['TF_VAR_project']) -topic = attribute('topic_name', default: ENV['TF_VAR_topic_name']) +project_id = attribute('project_id') +topic = attribute('topic_name') describe command("gcloud --project='#{project_id}' pubsub topics describe #{topic}") do its(:exit_status) { should be_zero } diff --git a/test/integration/default/inspec.yml b/test/integration/default/inspec.yml index 7c9278c..2df7ed1 100644 --- a/test/integration/default/inspec.yml +++ b/test/integration/default/inspec.yml @@ -16,3 +16,13 @@ name: cloud-pubsub title: Google Cloud Pub/Sub version: 0.1.0 inspec_version: '~> 2.3.24' +attributes: + - name: project_id + type: string + required: true + - name: topic_name + type: string + required: true + - name: credentials_file_path + type: string + required: true From 0e7a5683bb14f22c05479e4bb4ccd4b0a35984d1 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 08:19:25 +0900 Subject: [PATCH 09/17] move example.tf to examples/simple --- examples/{ => simple}/example.tf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{ => simple}/example.tf (100%) diff --git a/examples/example.tf b/examples/simple/example.tf similarity index 100% rename from examples/example.tf rename to examples/simple/example.tf From d8cd1f7c5409edceabe3de98168f75691affce2f Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 08:20:10 +0900 Subject: [PATCH 10/17] move variables from examples.tf to variables.tf --- examples/simple/example.tf | 16 ++-------------- examples/simple/variables.tf | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 examples/simple/variables.tf diff --git a/examples/simple/example.tf b/examples/simple/example.tf index 223d633..8fcd6ba 100644 --- a/examples/simple/example.tf +++ b/examples/simple/example.tf @@ -14,26 +14,14 @@ * limitations under the License. */ -variable "credentials_file_path" { - description = "Service account json auth path" -} - -variable "project_id" { - description = "The project id to run tests against" -} - -variable "topic_name" { - description = "The name for the topic" -} - provider "google" { version = "~> 1.19" credentials = "${file("${var.credentials_file_path}")}" region = "us-central1" } -module "terraform_google_cloud_pubsub" { - source = "../" +module "pubsub" { + source = "../../" topic = "${var.topic_name}" project_id = "${var.project_id}" diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf new file mode 100644 index 0000000..2796070 --- /dev/null +++ b/examples/simple/variables.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project id to run tests against" +} + +variable "topic_name" { + description = "The name for the topic" +} From ef40bf47f0fa462ed1ea0c053f3744cb034e8844 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 08:30:21 +0900 Subject: [PATCH 11/17] drop credentials from example, add a note in README --- README.md | 10 ++++++++++ examples/simple/example.tf | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a3ff54..38d2880 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,16 @@ In order to operate with the Service Account you must activate the following API - Cloud Pub/Sub API +#### Service Account Credentials + +You can pass the service account credentials into this module by setting the following environment variables: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + +See more [details](https://www.terraform.io/docs/providers/google/provider_reference.html#configuration-reference). + ## Testing ### Requirements diff --git a/examples/simple/example.tf b/examples/simple/example.tf index 8fcd6ba..5351d61 100644 --- a/examples/simple/example.tf +++ b/examples/simple/example.tf @@ -15,9 +15,8 @@ */ provider "google" { - version = "~> 1.19" - credentials = "${file("${var.credentials_file_path}")}" - region = "us-central1" + version = "~> 1.19" + region = "us-central1" } module "pubsub" { From 16171c6d6773c1caf3bb510b7646c93d7ffda28d Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 08:43:34 +0900 Subject: [PATCH 12/17] use google_pubsub_topic.topic.name instead of var.topic --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index ae33160..dac31e6 100644 --- a/main.tf +++ b/main.tf @@ -26,7 +26,7 @@ resource "google_pubsub_topic" "topic" { resource "google_pubsub_subscription" "push_subscriptions" { count = "${length(var.push_subscriptions)}" name = "${lookup(var.push_subscriptions[count.index], "name")}" - topic = "${var.topic}" + topic = "${google_pubsub_topic.topic.name}" project = "${var.project_id}" ack_deadline_seconds = "${lookup(var.push_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" @@ -40,7 +40,7 @@ resource "google_pubsub_subscription" "push_subscriptions" { resource "google_pubsub_subscription" "pull_subscriptions" { count = "${length(var.pull_subscriptions)}" name = "${lookup(var.pull_subscriptions[count.index], "name")}" - topic = "${var.topic}" + topic = "${google_pubsub_topic.topic.name}" project = "${var.project_id}" ack_deadline_seconds = "${lookup(var.pull_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" From 4cbcc5e878c1d184672bea6bfb73e4b7f2831737 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 08:44:35 +0900 Subject: [PATCH 13/17] expose subscription paths, use the concat func explicitly --- outputs.tf | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/outputs.tf b/outputs.tf index ecd5ac9..c41ad88 100644 --- a/outputs.tf +++ b/outputs.tf @@ -19,11 +19,20 @@ output "topic" { description = "The name of the Pub/Sub topic" } -output "subscriptions" { - value = [ - "${google_pubsub_subscription.push_subscriptions.*.name}", - "${google_pubsub_subscription.pull_subscriptions.*.name}", - ] +output "subscription_names" { + value = "${concat( + google_pubsub_subscription.push_subscriptions.*.name, + google_pubsub_subscription.pull_subscriptions.*.name + )}" description = "The name list of Pub/Sub subscriptions" } + +output "subscription_paths" { + value = "${concat( + google_pubsub_subscription.push_subscriptions.*.path, + google_pubsub_subscription.pull_subscriptions.*.path + )}" + + description = "The path list of Pub/Sub subscriptions" +} From 067b503014d7ca3a97a84e8aaa7089416cb01b42 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 09:17:08 +0900 Subject: [PATCH 14/17] support attributes argument --- README.md | 1 + examples/simple/example.tf | 8 ++++++++ main.tf | 5 +++++ test/fixtures/main.tf | 1 + 4 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 38d2880..9d25aec 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ module "pubsub" { name = "push" // required ack_deadline_seconds = 20 // optonal push_endpoint = "https://example.com" + attributes = "x-goog-version:v1beta1" } ] pull_subscriptions = [ diff --git a/examples/simple/example.tf b/examples/simple/example.tf index 5351d61..c12faa8 100644 --- a/examples/simple/example.tf +++ b/examples/simple/example.tf @@ -30,4 +30,12 @@ module "pubsub" { ack_deadline_seconds = 20 }, ] + push_subscriptions = [ + { + name = "push" + ack_deadline_seconds = 20 + push_endpoint = "https://${var.project_id}.appspot.com" + attributes = "x-goog-version:v1beta1" + }, + ] } diff --git a/main.tf b/main.tf index dac31e6..f2be37e 100644 --- a/main.tf +++ b/main.tf @@ -32,6 +32,11 @@ resource "google_pubsub_subscription" "push_subscriptions" { push_config { push_endpoint = "${lookup(var.push_subscriptions[count.index], "push_endpoint")}" + // FIXME: This should be programmable, but nested map isn't supported at this time. + // https://github.com/hashicorp/terraform/issues/2114 + attributes = { + x-goog-version = "${element(split(":", lookup(var.push_subscriptions[count.index], "attributes", "x-goog-version:v1")), 1)}" + } } depends_on = ["google_pubsub_topic.topic"] diff --git a/test/fixtures/main.tf b/test/fixtures/main.tf index db65123..e3cc015 100644 --- a/test/fixtures/main.tf +++ b/test/fixtures/main.tf @@ -27,6 +27,7 @@ module "pubsub" { { name = "push" push_endpoint = "https://${var.project}.appspot.com/" + attributes = "x-goog-version:v1beta1" ack_deadline_seconds = 20 }, ] From 442ebbe81a136b4ff14bc9c6a022348aa6071f6c Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 09:24:33 +0900 Subject: [PATCH 15/17] make -s --- README.md | 5 +++-- examples/simple/example.tf | 1 + main.tf | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d25aec..87cd003 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ module "pubsub" { | Name | Description | |------|-------------| -| subscriptions | The name list of Pub/Sub subscriptions | +| subscription_names | The name list of Pub/Sub subscriptions | +| subscription_paths | The path list of Pub/Sub subscriptions | | topic | The name of the Pub/Sub topic | [^]: (autogen_docs_end) @@ -154,4 +155,4 @@ are as follows: - Golang - gofmt. gofmt comes with the standard golang installation. golang -s a compiled language so there is no standard linter. - Terraform - terraform has a built-in linter in the `terraform validate` command. -- Dockerfiles - hadolint. Can be found in homebrew +- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/examples/simple/example.tf b/examples/simple/example.tf index c12faa8..f533972 100644 --- a/examples/simple/example.tf +++ b/examples/simple/example.tf @@ -30,6 +30,7 @@ module "pubsub" { ack_deadline_seconds = 20 }, ] + push_subscriptions = [ { name = "push" diff --git a/main.tf b/main.tf index f2be37e..bca7ee1 100644 --- a/main.tf +++ b/main.tf @@ -32,6 +32,7 @@ resource "google_pubsub_subscription" "push_subscriptions" { push_config { push_endpoint = "${lookup(var.push_subscriptions[count.index], "push_endpoint")}" + // FIXME: This should be programmable, but nested map isn't supported at this time. // https://github.com/hashicorp/terraform/issues/2114 attributes = { From 039a43c453f20b398ff64949d64ad03d447431e3 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 10:01:55 +0900 Subject: [PATCH 16/17] expose x-goog-version instead of attributes argument --- README.md | 2 +- examples/simple/example.tf | 2 +- main.tf | 2 +- test/fixtures/main.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 87cd003..2d6b2f5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ module "pubsub" { name = "push" // required ack_deadline_seconds = 20 // optonal push_endpoint = "https://example.com" - attributes = "x-goog-version:v1beta1" + x-goog-version = "v1beta1" } ] pull_subscriptions = [ diff --git a/examples/simple/example.tf b/examples/simple/example.tf index f533972..c3475d1 100644 --- a/examples/simple/example.tf +++ b/examples/simple/example.tf @@ -36,7 +36,7 @@ module "pubsub" { name = "push" ack_deadline_seconds = 20 push_endpoint = "https://${var.project_id}.appspot.com" - attributes = "x-goog-version:v1beta1" + x-goog-version = "v1beta1" }, ] } diff --git a/main.tf b/main.tf index bca7ee1..7b2f878 100644 --- a/main.tf +++ b/main.tf @@ -36,7 +36,7 @@ resource "google_pubsub_subscription" "push_subscriptions" { // FIXME: This should be programmable, but nested map isn't supported at this time. // https://github.com/hashicorp/terraform/issues/2114 attributes = { - x-goog-version = "${element(split(":", lookup(var.push_subscriptions[count.index], "attributes", "x-goog-version:v1")), 1)}" + x-goog-version = "${lookup(var.push_subscriptions[count.index], "x-goog-version", "v1")}" } } diff --git a/test/fixtures/main.tf b/test/fixtures/main.tf index e3cc015..1b3b3ca 100644 --- a/test/fixtures/main.tf +++ b/test/fixtures/main.tf @@ -27,7 +27,7 @@ module "pubsub" { { name = "push" push_endpoint = "https://${var.project}.appspot.com/" - attributes = "x-goog-version:v1beta1" + x-goog-version = "v1beta1" ack_deadline_seconds = 20 }, ] From 861ba0df788055bca5b584a5eaa8bab8140ef415 Mon Sep 17 00:00:00 2001 From: Kunpei Sakai Date: Fri, 26 Oct 2018 10:03:11 +0900 Subject: [PATCH 17/17] add argument requirements in code example --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d6b2f5..f9520aa 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ module "pubsub" { { name = "push" // required ack_deadline_seconds = 20 // optonal - push_endpoint = "https://example.com" - x-goog-version = "v1beta1" + push_endpoint = "https://example.com" // required + x-goog-version = "v1beta1" // optional } ] pull_subscriptions = [ @@ -155,4 +155,4 @@ are as follows: - Golang - gofmt. gofmt comes with the standard golang installation. golang -s a compiled language so there is no standard linter. - Terraform - terraform has a built-in linter in the `terraform validate` command. -- Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file +- Dockerfiles - hadolint. Can be found in homebrew