diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82d1b5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.kitchen/ +.kitchen.local.yml diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..81f83c6 --- /dev/null +++ b/Gemfile @@ -0,0 +1,19 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source 'https://rubygems.org/' + +gem 'test-kitchen' +gem 'kitchen-terraform', '~> 4.0.3' +gem 'kitchen-inspec' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..6cd1f0d --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,260 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + aws-sdk (2.11.156) + aws-sdk-resources (= 2.11.156) + aws-sdk-core (2.11.156) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.11.156) + aws-sdk-core (= 2.11.156) + aws-sigv4 (1.0.3) + azure_graph_rbac (0.17.0) + ms_rest_azure (~> 0.11.0) + azure_mgmt_key_vault (0.17.2) + ms_rest_azure (~> 0.11.0) + azure_mgmt_resources (0.17.2) + ms_rest_azure (~> 0.11.0) + builder (3.2.3) + coderay (1.1.2) + concurrent-ruby (1.0.5) + declarative (0.0.10) + declarative-option (0.1.0) + diff-lcs (1.3) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.7.0) + concurrent-ruby (~> 1.0) + dry-container (0.6.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.1) + dry-inflector (0.1.2) + dry-logic (0.4.2) + dry-container (~> 0.2, >= 0.2.6) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-types (0.13.2) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) + dry-validation (0.12.2) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (~> 0.2, >= 0.2.1) + dry-equalizer (~> 0.2) + dry-logic (~> 0.4, >= 0.4.0) + dry-types (~> 0.13.1) + erubis (2.7.0) + excon (0.62.0) + faraday (0.15.3) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + faraday_middleware (0.12.2) + faraday (>= 0.7.4, < 1.0) + ffi (1.9.25) + google-api-client (0.23.9) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.5, < 0.7.0) + httpclient (>= 2.8.1, < 3.0) + mime-types (~> 3.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.9) + googleauth (0.6.7) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + hashie (3.6.0) + htmlentities (4.3.4) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + inifile (3.0.0) + inspec (2.3.24) + addressable (~> 2.4) + faraday (>= 0.9.0) + faraday_middleware (~> 0.12.2) + hashie (~> 3.4) + htmlentities + json (>= 1.8, < 3.0) + method_source (~> 0.8) + mixlib-log + multipart-post + parallel (~> 1.9) + parslet (~> 1.5) + pry (~> 0) + rspec (~> 3) + rspec-its (~> 1.2) + rubyzip (~> 1.2, >= 1.2.2) + semverse + sslshake (~> 1.2) + term-ansicolor + thor (~> 0.20) + tomlrb (~> 1.2) + train (~> 1.5) + jmespath (1.4.0) + json (2.1.0) + jwt (2.1.0) + kitchen-inspec (0.25.0) + hashie (~> 3.4) + inspec (>= 0.34.0, < 4.0.0) + test-kitchen (~> 1.6) + kitchen-terraform (4.0.3) + dry-types (~> 0.9) + dry-validation (~> 0.10) + inspec (~> 2.2, != 2.3.5, != 2.3.4, != 2.2.112, != 2.2.102, != 2.2.101) + mixlib-shellout (~> 2.2) + test-kitchen (~> 1.23) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + method_source (0.9.0) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + mixlib-install (3.11.5) + mixlib-shellout + mixlib-versioning + thor + mixlib-log (2.0.4) + mixlib-shellout (2.4.0) + mixlib-versioning (1.2.2) + ms_rest (0.7.3) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.11.0) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.2) + multi_json (1.13.1) + multipart-post (2.0.0) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + nori (2.6.0) + os (1.0.0) + parallel (1.12.1) + parslet (1.8.2) + pry (0.11.3) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.0.3) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) + retriable (3.1.2) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + semverse (2.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + sslshake (1.2.0) + term-ansicolor (1.6.0) + tins (~> 1.0) + test-kitchen (1.23.2) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.20.0) + timeliness (0.3.8) + tins (1.17.0) + tomlrb (1.2.7) + train (1.5.4) + aws-sdk (~> 2) + azure_graph_rbac (~> 0.16) + azure_mgmt_key_vault (~> 0.17) + azure_mgmt_resources (~> 0.15) + docker-api (~> 1.26) + google-api-client (~> 0.23.9) + googleauth (~> 0.6.6) + inifile + json (>= 1.8, < 3.0) + mixlib-shellout (~> 2.0) + net-scp (~> 1.2) + net-ssh (>= 2.9, < 6.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + uber (0.1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + winrm (2.3.0) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.3.1) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + kitchen-inspec + kitchen-terraform (~> 4.0.3) + test-kitchen + +BUNDLED WITH + 1.16.6 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b9b1fda --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make will use bash instead of sh +SHELL := /usr/bin/env bash + +# All is the first target in the file so it will get picked up when you just run 'make' on its own +all: check_shell check_python check_golang check_terraform check_docker check_base_files check_trailing_whitespace generate_docs + +# The .PHONY directive tells make that this isn't a real target and so +# the presence of a file named 'check_shell' won't cause this target to stop +# working +.PHONY: check_shell +check_shell: + @source test/make.sh && check_shell + +.PHONY: check_python +check_python: + @source test/make.sh && check_python + +.PHONY: check_golang +check_golang: + @source test/make.sh && golang + +.PHONY: check_terraform +check_terraform: + @source test/make.sh && check_terraform + +.PHONY: check_docker +check_docker: + @source test/make.sh && docker + +.PHONY: check_base_files +check_base_files: + @source test/make.sh && basefiles + +.PHONY: check_shebangs +check_shebangs: + @source test/make.sh && check_bash + +.PHONY: check_trailing_whitespace +check_trailing_whitespace: + @source test/make.sh && check_trailing_whitespace + +.PHONY: test_check_headers +test_check_headers: + @python test/test_verify_boilerplate.py + +.PHONY: check_headers +check_headers: + @python test/verify_boilerplate.py + +.PHONY: generate_docs +generate_docs: + @source test/make.sh && generate_docs + +.PHONY: test_integration +test_integration: + @source test/test.sh diff --git a/README.md b/README.md index cb7274b..f9520aa 100644 --- a/README.md +++ b/README.md @@ -1 +1,158 @@ -# terraform-google-pubsub \ No newline at end of file +# terraform-google-pubsub + +This module makes it easy to create Google Cloud Pub/Sub topic and subscriptions associated with the topic. + +## Usage + +This is a simple usage of the module. Please see also a simple setup provided in the example directory. + +```hcl +module "pubsub" { + source = "github.com/terraform-google-modules/terraform-google-pubsub" + topic = "tf-topic" + project_id = "my-pubsub-project" + push_subscriptions = [ + { + name = "push" // required + ack_deadline_seconds = 20 // optonal + push_endpoint = "https://example.com" // required + x-goog-version = "v1beta1" // optional + } + ] + pull_subscriptions = [ + { + name = "pull" // required + ack_deadline_seconds = 20 // optional + } + ] +} +``` + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| project_id | The project ID to manage the Pub/Sub resources | string | - | yes | +| pull_subscriptions | The list of the pull subscriptions | list | `` | no | +| push_subscriptions | The list of the push subscriptions | list | `` | no | +| topic | The Pub/Sub topic name | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| subscription_names | The name list of Pub/Sub subscriptions | +| subscription_paths | The path list of Pub/Sub subscriptions | +| topic | The name of the Pub/Sub topic | + +[^]: (autogen_docs_end) + +## Requirements + +### Installation Dependencies + +- [terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.12.x + +### Configure a Service Account + +In order to execute this module you must have a Service Account with the following: + +#### Roles + +- `roles/pubsub.editor` + +### Enable APIs + +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Cloud Pub/Sub API + +#### Service Account Credentials + +You can pass the service account credentials into this module by setting the following environment variables: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + +See more [details](https://www.terraform.io/docs/providers/google/provider_reference.html#configuration-reference). + +## Testing + +### Requirements + +- [bundler](https://bundler.io/) +- [ruby](https://www.ruby-lang.org/) 2.5.x +- [python](https://www.python.org/getit/) 2.7.x +- [terraform-docs](https://github.com/segmentio/terraform-docs) 0.4.5 +- [google-cloud-sdk](https://cloud.google.com/sdk/) + +### Generate docs automatically + +```sh +$ make generate_docs +``` + +### Integration Test + +The integration tests for this module leverage kitchen-terraform and kitchen-inspec. + +You must set up by manually before running the integration test: + +- Copy from `test/fixtures/terraform.tfvars.sample` to `test/fixtures/terraform.tfvars`. +- Modify values to match your environment. + +The tests will do the following: + +- Perform `bundle install` command + - Installs `test-kitchen`, `kitchen-terraform` and `kitchen-inspec` +- Perform `bundle exec kitchen create` command + - Performs `terraform init` +- Perform `bundle exec kitchen converge` command + - Performs `terraform apply -auto-approve` +- Perform `bundle exec kitchen verify` command + - Performs inspec tests +- Perform `bundle exec kitchen destroy` command + - Performs `terraform destroy -force` + +You can use the following command to run the integration test in the root directory. + +```sh +$ make test_integration +``` + +## Linting + +The makefile in this project will lint or sometimes just format any shell, Python, golang, Terraform, or Dockerfiles. The linters will only be run if the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to run + +```sh +$ make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running terraform fmt +Running hadolint on Dockerfiles +Checking for required files +The following lines have trailing whitespace +Generating markdown docs with terraform-docs +``` + +The linters +are as follows: +- Shell - shellcheck. Can be found in homebrew +- Python - flake8. Can be installed with `pip install flake8` +- Golang - gofmt. gofmt comes with the standard golang installation. golang +-s a compiled language so there is no standard linter. +- Terraform - terraform has a built-in linter in the `terraform validate` command. +- Dockerfiles - hadolint. Can be found in homebrew diff --git a/examples/simple/example.tf b/examples/simple/example.tf new file mode 100644 index 0000000..c3475d1 --- /dev/null +++ b/examples/simple/example.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + version = "~> 1.19" + region = "us-central1" +} + +module "pubsub" { + source = "../../" + topic = "${var.topic_name}" + project_id = "${var.project_id}" + + pull_subscriptions = [ + { + name = "pull" + ack_deadline_seconds = 20 + }, + ] + + push_subscriptions = [ + { + name = "push" + ack_deadline_seconds = 20 + push_endpoint = "https://${var.project_id}.appspot.com" + x-goog-version = "v1beta1" + }, + ] +} diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf new file mode 100644 index 0000000..2796070 --- /dev/null +++ b/examples/simple/variables.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project id to run tests against" +} + +variable "topic_name" { + description = "The name for the topic" +} diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py new file mode 100644 index 0000000..b01af7c --- /dev/null +++ b/helpers/combine_docfiles.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Combine file from: + * script argument 1 + with content of file from: + * script argument 2 + using the beginning of line separators + hardcoded using regexes in this file: + + We exclude any text using the separate + regex specified here +''' + +import re +import sys + +insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' +exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' + +if len(sys.argv) != 3: + sys.exit(1) + +input = open(sys.argv[1], "r").read() +replace_content = open(sys.argv[2], "r").read() + +# Exclude the specified content from the replacement content +groups = re.match(exclude_separator_regex, replace_content, re.DOTALL).groups(0) +replace_content = groups[0] + groups[1] + +# Find where to put the replacement content, overwrite the input file +groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0) +output = groups[0] + replace_content + groups[2] +open(sys.argv[1], "w").write(output) diff --git a/kitchen.yml b/kitchen.yml new file mode 100644 index 0000000..17d9bd2 --- /dev/null +++ b/kitchen.yml @@ -0,0 +1,37 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +provisioner: + name: terraform + +transport: + name: exec + connection_timeout: 60 + +verifier: + name: terraform + systems: + - name: system + backend: local + +platforms: + - name: local + +suites: + - name: default + driver: + name: terraform + root_module_directory: test/fixtures diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..7b2f878 --- /dev/null +++ b/main.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + default_ack_deadline_seconds = 10 +} + +resource "google_pubsub_topic" "topic" { + name = "${var.topic}" + project = "${var.project_id}" +} + +resource "google_pubsub_subscription" "push_subscriptions" { + count = "${length(var.push_subscriptions)}" + name = "${lookup(var.push_subscriptions[count.index], "name")}" + topic = "${google_pubsub_topic.topic.name}" + project = "${var.project_id}" + ack_deadline_seconds = "${lookup(var.push_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" + + push_config { + push_endpoint = "${lookup(var.push_subscriptions[count.index], "push_endpoint")}" + + // FIXME: This should be programmable, but nested map isn't supported at this time. + // https://github.com/hashicorp/terraform/issues/2114 + attributes = { + x-goog-version = "${lookup(var.push_subscriptions[count.index], "x-goog-version", "v1")}" + } + } + + depends_on = ["google_pubsub_topic.topic"] +} + +resource "google_pubsub_subscription" "pull_subscriptions" { + count = "${length(var.pull_subscriptions)}" + name = "${lookup(var.pull_subscriptions[count.index], "name")}" + topic = "${google_pubsub_topic.topic.name}" + project = "${var.project_id}" + ack_deadline_seconds = "${lookup(var.pull_subscriptions[count.index], "ack_deadline_seconds", local.default_ack_deadline_seconds)}" + + depends_on = ["google_pubsub_topic.topic"] +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..c41ad88 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "topic" { + value = "${google_pubsub_topic.topic.name}" + description = "The name of the Pub/Sub topic" +} + +output "subscription_names" { + value = "${concat( + google_pubsub_subscription.push_subscriptions.*.name, + google_pubsub_subscription.pull_subscriptions.*.name + )}" + + description = "The name list of Pub/Sub subscriptions" +} + +output "subscription_paths" { + value = "${concat( + google_pubsub_subscription.push_subscriptions.*.path, + google_pubsub_subscription.pull_subscriptions.*.path + )}" + + description = "The path list of Pub/Sub subscriptions" +} diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000..557e16f --- /dev/null +++ b/test/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.rb.txt b/test/boilerplate/boilerplate.rb.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.rb.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.tf.txt b/test/boilerplate/boilerplate.tf.txt new file mode 100644 index 0000000..cfccff8 --- /dev/null +++ b/test/boilerplate/boilerplate.tf.txt @@ -0,0 +1,15 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt new file mode 100644 index 0000000..3d98cdc --- /dev/null +++ b/test/boilerplate/boilerplate.xml.txt @@ -0,0 +1,15 @@ + diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.yaml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.yml.txt b/test/boilerplate/boilerplate.yml.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.yml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/fixtures/main.tf b/test/fixtures/main.tf new file mode 100644 index 0000000..1b3b3ca --- /dev/null +++ b/test/fixtures/main.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + credentials = "${file(var.credentials_file_path)}" +} + +module "pubsub" { + source = "../../" + project_id = "${var.project}" + topic = "${var.topic_name}" + + push_subscriptions = [ + { + name = "push" + push_endpoint = "https://${var.project}.appspot.com/" + x-goog-version = "v1beta1" + ack_deadline_seconds = 20 + }, + ] + + pull_subscriptions = [ + { + name = "pull" + }, + ] +} diff --git a/test/fixtures/outputs.tf b/test/fixtures/outputs.tf new file mode 100644 index 0000000..b11382b --- /dev/null +++ b/test/fixtures/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${var.project}" +} + +output "topic_name" { + value = "${var.topic_name}" +} diff --git a/test/fixtures/terraform.tfvars.sample b/test/fixtures/terraform.tfvars.sample new file mode 100644 index 0000000..540c233 --- /dev/null +++ b/test/fixtures/terraform.tfvars.sample @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# PLEASE FILL THE VARIABLES WITH VALID VALUES FOR TESTING # +# DO NOT REMOVE ANY OF THE VARIABLES # +################################################################# + +## These values you *MUST* modify to match your environment +credentials_file_path="../integration/default/sa-key.json" +project="tf-proj" +topic_name="tf-proj-topic" diff --git a/test/fixtures/variables.tf b/test/fixtures/variables.tf new file mode 100644 index 0000000..52172b0 --- /dev/null +++ b/test/fixtures/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_file_path" { + description = "Service account json auth path" +} + +variable "project" { + description = "The project to run tests against" +} + +variable "topic_name" { + description = "The project to run tests against" +} diff --git a/test/integration/default/controls/pubsub.rb b/test/integration/default/controls/pubsub.rb new file mode 100644 index 0000000..a675e1e --- /dev/null +++ b/test/integration/default/controls/pubsub.rb @@ -0,0 +1,38 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +topic = attribute('topic_name') + +describe command("gcloud --project='#{project_id}' pubsub topics describe #{topic}") do + its(:exit_status) { should be_zero } + it { expect(subject.stdout).to match(%r{name: projects/#{project_id}/topics/#{topic}}) } +end + +describe command("gcloud --project='#{project_id}' pubsub subscriptions describe pull --format=json") do + let(:stdout) { JSON.parse(subject.stdout, symbolize_names: true) } + its(:exit_status) { should be_zero } + it { expect(stdout).to include(name: "projects/#{project_id}/subscriptions/pull") } + it { expect(stdout).to include(topic: "projects/#{project_id}/topics/#{topic}") } + it { expect(stdout).to include(ackDeadlineSeconds: 10) } +end + +describe command("gcloud --project='#{project_id}' pubsub subscriptions describe push --format=json") do + let(:stdout) { JSON.parse(subject.stdout, symbolize_names: true) } + its(:exit_status) { should be_zero } + it { expect(stdout).to include(name: "projects/#{project_id}/subscriptions/push") } + it { expect(stdout).to include(topic: "projects/#{project_id}/topics/#{topic}") } + it { expect(stdout).to include(pushConfig: { pushEndpoint: "https://#{project_id}.appspot.com/" }) } + it { expect(stdout).to include(ackDeadlineSeconds: 20) } +end diff --git a/test/integration/default/inspec.yml b/test/integration/default/inspec.yml new file mode 100644 index 0000000..2df7ed1 --- /dev/null +++ b/test/integration/default/inspec.yml @@ -0,0 +1,28 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: cloud-pubsub +title: Google Cloud Pub/Sub +version: 0.1.0 +inspec_version: '~> 2.3.24' +attributes: + - name: project_id + type: string + required: true + - name: topic_name + type: string + required: true + - name: credentials_file_path + type: string + required: true diff --git a/test/make.sh b/test/make.sh new file mode 100755 index 0000000..8bdbfe4 --- /dev/null +++ b/test/make.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function checks to make sure that every +# shebang has a '- e' flag, which causes it +# to exit on error +function check_bash() { +find . -name "*.sh" | while IFS= read -d '' -r file; +do + if [[ "$file" != *"bash -e"* ]]; + then + echo "$file is missing shebang with -e"; + exit 1; + fi; +done; +} + +# This function makes sure that the required files for +# releasing to OSS are present +function basefiles() { + echo "Checking for required files" + test -f LICENSE || echo "Missing LICENSE" + test -f README.md || echo "Missing README.md" +} + +# This function runs the hadolint linter on +# every file named 'Dockerfile' +function docker() { + echo "Running hadolint on Dockerfiles" + find . -name "Dockerfile" -exec hadolint {} \; +} + +# This function runs 'terraform validate' against all +# files ending in '.tf' +function check_terraform() { + echo "Running terraform validate" + #shellcheck disable=SC2156 + find . -name "*.tf" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; + echo "Running terraform fmt" + terraform fmt -check=true -write=false +} + +# This function runs 'go fmt' and 'go vet' on every file +# that ends in '.go' +function golang() { + echo "Running go fmt and go vet" + find . -name "*.go" -exec go fmt {} \; + find . -name "*.go" -exec go vet {} \; +} + +# This function runs the flake8 linter on every file +# ending in '.py' +function check_python() { + echo "Running flake8" + find . -name "*.py" -exec flake8 {} \; +} + +# This function runs the shellcheck linter on every +# file ending in '.sh' +function check_shell() { + echo "Running shellcheck" + find . -name "*.sh" -exec shellcheck -x {} \; +} + +# This function makes sure that there is no trailing whitespace +# in any files in the project. +# There are some exclusions +function check_trailing_whitespace() { + echo "The following lines have trailing whitespace" + grep -r '[[:blank:]]$' --exclude-dir=".kitchen" --exclude-dir=".terraform" --exclude="*.png" --exclude="*.pyc" --exclude-dir=".git" . + rc=$? + if [ $rc = 0 ]; then + exit 1 + fi +} + +function generate_docs() { + echo "Generating markdown docs with terraform-docs" + TMPFILE=$(mktemp) + for j in $(for i in $(find . -type f | grep \.tf$ | grep -v "^./\(test/fixtures\|examples\)") ; do dirname "$i" ; done | sort -u) ; do + terraform-docs markdown "$j" > "$TMPFILE" + python helpers/combine_docfiles.py "$j"/README.md "$TMPFILE" + done + rm -f "$TMPFILE" +} diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 0000000..0fcb160 --- /dev/null +++ b/test/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +bundle install +bundle exec kitchen create +bundle exec kitchen converge +bundle exec kitchen verify +bundle exec kitchen destroy diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py new file mode 100755 index 0000000..60eca23 --- /dev/null +++ b/test/test_verify_boilerplate.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' A simple test for the verify_boilerplate python script. +This will create a set of test files, both valid and invalid, +and confirm that the has_valid_header call returns the correct +value. +It also checks the number of files that are found by the +get_files call. +''' +from copy import deepcopy +from tempfile import mkdtemp +from shutil import rmtree +import unittest +from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ + get_args, get_files + + +class AllTestCase(unittest.TestCase): + """ + All of the setup, teardown, and tests are contained in this + class. + """ + + def write_file(self, filename, content, expected): + """ + A utility method that creates test files, and adds them to + the cases that will be tested. + Args: + filename: (string) the file name (path) to be created. + content: (list of strings) the contents of the file. + expected: (boolean) True if the header is expected to be valid, + false if not. + """ + + file = open(filename, 'w+') + for line in content: + file.write(line + "\n") + file.close() + self.cases[filename] = expected + + def create_test_files(self, tmp_path, extension, header): + """ + Creates 2 test files for .tf, .xml, .go, etc and one for + Dockerfile, and Makefile. + The reason for the difference is that Makefile and Dockerfile + don't have an extension. These would be substantially more + difficult to create negative test cases, unless the files + were written, deleted, and re-written. + Args: + tmp_path: (string) the path in which to create the files + extension: (string) the file extension + header: (list of strings) the header/boilerplate content + """ + + content = "\n...blah \ncould be code or could be garbage\n" + special_cases = ["Dockerfile", "Makefile"] + header_template = deepcopy(header) + valid_filename = tmp_path + extension + valid_content = header_template.append(content) + if extension not in special_cases: + # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) + invalid_header = [] + for line in header_template: + if "2018" in line: + invalid_header.append(line.replace('2018', 'YEAR')) + else: + invalid_header.append(line) + invalid_header.append(content) + invalid_content = invalid_header + invalid_filename = tmp_path + "invalid." + extension + self.write_file(invalid_filename, invalid_content, False) + valid_filename = tmp_path + "testfile." + extension + + valid_content = header_template + self.write_file(valid_filename, valid_content, True) + + def setUp(self): + """ + Set initial counts and values, and initializes the setup of the + test files. + """ + self.cases = {} + self.tmp_path = mkdtemp() + "/" + self.my_args = get_args() + self.my_refs = get_refs(self.my_args) + self.my_regex = get_regexs() + self.prexisting_file_count = len( + get_files(self.my_refs.keys(), self.my_args)) + for key in self.my_refs: + self.create_test_files(self.tmp_path, key, + self.my_refs.get(key)) + + def tearDown(self): + """ Delete the test directory. """ + rmtree(self.tmp_path) + + def test_files_headers(self): + """ + Confirms that the expected output of has_valid_header is correct. + """ + for case in self.cases: + if self.cases[case]: + self.assertTrue(has_valid_header(case, self.my_refs, + self.my_regex)) + else: + self.assertFalse(has_valid_header(case, self.my_refs, + self.my_regex)) + + def test_invalid_count(self): + """ + Test that the initial files found isn't zero, indicating + a problem with the code. + """ + self.assertFalse(self.prexisting_file_count == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py new file mode 100755 index 0000000..a632fde --- /dev/null +++ b/test/verify_boilerplate.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Verifies that all source files contain the necessary copyright boilerplate +# snippet. +# This is based on existing work +# https://github.com/kubernetes/test-infra/blob/master/hack +# /verify_boilerplate.py +from __future__ import print_function +import argparse +import glob +import os +import re +import sys + + +def get_args(): + """Parses command line arguments. + + Configures and runs argparse.ArgumentParser to extract command line + arguments. + + Returns: + An argparse.Namespace containing the arguments parsed from the + command line + """ + parser = argparse.ArgumentParser() + parser.add_argument("filenames", + help="list of files to check, " + "all files if unspecified", + nargs='*') + rootdir = os.path.dirname(__file__) + "/../" + rootdir = os.path.abspath(rootdir) + parser.add_argument( + "--rootdir", + default=rootdir, + help="root directory to examine") + + default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") + parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) + return parser.parse_args() + + +def get_refs(ARGS): + """Converts the directory of boilerplate files into a map keyed by file + extension. + + Reads each boilerplate file's contents into an array, then adds that array + to a map keyed by the file extension. + + Returns: + A map of boilerplate lines, keyed by file extension. For example, + boilerplate.py.txt would result in the k,v pair {".py": py_lines} where + py_lines is an array containing each line of the file. + """ + refs = {} + + # Find and iterate over the absolute path for each boilerplate template + for path in glob.glob(os.path.join( + ARGS.boilerplate_dir, + "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + return refs + + +# pylint: disable=too-many-locals +def has_valid_header(filename, refs, regexs): + """Test whether a file has the correct boilerplate header. + + Tests each file against the boilerplate stored in refs for that file type + (based on extension), or by the entire filename (eg Dockerfile, Makefile). + Some heuristics are applied to remove build tags and shebangs, but little + variance in header formatting is tolerated. + + Args: + filename: A string containing the name of the file to test + refs: A map of boilerplate headers, keyed by file extension + regexs: a map of compiled regex objects used in verifying boilerplate + + Returns: + True if the file has the correct boilerplate header, otherwise returns + False. + """ + try: + with open(filename, 'r') as fp: # pylint: disable=invalid-name + data = fp.read() + except IOError: + return False + basename = os.path.basename(filename) + extension = get_file_extension(filename) + if extension: + ref = refs[extension] + else: + ref = refs[basename] + # remove build tags from the top of Go files + if extension == "go": + con = regexs["go_build_constraints"] + (data, found) = con.subn("", data, 1) + # remove shebang + elif extension == "sh" or extension == "py": + she = regexs["shebang"] + (data, found) = she.subn("", data, 1) + data = data.splitlines() + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + return False + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + year = regexs["year"] + for datum in data: + if year.search(datum): + return False + + # if we don't match the reference at this point, fail + if ref != data: + return False + return True + + +def get_file_extension(filename): + """Extracts the extension part of a filename. + + Identifies the extension as everything after the last period in filename. + + Args: + filename: string containing the filename + + Returns: + A string containing the extension in lowercase + """ + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# These directories will be omitted from header checks +SKIPPED_DIRS = [ + 'Godeps', 'third_party', '_gopath', '_output', + '.git', 'vendor', '__init__.py', 'node_modules' +] + + +def normalize_files(files): + """Extracts the files that require boilerplate checking from the files + argument. + + A new list will be built. Each path from the original files argument will + be added unless it is within one of SKIPPED_DIRS. All relative paths will + be converted to absolute paths by prepending the root_dir path parsed from + the command line, or its default value. + + Args: + files: a list of file path strings + + Returns: + A modified copy of the files list where any any path in a skipped + directory is removed, and all paths have been made absolute. + """ + newfiles = [] + for pathname in files: + if any(x in pathname for x in SKIPPED_DIRS): + continue + newfiles.append(pathname) + for idx, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[idx] = os.path.join(ARGS.rootdir, pathname) + return newfiles + + +def get_files(extensions, ARGS): + """Generates a list of paths whose boilerplate should be verified. + + If a list of file names has been provided on the command line, it will be + treated as the initial set to search. Otherwise, all paths within rootdir + will be discovered and used as the initial set. + + Once the initial set of files is identified, it is normalized via + normalize_files() and further stripped of any file name whose extension is + not in extensions. + + Args: + extensions: a list of file extensions indicating which file types + should have their boilerplate verified + + Returns: + A list of absolute file paths + """ + files = [] + if ARGS.filenames: + files = ARGS.filenames + else: + for root, dirs, walkfiles in os.walk(ARGS.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for dpath in SKIPPED_DIRS: + if dpath in dirs: + dirs.remove(dpath) + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = get_file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + """Builds a map of regular expressions used in boilerplate validation. + + There are two scenarios where these regexes are used. The first is in + validating the date referenced is the boilerplate, by ensuring it is an + acceptable year. The second is in identifying non-boilerplate elements, + like shebangs and compiler hints that should be ignored when validating + headers. + + Returns: + A map of compiled regular expression objects, keyed by mnemonic. + """ + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the + # real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016 or 2017, company holder names can be + # anything + regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", + re.MULTILINE) + # strip #!.* from shell/python scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(args): + """Identifies and verifies files that should have the desired boilerplate. + + Retrieves the lists of files to be validated and tests each one in turn. + If all files contain correct boilerplate, this function terminates + normally. Otherwise it prints the name of each non-conforming file and + exists with a non-zero status code. + """ + regexs = get_regexs() + refs = get_refs(args) + filenames = get_files(refs.keys(), args) + nonconforming_files = [] + for filename in filenames: + if not has_valid_header(filename, refs, regexs): + nonconforming_files.append(filename) + if nonconforming_files: + print('%d files have incorrect boilerplate headers:' % len( + nonconforming_files)) + for filename in sorted(nonconforming_files): + print(os.path.relpath(filename, args.rootdir)) + sys.exit(1) + + +if __name__ == "__main__": + ARGS = get_args() + main(ARGS) diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..86caf7c --- /dev/null +++ b/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to manage the Pub/Sub resources" +} + +variable "topic" { + description = "The Pub/Sub topic name" +} + +variable "push_subscriptions" { + type = "list" + description = "The list of the push subscriptions" + default = [] +} + +variable "pull_subscriptions" { + type = "list" + description = "The list of the pull subscriptions" + default = [] +}