diff --git a/README.md b/README.md index d99bfa2..58c6477 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,18 @@ module "pubsub" { drop_unknown_fields = false // optional } ] + cloud_storage_subscriptions = [ + { + name = "cloud-storage" // required + bucket = "example-bucket" // required + filename_prefix = "log_events_" // optional + filename_suffix = ".avro" // optional + max_duration = "60s" // optional + max_bytes = "10000000" // optional + output_format = "avro" // optional + write_metadata = false // optional + } + ] } ``` @@ -67,7 +79,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the bigquery push subscriptions. | `list(map(string))` | `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | `list(map(string))` | `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | `list(map(string))` | `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_token\_creator | Specify true if you want to add token creator role to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/examples/cloud_storage/README.md b/examples/cloud_storage/README.md new file mode 100644 index 0000000..4a7d812 --- /dev/null +++ b/examples/cloud_storage/README.md @@ -0,0 +1,35 @@ +# Cloud Storage Example + +This example illustrates how to use the `pubsub` module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| bucket\_name | The name of the Cloud Storage bucket created | +| project\_id | The project ID | +| topic\_name | The name of the Pub/Sub topic created | + + + +## Requirements + +The following sections describe the requirements which must be met in +order to invoke this example. The requirements of the +[root module][root-module-requirements] must be met. + +## Usage + +To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within +this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/cloud_storage/main.tf b/examples/cloud_storage/main.tf new file mode 100644 index 0000000..4fec517 --- /dev/null +++ b/examples/cloud_storage/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018-2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "bucket_suffix" { + byte_length = 4 +} + +provider "google" { + region = "europe-west1" +} + +module "pubsub" { + source = "../../" + project_id = var.project_id + topic = "cft-tf-pubsub-topic-cloud-storage" + + topic_labels = { + foo_label = "foo_value" + } + + cloud_storage_subscriptions = [ + { + name = "example_bucket_subscription" + bucket = google_storage_bucket.test.name + + ack_deadline_seconds = 300 + }, + ] +} + +resource "google_storage_bucket" "test" { + project = var.project_id + name = join("-", ["test_bucket", random_id.bucket_suffix.hex]) + location = "europe-west1" +} diff --git a/examples/cloud_storage/outputs.tf b/examples/cloud_storage/outputs.tf new file mode 100644 index 0000000..073d4cc --- /dev/null +++ b/examples/cloud_storage/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2018-2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id + description = "The project ID" +} + +output "bucket_name" { + value = google_storage_bucket.test.name + description = "The name of the Cloud Storage bucket created" +} + +output "topic_name" { + value = module.pubsub.topic + description = "The name of the Pub/Sub topic created" +} diff --git a/examples/cloud_storage/variables.tf b/examples/cloud_storage/variables.tf new file mode 100644 index 0000000..5abd8d8 --- /dev/null +++ b/examples/cloud_storage/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources" +} diff --git a/main.tf b/main.tf index 3223355..2e80964 100644 --- a/main.tf +++ b/main.tf @@ -45,6 +45,13 @@ resource "google_project_iam_member" "bigquery_data_editor_binding" { member = "serviceAccount:${local.pubsub_svc_account_email}" } +resource "google_project_iam_member" "storage_admin_binding" { + count = length(var.cloud_storage_subscriptions) != 0 ? 1 : 0 + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + resource "google_project_iam_member" "token_creator_binding" { count = var.grant_token_creator ? 1 : 0 project = var.project_id @@ -337,6 +344,82 @@ resource "google_pubsub_subscription" "bigquery_subscriptions" { ] } +resource "google_pubsub_subscription" "cloud_storage_subscriptions" { + for_each = var.create_subscriptions ? { for i in var.cloud_storage_subscriptions : i.name => i } : {} + + name = each.value.name + topic = var.create_topic ? google_pubsub_topic.topic[0].name : var.topic + project = var.project_id + labels = var.subscription_labels + ack_deadline_seconds = lookup( + each.value, + "ack_deadline_seconds", + local.default_ack_deadline_seconds, + ) + message_retention_duration = lookup( + each.value, + "message_retention_duration", + null, + ) + retain_acked_messages = lookup( + each.value, + "retain_acked_messages", + null, + ) + filter = lookup( + each.value, + "filter", + null, + ) + enable_message_ordering = lookup( + each.value, + "enable_message_ordering", + null, + ) + dynamic "expiration_policy" { + // check if the 'expiration_policy' key exists, if yes, return a list containing it. + for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : [] + content { + ttl = expiration_policy.value + } + } + + dynamic "dead_letter_policy" { + for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : [] + content { + dead_letter_topic = lookup(each.value, "dead_letter_topic", "") + max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5") + } + } + + dynamic "retry_policy" { + for_each = (lookup(each.value, "maximum_backoff", "") != "") ? [each.value.maximum_backoff] : [] + content { + maximum_backoff = lookup(each.value, "maximum_backoff", "") + minimum_backoff = lookup(each.value, "minimum_backoff", "") + } + } + + cloud_storage_config { + bucket = each.value["bucket"] + filename_prefix = lookup(each.value, "filename_prefix", null) + filename_suffix = lookup(each.value, "filename_suffix", null) + max_duration = lookup(each.value, "max_duration", null) + max_bytes = lookup(each.value, "max_bytes", null) + dynamic "avro_config" { + for_each = (lookup(each.value, "output_format", "") == "avro") ? [true] : [] + content { + write_metadata = lookup(each.value, "write_metadata", null) + } + } + } + + depends_on = [ + google_pubsub_topic.topic, + google_project_iam_member.storage_admin_binding + ] +} + resource "google_pubsub_subscription_iam_member" "pull_subscription_sa_binding_subscriber" { for_each = var.create_subscriptions ? { for i in var.pull_subscriptions : i.name => i if lookup(i, "service_account", null) != null } : {} diff --git a/outputs.tf b/outputs.tf index 9ad7fa0..c10c907 100644 --- a/outputs.tf +++ b/outputs.tf @@ -39,6 +39,7 @@ output "subscription_names" { values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.name }), values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.name }), values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }), ) description = "The name list of Pub/Sub subscriptions" @@ -49,6 +50,7 @@ output "subscription_paths" { values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.id }), values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.id }), values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }), ) description = "The path list of Pub/Sub subscriptions" diff --git a/test/setup/iam.tf b/test/setup/iam.tf index a501766..5a41966 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -18,7 +18,8 @@ locals { int_required_roles = [ "roles/pubsub.admin", "roles/resourcemanager.projectIamAdmin", - "roles/bigquery.admin" + "roles/bigquery.admin", + "roles/storage.admin" ] } diff --git a/test/setup/main.tf b/test/setup/main.tf index b4f07ec..3bacc20 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -29,5 +29,6 @@ module "project-ci-int-pubsub" { "pubsub.googleapis.com", "serviceusage.googleapis.com", "bigquery.googleapis.com", + "storage.googleapis.com" ] } diff --git a/test/setup/versions.tf b/test/setup/versions.tf index 910d5df..f106432 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -19,11 +19,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.23" + version = ">= 4.78" } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.23" + version = ">= 4.78" } null = { source = "hashicorp/null" diff --git a/variables.tf b/variables.tf index 8b5e4de..c12f340 100644 --- a/variables.tf +++ b/variables.tf @@ -55,7 +55,13 @@ variable "pull_subscriptions" { variable "bigquery_subscriptions" { type = list(map(string)) - description = "The list of the bigquery push subscriptions." + description = "The list of the Bigquery push subscriptions." + default = [] +} + +variable "cloud_storage_subscriptions" { + type = list(map(string)) + description = "The list of the Cloud Storage push subscriptions." default = [] } diff --git a/versions.tf b/versions.tf index d7533ff..7bcd0f0 100644 --- a/versions.tf +++ b/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = ">= 4.32, < 6" + version = ">= 4.78, < 6" } }