feat(TPG>=4.78)!: add cloud storage subscription support (#145)

Co-authored-by: Andrew Peabody <[email protected]>

README.md

@@ -59,6 +59,18 @@ module "pubsub" {
       drop_unknown_fields = false                   // optional
     }
   ]
+  cloud_storage_subscriptions = [
+    {
+      name            = "cloud-storage"  // required
+      bucket          = "example-bucket" // required
+      filename_prefix = "log_events_"    // optional
+      filename_suffix = ".avro"          // optional
+      max_duration    = "60s"            // optional
+      max_bytes       = "10000000"       // optional
+      output_format   = "avro"           // optional
+      write_metadata  = false            // optional
+    }
+  ]
 }
 ```
 
@@ -67,7 +79,8 @@ module "pubsub" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| bigquery\_subscriptions | The list of the bigquery push subscriptions. | `list(map(string))` | `[]` | no |
+| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | `list(map(string))` | `[]` | no |
+| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | `list(map(string))` | `[]` | no |
 | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no |
 | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no |
 | grant\_token\_creator | Specify true if you want to add token creator role to the default Pub/Sub SA. | `bool` | `true` | no |

examples/cloud_storage/README.md

@@ -0,0 +1,35 @@
+# Cloud Storage Example
+
+This example illustrates how to use the `pubsub` module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| bucket\_name | The name of the Cloud Storage bucket created |
+| project\_id | The project ID |
+| topic\_name | The name of the Pub/Sub topic created |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Requirements
+
+The following sections describe the requirements which must be met in
+order to invoke this example. The requirements of the
+[root module][root-module-requirements] must be met.
+
+## Usage
+
+To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within
+this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/cloud_storage/main.tf

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2018-2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_id" "bucket_suffix" {
+  byte_length = 4
+}
+
+provider "google" {
+  region = "europe-west1"
+}
+
+module "pubsub" {
+  source     = "../../"
+  project_id = var.project_id
+  topic      = "cft-tf-pubsub-topic-cloud-storage"
+
+  topic_labels = {
+    foo_label = "foo_value"
+  }
+
+  cloud_storage_subscriptions = [
+    {
+      name   = "example_bucket_subscription"
+      bucket = google_storage_bucket.test.name
+
+      ack_deadline_seconds = 300
+    },
+  ]
+}
+
+resource "google_storage_bucket" "test" {
+  project  = var.project_id
+  name     = join("-", ["test_bucket", random_id.bucket_suffix.hex])
+  location = "europe-west1"
+}

examples/cloud_storage/outputs.tf

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2018-2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  value       = var.project_id
+  description = "The project ID"
+}
+
+output "bucket_name" {
+  value       = google_storage_bucket.test.name
+  description = "The name of the Cloud Storage bucket created"
+}
+
+output "topic_name" {
+  value       = module.pubsub.topic
+  description = "The name of the Pub/Sub topic created"
+}

examples/cloud_storage/variables.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  type        = string
+  description = "The project ID to manage the Pub/Sub resources"
+}

main.tf

@@ -45,6 +45,13 @@ resource "google_project_iam_member" "bigquery_data_editor_binding" {
   member  = "serviceAccount:${local.pubsub_svc_account_email}"
 }
 
+resource "google_project_iam_member" "storage_admin_binding" {
+  count   = length(var.cloud_storage_subscriptions) != 0 ? 1 : 0
+  project = var.project_id
+  role    = "roles/storage.admin"
+  member  = "serviceAccount:${local.pubsub_svc_account_email}"
+}
+
 resource "google_project_iam_member" "token_creator_binding" {
   count   = var.grant_token_creator ? 1 : 0
   project = var.project_id
@@ -337,6 +344,82 @@ resource "google_pubsub_subscription" "bigquery_subscriptions" {
   ]
 }
 
+resource "google_pubsub_subscription" "cloud_storage_subscriptions" {
+  for_each = var.create_subscriptions ? { for i in var.cloud_storage_subscriptions : i.name => i } : {}
+
+  name    = each.value.name
+  topic   = var.create_topic ? google_pubsub_topic.topic[0].name : var.topic
+  project = var.project_id
+  labels  = var.subscription_labels
+  ack_deadline_seconds = lookup(
+    each.value,
+    "ack_deadline_seconds",
+    local.default_ack_deadline_seconds,
+  )
+  message_retention_duration = lookup(
+    each.value,
+    "message_retention_duration",
+    null,
+  )
+  retain_acked_messages = lookup(
+    each.value,
+    "retain_acked_messages",
+    null,
+  )
+  filter = lookup(
+    each.value,
+    "filter",
+    null,
+  )
+  enable_message_ordering = lookup(
+    each.value,
+    "enable_message_ordering",
+    null,
+  )
+  dynamic "expiration_policy" {
+    // check if the 'expiration_policy' key exists, if yes, return a list containing it.
+    for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : []
+    content {
+      ttl = expiration_policy.value
+    }
+  }
+
+  dynamic "dead_letter_policy" {
+    for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : []
+    content {
+      dead_letter_topic     = lookup(each.value, "dead_letter_topic", "")
+      max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5")
+    }
+  }
+
+  dynamic "retry_policy" {
+    for_each = (lookup(each.value, "maximum_backoff", "") != "") ? [each.value.maximum_backoff] : []
+    content {
+      maximum_backoff = lookup(each.value, "maximum_backoff", "")
+      minimum_backoff = lookup(each.value, "minimum_backoff", "")
+    }
+  }
+
+  cloud_storage_config {
+    bucket          = each.value["bucket"]
+    filename_prefix = lookup(each.value, "filename_prefix", null)
+    filename_suffix = lookup(each.value, "filename_suffix", null)
+    max_duration    = lookup(each.value, "max_duration", null)
+    max_bytes       = lookup(each.value, "max_bytes", null)
+    dynamic "avro_config" {
+      for_each = (lookup(each.value, "output_format", "") == "avro") ? [true] : []
+      content {
+        write_metadata = lookup(each.value, "write_metadata", null)
+      }
+    }
+  }
+
+  depends_on = [
+    google_pubsub_topic.topic,
+    google_project_iam_member.storage_admin_binding
+  ]
+}
+
 resource "google_pubsub_subscription_iam_member" "pull_subscription_sa_binding_subscriber" {
   for_each = var.create_subscriptions ? { for i in var.pull_subscriptions : i.name => i if lookup(i, "service_account", null) != null } : {}
 

outputs.tf

@@ -39,6 +39,7 @@ output "subscription_names" {
     values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.name }),
     values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.name }),
     values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }),
+    values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }),
   )
 
   description = "The name list of Pub/Sub subscriptions"
@@ -49,6 +50,7 @@ output "subscription_paths" {
     values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.id }),
     values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.id }),
     values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }),
+    values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }),
   )
 
   description = "The path list of Pub/Sub subscriptions"

test/setup/iam.tf

@@ -18,7 +18,8 @@ locals {
   int_required_roles = [
     "roles/pubsub.admin",
     "roles/resourcemanager.projectIamAdmin",
-    "roles/bigquery.admin"
+    "roles/bigquery.admin",
+    "roles/storage.admin"
   ]
 }
 

test/setup/main.tf

@@ -29,5 +29,6 @@ module "project-ci-int-pubsub" {
     "pubsub.googleapis.com",
     "serviceusage.googleapis.com",
     "bigquery.googleapis.com",
+    "storage.googleapis.com"
   ]
 }

test/setup/versions.tf

@@ -19,11 +19,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.23"
+      version = ">= 4.78"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.23"
+      version = ">= 4.78"
     }
     null = {
       source = "hashicorp/null"

variables.tf

@@ -55,7 +55,13 @@ variable "pull_subscriptions" {
 
 variable "bigquery_subscriptions" {
   type        = list(map(string))
-  description = "The list of the bigquery push subscriptions."
+  description = "The list of the Bigquery push subscriptions."
+  default     = []
+}
+
+variable "cloud_storage_subscriptions" {
+  type        = list(map(string))
+  description = "The list of the Cloud Storage push subscriptions."
   default     = []
 }
 

versions.tf

@@ -20,7 +20,7 @@ terraform {
 
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.32, < 6"
+      version = ">= 4.78, < 6"
     }
   }