feat: add support for require_partition_filter in the external tables

terraform-google-modules · Aug 18, 2024 · 66d6d05 · 66d6d05
1 parent 0974205
commit 66d6d05
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -194,7 +194,7 @@ This module provisions a dataset and a list of tables with associated JSON schem
 | deletion\_protection | Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion\_protection configuration. | `bool` | `false` | no |
 | description | Dataset description. | `string` | `null` | no |
 | encryption\_key | Default encryption key to apply to the dataset. Defaults to null (Google-managed). | `string` | `null` | no |
-| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | <pre>list(object({<br>    table_id              = string,<br>    description           = optional(string),<br>    autodetect            = bool,<br>    compression           = string,<br>    ignore_unknown_values = bool,<br>    max_bad_records       = number,<br>    schema                = string,<br>    source_format         = string,<br>    source_uris           = list(string),<br>    csv_options = object({<br>      quote                 = string,<br>      allow_jagged_rows     = bool,<br>      allow_quoted_newlines = bool,<br>      encoding              = string,<br>      field_delimiter       = string,<br>      skip_leading_rows     = number,<br>    }),<br>    google_sheets_options = object({<br>      range             = string,<br>      skip_leading_rows = number,<br>    }),<br>    hive_partitioning_options = object({<br>      mode              = string,<br>      source_uri_prefix = string,<br>    }),<br>    expiration_time     = string,<br>    max_staleness       = optional(string),<br>    deletion_protection = optional(bool),<br>    labels              = map(string),<br>  }))</pre> | `[]` | no |
+| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | <pre>list(object({<br>    table_id              = string,<br>    description           = optional(string),<br>    autodetect            = bool,<br>    compression           = string,<br>    ignore_unknown_values = bool,<br>    max_bad_records       = number,<br>    schema                = string,<br>    source_format         = string,<br>    source_uris           = list(string),<br>    csv_options = object({<br>      quote                 = string,<br>      allow_jagged_rows     = bool,<br>      allow_quoted_newlines = bool,<br>      encoding              = string,<br>      field_delimiter       = string,<br>      skip_leading_rows     = number,<br>    }),<br>    google_sheets_options = object({<br>      range             = string,<br>      skip_leading_rows = number,<br>    }),<br>    hive_partitioning_options = object({<br>      mode                     = string,<br>      source_uri_prefix        = string,<br>      require_partition_filter = optional(bool),<br>    }),<br>    expiration_time     = string,<br>    max_staleness       = optional(string),<br>    deletion_protection = optional(bool),<br>    labels              = map(string),<br>  }))</pre> | `[]` | no |
 | location | The regional location for the dataset only US and EU are allowed in module | `string` | `"US"` | no |
 | materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | <pre>list(object({<br>    view_id             = string,<br>    description         = optional(string),<br>    query               = string,<br>    enable_refresh      = bool,<br>    refresh_interval_ms = string,<br>    clustering          = list(string),<br>    time_partitioning = object({<br>      expiration_ms            = string,<br>      field                    = string,<br>      type                     = string,<br>      require_partition_filter = bool,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time = string,<br>    max_staleness   = optional(string),<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
 | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no |

diff --git a/examples/multiple_tables/main.tf b/examples/multiple_tables/main.tf
@@ -116,7 +116,8 @@ module "bigquery" {
       ]
       csv_options = null
       hive_partitioning_options = {
-        mode = "AUTO"
+        mode                     = "AUTO"
+        require_partition_filter = true
         # DO NOT CHANGE - see above source_uris
         source_uri_prefix = "gs://ci-bq-external-data/hive_partition_example/"
       }

diff --git a/main.tf b/main.tf
@@ -220,8 +220,9 @@ resource "google_bigquery_table" "external_table" {
     dynamic "hive_partitioning_options" {
       for_each = each.value["hive_partitioning_options"] != null ? [each.value["hive_partitioning_options"]] : []
       content {
-        mode              = hive_partitioning_options.value["mode"]
-        source_uri_prefix = hive_partitioning_options.value["source_uri_prefix"]
+        mode                     = hive_partitioning_options.value["mode"]
+        source_uri_prefix        = hive_partitioning_options.value["source_uri_prefix"]
+        require_partition_filter = hive_partitioning_options.value["require_partition_filter"]
       }
     }
   }

diff --git a/metadata.yaml b/metadata.yaml
@@ -111,8 +111,9 @@ spec:
                 skip_leading_rows = number,
               }),
               hive_partitioning_options = object({
-                mode              = string,
-                source_uri_prefix = string,
+                mode                     = string,
+                source_uri_prefix        = string,
+                require_partition_filter = optional(bool),
               }),
               expiration_time     = string,
               max_staleness       = optional(string),

diff --git a/variables.tf b/variables.tf
@@ -191,8 +191,9 @@ variable "external_tables" {
       skip_leading_rows = number,
     }),
     hive_partitioning_options = object({
-      mode              = string,
-      source_uri_prefix = string,
+      mode                     = string,
+      source_uri_prefix        = string,
+      require_partition_filter = optional(bool),
     }),
     expiration_time     = string,
     max_staleness       = optional(string),