From 66d6d05bdbce9b6d85c9fd925d09e00ccfa03dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Sauv=C3=A8re?= Date: Thu, 8 Aug 2024 09:57:23 +0200 Subject: [PATCH] feat: add support for `require_partition_filter` in the external tables --- README.md | 2 +- examples/multiple_tables/main.tf | 3 ++- main.tf | 5 +++-- metadata.yaml | 5 +++-- variables.tf | 5 +++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 29f08442..9ab28021 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | deletion\_protection | Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion\_protection configuration. | `bool` | `false` | no | | description | Dataset description. | `string` | `null` | no | | encryption\_key | Default encryption key to apply to the dataset. Defaults to null (Google-managed). | `string` | `null` | no | -| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | 
list(object({
    table_id              = string,
    description           = optional(string),
    autodetect            = bool,
    compression           = string,
    ignore_unknown_values = bool,
    max_bad_records       = number,
    schema                = string,
    source_format         = string,
    source_uris           = list(string),
    csv_options = object({
      quote                 = string,
      allow_jagged_rows     = bool,
      allow_quoted_newlines = bool,
      encoding              = string,
      field_delimiter       = string,
      skip_leading_rows     = number,
    }),
    google_sheets_options = object({
      range             = string,
      skip_leading_rows = number,
    }),
    hive_partitioning_options = object({
      mode              = string,
      source_uri_prefix = string,
    }),
    expiration_time     = string,
    max_staleness       = optional(string),
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | +| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | 
list(object({
    table_id              = string,
    description           = optional(string),
    autodetect            = bool,
    compression           = string,
    ignore_unknown_values = bool,
    max_bad_records       = number,
    schema                = string,
    source_format         = string,
    source_uris           = list(string),
    csv_options = object({
      quote                 = string,
      allow_jagged_rows     = bool,
      allow_quoted_newlines = bool,
      encoding              = string,
      field_delimiter       = string,
      skip_leading_rows     = number,
    }),
    google_sheets_options = object({
      range             = string,
      skip_leading_rows = number,
    }),
    hive_partitioning_options = object({
      mode                     = string,
      source_uri_prefix        = string,
      require_partition_filter = optional(bool),
    }),
    expiration_time     = string,
    max_staleness       = optional(string),
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | location | The regional location for the dataset only US and EU are allowed in module | `string` | `"US"` | no | | materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | 
list(object({
    view_id             = string,
    description         = optional(string),
    query               = string,
    enable_refresh      = bool,
    refresh_interval_ms = string,
    clustering          = list(string),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
      require_partition_filter = bool,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    max_staleness   = optional(string),
    labels          = map(string),
  }))
| `[]` | no | | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | diff --git a/examples/multiple_tables/main.tf b/examples/multiple_tables/main.tf index 39a9bf5e..db4d71a4 100644 --- a/examples/multiple_tables/main.tf +++ b/examples/multiple_tables/main.tf @@ -116,7 +116,8 @@ module "bigquery" { ] csv_options = null hive_partitioning_options = { - mode = "AUTO" + mode = "AUTO" + require_partition_filter = true # DO NOT CHANGE - see above source_uris source_uri_prefix = "gs://ci-bq-external-data/hive_partition_example/" } diff --git a/main.tf b/main.tf index aa7a6297..d239ac12 100644 --- a/main.tf +++ b/main.tf @@ -220,8 +220,9 @@ resource "google_bigquery_table" "external_table" { dynamic "hive_partitioning_options" { for_each = each.value["hive_partitioning_options"] != null ? [each.value["hive_partitioning_options"]] : [] content { - mode = hive_partitioning_options.value["mode"] - source_uri_prefix = hive_partitioning_options.value["source_uri_prefix"] + mode = hive_partitioning_options.value["mode"] + source_uri_prefix = hive_partitioning_options.value["source_uri_prefix"] + require_partition_filter = hive_partitioning_options.value["require_partition_filter"] } } } diff --git a/metadata.yaml b/metadata.yaml index cd5af95e..d512b61a 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -111,8 +111,9 @@ spec: skip_leading_rows = number, }), hive_partitioning_options = object({ - mode = string, - source_uri_prefix = string, + mode = string, + source_uri_prefix = string, + require_partition_filter = optional(bool), }), expiration_time = string, max_staleness = optional(string), diff --git a/variables.tf b/variables.tf index 68a0667d..c1a9b2af 100644 --- a/variables.tf +++ b/variables.tf @@ -191,8 +191,9 @@ variable "external_tables" { skip_leading_rows = number, }), hive_partitioning_options = object({ - mode = string, - source_uri_prefix = string, + mode = string, + source_uri_prefix = string, + require_partition_filter = optional(bool), }), expiration_time = string, max_staleness = optional(string),