From 46a448f43862eae4af0cd46cd0ac9215b33f6b32 Mon Sep 17 00:00:00 2001 From: lord-skinner Date: Tue, 10 Sep 2024 10:45:40 -0500 Subject: [PATCH 1/9] initial commit --- README.md | 3 +++ main.tf | 2 +- variables.tf | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f1393d4..8c5ca052 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,9 @@ module "bigquery" { env = "dev" billable = "true" } + resource_tags = { + short_name = "123456789012/environment" + } } ``` diff --git a/main.tf b/main.tf index aa7a6297..3b42ae2c 100644 --- a/main.tf +++ b/main.tf @@ -38,7 +38,7 @@ resource "google_bigquery_dataset" "main" { max_time_travel_hours = var.max_time_travel_hours project = var.project_id labels = var.dataset_labels - + resource_tags = var.resource_tags dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [var.encryption_key] content { diff --git a/variables.tf b/variables.tf index 68a0667d..b99cf6e3 100644 --- a/variables.tf +++ b/variables.tf @@ -77,6 +77,11 @@ variable "dataset_labels" { type = map(string) default = {} } +variable "resource_tags" { + description = "A map of resource tags to add to the dataset" + type = map(string) + default = {} +} # Format: list(objects) # domain: A domain to grant access to. From 3f74bb671b94f359fb6cecab1afc23cc41ae34ad Mon Sep 17 00:00:00 2001 From: lord-skinner Date: Tue, 10 Sep 2024 10:58:27 -0500 Subject: [PATCH 2/9] updating things --- README.md | 7 +++---- main.tf | 16 +++++++++++++++- variables.tf | 14 ++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8c5ca052..a280fcb2 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ module "bigquery" { project_id = "" location = "US" default_table_expiration_ms = 3600000 - + tag_key = "56789" + tag_value = "production" + tables = [ { table_id = "foo", @@ -101,9 +103,6 @@ module "bigquery" { env = "dev" billable = "true" } - resource_tags = { - short_name = "123456789012/environment" - } } ``` diff --git a/main.tf b/main.tf index 3b42ae2c..921c3771 100644 --- a/main.tf +++ b/main.tf @@ -28,6 +28,12 @@ locals { } } +data "google_tags_tag_value" "tag_value" { + count = var.tag_key != "" && var.tag_value != "" ? 1 : 0 + parent = "tagKeys/${var.tag_key}" + short_name = var.tag_value +} + resource "google_bigquery_dataset" "main" { dataset_id = var.dataset_id friendly_name = var.dataset_name @@ -38,7 +44,15 @@ resource "google_bigquery_dataset" "main" { max_time_travel_hours = var.max_time_travel_hours project = var.project_id labels = var.dataset_labels - resource_tags = var.resource_tags + + dynamic "resource_tags" { + for_each = var.tag_key != "" && var.tag_value != "" ? [1] : [] + content { + key = data.google_tags_tag_value.tag_value[0].parent + value = data.google_tags_tag_value.tag_value[0].id + } + } + dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [var.encryption_key] content { diff --git a/variables.tf b/variables.tf index b99cf6e3..68175e3c 100644 --- a/variables.tf +++ b/variables.tf @@ -77,10 +77,16 @@ variable "dataset_labels" { type = map(string) default = {} } -variable "resource_tags" { - description = "A map of resource tags to add to the dataset" - type = map(string) - default = {} +variable "tag_key" { + description = "The ID of the parent TagKey" + type = string + default = "" +} + +variable "tag_value" { + description = "The short name of the TagValue" + type = string + default = "" } # Format: list(objects) From ff4dd7cb8192bf6cd1d03bbe0fad343ed63ced2d Mon Sep 17 00:00:00 2001 From: Anuj Rohilla Date: Wed, 11 Sep 2024 13:02:50 +0530 Subject: [PATCH 3/9] Fix the dataset tag --- README.md | 3 +-- main.tf | 15 +-------------- variables.tf | 13 ++++--------- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a280fcb2..68d110ed 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,7 @@ module "bigquery" { project_id = "" location = "US" default_table_expiration_ms = 3600000 - tag_key = "56789" - tag_value = "production" + resource_tags = {"/":""} tables = [ { diff --git a/main.tf b/main.tf index 921c3771..aa692341 100644 --- a/main.tf +++ b/main.tf @@ -28,12 +28,6 @@ locals { } } -data "google_tags_tag_value" "tag_value" { - count = var.tag_key != "" && var.tag_value != "" ? 1 : 0 - parent = "tagKeys/${var.tag_key}" - short_name = var.tag_value -} - resource "google_bigquery_dataset" "main" { dataset_id = var.dataset_id friendly_name = var.dataset_name @@ -44,14 +38,7 @@ resource "google_bigquery_dataset" "main" { max_time_travel_hours = var.max_time_travel_hours project = var.project_id labels = var.dataset_labels - - dynamic "resource_tags" { - for_each = var.tag_key != "" && var.tag_value != "" ? [1] : [] - content { - key = data.google_tags_tag_value.tag_value[0].parent - value = data.google_tags_tag_value.tag_value[0].id - } - } + resource_tags = var.resource_tags dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [var.encryption_key] diff --git a/variables.tf b/variables.tf index 68175e3c..02dcc6f6 100644 --- a/variables.tf +++ b/variables.tf @@ -77,16 +77,11 @@ variable "dataset_labels" { type = map(string) default = {} } -variable "tag_key" { - description = "The ID of the parent TagKey" - type = string - default = "" -} -variable "tag_value" { - description = "The short name of the TagValue" - type = string - default = "" +variable "resource_tags" { + description = "A map of resource tags to add to the dataset" + type = map(string) + default = {} } # Format: list(objects) From e35b335e5d9098b9d474d84b0faee157adf66abe Mon Sep 17 00:00:00 2001 From: Anuj Rohilla Date: Wed, 9 Oct 2024 13:21:52 +0530 Subject: [PATCH 4/9] Upgrade the version constraint --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index bb8bc259..21d80fc4 100644 --- a/versions.tf +++ b/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = ">= 5.3, < 7" + version = ">= 5.39, < 7" } } From d4d372fa525d12159a50fe586df7e72b8d55d676 Mon Sep 17 00:00:00 2001 From: Anuj Rohilla Date: Wed, 16 Oct 2024 16:16:34 +0530 Subject: [PATCH 5/9] docker_generate_docs run --- metadata.display.yaml | 3 + metadata.yaml | 189 ++++++++++++------------ modules/authorization/metadata.yaml | 45 +++--- modules/data_warehouse/metadata.yaml | 45 +++--- modules/scheduled_queries/metadata.yaml | 3 + modules/udf/metadata.yaml | 11 +- 6 files changed, 159 insertions(+), 137 deletions(-) diff --git a/metadata.display.yaml b/metadata.display.yaml index 81c5ec4a..3efb9c25 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -69,6 +69,9 @@ spec: project_id: name: project_id title: Project Id + resource_tags: + name: resource_tags + title: Resource Tags routines: name: routines title: Routines diff --git a/metadata.yaml b/metadata.yaml index ab23c9aa..1c79795d 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -52,26 +52,20 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: access - description: An array of objects that define dataset access for one or more entities. - varType: any - defaultValue: - - role: roles/bigquery.dataOwner - special_group: projectOwners - name: dataset_id description: Unique ID for the dataset being provisioned. varType: string required: true - - name: dataset_labels - description: Key value pairs in a map for dataset labels - varType: map(string) - defaultValue: {} - name: dataset_name description: Friendly name for the dataset being provisioned. varType: string - - name: default_table_expiration_ms - description: TTL of tables using the dataset in MS - varType: number + - name: description + description: Dataset description. + varType: string + - name: location + description: The regional location for the dataset only US and EU are allowed in module + varType: string + defaultValue: US - name: delete_contents_on_destroy description: (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. varType: bool @@ -79,51 +73,72 @@ spec: description: Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion_protection configuration. varType: bool defaultValue: false - - name: description - description: Dataset description. + - name: default_table_expiration_ms + description: TTL of tables using the dataset in MS + varType: number + - name: max_time_travel_hours + description: Defines the time travel window in hours + varType: number + - name: project_id + description: Project where the dataset and table are created varType: string + required: true - name: encryption_key description: Default encryption key to apply to the dataset. Defaults to null (Google-managed). varType: string - - name: external_tables - description: A list of objects which include table_id, expiration_time, external_data_configuration, and labels. + - name: dataset_labels + description: Key value pairs in a map for dataset labels + varType: map(string) + defaultValue: {} + - name: resource_tags + description: A map of resource tags to add to the dataset + varType: map(string) + defaultValue: {} + - name: access + description: An array of objects that define dataset access for one or more entities. + varType: any + defaultValue: + - role: roles/bigquery.dataOwner + special_group: projectOwners + - name: tables + description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. varType: |- list(object({ - table_id = string, - description = optional(string), - autodetect = bool, - compression = string, - ignore_unknown_values = bool, - max_bad_records = number, - schema = string, - source_format = string, - source_uris = list(string), - csv_options = object({ - quote = string, - allow_jagged_rows = bool, - allow_quoted_newlines = bool, - encoding = string, - field_delimiter = string, - skip_leading_rows = number, - }), - google_sheets_options = object({ - range = string, - skip_leading_rows = number, + table_id = string, + description = optional(string), + table_name = optional(string), + schema = string, + clustering = list(string), + require_partition_filter = optional(bool), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, }), - hive_partitioning_options = object({ - mode = string, - source_uri_prefix = string, + range_partitioning = object({ + field = string, + range = object({ + start = string, + end = string, + interval = string, + }), }), expiration_time = string, - max_staleness = optional(string), deletion_protection = optional(bool), labels = map(string), })) defaultValue: [] - - name: location - description: The regional location for the dataset only US and EU are allowed in module - varType: string - defaultValue: US + - name: views + description: A list of objects which include view_id and view query + varType: |- + list(object({ + view_id = string, + description = optional(string), + query = string, + use_legacy_sql = bool, + labels = map(string), + })) + defaultValue: [] - name: materialized_views description: A list of objects which includes view_id, view_query, clustering, time_partitioning, range_partitioning, expiration_time and labels varType: |- @@ -153,13 +168,41 @@ spec: labels = map(string), })) defaultValue: [] - - name: max_time_travel_hours - description: Defines the time travel window in hours - varType: number - - name: project_id - description: Project where the dataset and table are created - varType: string - required: true + - name: external_tables + description: A list of objects which include table_id, expiration_time, external_data_configuration, and labels. + varType: |- + list(object({ + table_id = string, + description = optional(string), + autodetect = bool, + compression = string, + ignore_unknown_values = bool, + max_bad_records = number, + schema = string, + source_format = string, + source_uris = list(string), + csv_options = object({ + quote = string, + allow_jagged_rows = bool, + allow_quoted_newlines = bool, + encoding = string, + field_delimiter = string, + skip_leading_rows = number, + }), + google_sheets_options = object({ + range = string, + skip_leading_rows = number, + }), + hive_partitioning_options = object({ + mode = string, + source_uri_prefix = string, + }), + expiration_time = string, + max_staleness = optional(string), + deletion_protection = optional(bool), + labels = map(string), + })) + defaultValue: [] - name: routines description: A list of objects which include routine_id, routine_type, routine_language, definition_body, return_type, routine_description and arguments. varType: |- @@ -178,45 +221,6 @@ spec: })), })) defaultValue: [] - - name: tables - description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. - varType: |- - list(object({ - table_id = string, - description = optional(string), - table_name = optional(string), - schema = string, - clustering = list(string), - require_partition_filter = optional(bool), - time_partitioning = object({ - expiration_ms = string, - field = string, - type = string, - }), - range_partitioning = object({ - field = string, - range = object({ - start = string, - end = string, - interval = string, - }), - }), - expiration_time = string, - deletion_protection = optional(bool), - labels = map(string), - })) - defaultValue: [] - - name: views - description: A list of objects which include view_id and view query - varType: |- - list(object({ - view_id = string, - description = optional(string), - query = string, - use_legacy_sql = bool, - labels = map(string), - })) - defaultValue: [] outputs: - name: bigquery_dataset description: Bigquery dataset resource. @@ -257,3 +261,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/authorization/metadata.yaml b/modules/authorization/metadata.yaml index b528faa6..d18c7f1f 100644 --- a/modules/authorization/metadata.yaml +++ b/modules/authorization/metadata.yaml @@ -44,6 +44,27 @@ spec: location: examples/scheduled_queries interfaces: variables: + - name: dataset_id + description: Unique ID for the dataset being provisioned. + varType: string + required: true + - name: project_id + description: Project where the dataset and table are created + varType: string + required: true + - name: roles + description: An array of objects that define dataset access for one or more entities. + varType: any + defaultValue: [] + - name: authorized_views + description: An array of views to give authorize for the dataset + varType: |- + list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + defaultValue: [] - name: authorized_datasets description: An array of datasets to be authorized on the dataset varType: |- @@ -61,27 +82,6 @@ spec: routine_id = string })) defaultValue: [] - - name: authorized_views - description: An array of views to give authorize for the dataset - varType: |- - list(object({ - dataset_id = string, - project_id = string, - table_id = string # this is the view id, but we keep table_id to stay consistent as the resource - })) - defaultValue: [] - - name: dataset_id - description: Unique ID for the dataset being provisioned. - varType: string - required: true - - name: project_id - description: Project where the dataset and table are created - varType: string - required: true - - name: roles - description: An array of objects that define dataset access for one or more entities. - varType: any - defaultValue: [] outputs: - name: authorized_dataset description: Authorized datasets for the BQ dataset @@ -104,3 +104,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/data_warehouse/metadata.yaml b/modules/data_warehouse/metadata.yaml index b8810886..eb152de3 100644 --- a/modules/data_warehouse/metadata.yaml +++ b/modules/data_warehouse/metadata.yaml @@ -77,17 +77,26 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: create_ignore_service_accounts - description: Whether or not to ignore creation of a service account if an account of the same name already exists + - name: project_id + description: Google Cloud Project ID varType: string - defaultValue: true + required: true + - name: region + description: Google Cloud Region + varType: string + required: true - name: dataform_region description: Region that is used to deploy Dataform resources. This does not limit where resources can be run or what region data must be located in. varType: string - - name: deletion_protection - description: Whether or not to protect GCS resources from deletion when solution is modified or changed. + - name: text_generation_model_name + description: Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation varType: string - defaultValue: false + defaultValue: text_generate_model + - name: labels + description: A map of labels to apply to contained resources. + varType: map(string) + defaultValue: + data-warehouse: true - name: enable_apis description: Whether or not to enable underlying apis in this solution. varType: string @@ -96,23 +105,14 @@ spec: description: Whether or not to protect BigQuery resources from deletion when solution is modified or changed. varType: string defaultValue: true - - name: labels - description: A map of labels to apply to contained resources. - varType: map(string) - defaultValue: - data-warehouse: true - - name: project_id - description: Google Cloud Project ID - varType: string - required: true - - name: region - description: Google Cloud Region + - name: deletion_protection + description: Whether or not to protect GCS resources from deletion when solution is modified or changed. varType: string - required: true - - name: text_generation_model_name - description: Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation + defaultValue: false + - name: create_ignore_service_accounts + description: Whether or not to ignore creation of a service account if an account of the same name already exists varType: string - defaultValue: text_generate_model + defaultValue: true outputs: - name: bigquery_editor_url description: The URL to launch the BigQuery editor with the sample query procedure opened @@ -139,3 +139,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/scheduled_queries/metadata.yaml b/modules/scheduled_queries/metadata.yaml index d81af5f8..230e3223 100644 --- a/modules/scheduled_queries/metadata.yaml +++ b/modules/scheduled_queries/metadata.yaml @@ -70,3 +70,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/udf/metadata.yaml b/modules/udf/metadata.yaml index be290015..013362cf 100644 --- a/modules/udf/metadata.yaml +++ b/modules/udf/metadata.yaml @@ -44,10 +44,6 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: add_udfs - description: Whether or not this module should be enabled. - varType: string - defaultValue: false - name: dataset_id description: Dataset id varType: string @@ -56,6 +52,10 @@ spec: description: Project ID that contains the dataset varType: string required: true + - name: add_udfs + description: Whether or not this module should be enabled. + varType: string + defaultValue: false outputs: - name: added_udfs description: List of UDFs utility functions added. @@ -74,3 +74,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" From f80a4dd01846674b2491e2b366b8ed1318c2eae1 Mon Sep 17 00:00:00 2001 From: Anuj Rohilla Date: Wed, 16 Oct 2024 16:16:58 +0530 Subject: [PATCH 6/9] Fix lint error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68d110ed..38eefd72 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ module "bigquery" { location = "US" default_table_expiration_ms = 3600000 resource_tags = {"/":""} - + tables = [ { table_id = "foo", From 617f9bbae0912f5e6d4f9672d5a78c7e74885bf9 Mon Sep 17 00:00:00 2001 From: Anuj Rohilla Date: Wed, 16 Oct 2024 16:20:53 +0530 Subject: [PATCH 7/9] docker_generate_docs changes --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 38eefd72..7892fe08 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | 
list(object({
    view_id             = string,
    description         = optional(string),
    query               = string,
    enable_refresh      = bool,
    refresh_interval_ms = string,
    clustering          = list(string),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
      require_partition_filter = bool,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    max_staleness   = optional(string),
    labels          = map(string),
  }))
| `[]` | no | | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | +| resource\_tags | A map of resource tags to add to the dataset | `map(string)` | `{}` | no | | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | 
list(object({
    routine_id      = string,
    routine_type    = string,
    language        = string,
    definition_body = string,
    return_type     = string,
    description     = string,
    arguments = list(object({
      name          = string,
      data_type     = string,
      argument_kind = string,
      mode          = string,
    })),
  }))
| `[]` | no | | tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | 
list(object({
    table_id                 = string,
    description              = optional(string),
    table_name               = optional(string),
    schema                   = string,
    clustering               = list(string),
    require_partition_filter = optional(bool),
    time_partitioning = object({
      expiration_ms = string,
      field         = string,
      type          = string,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time     = string,
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | views | A list of objects which include view\_id and view query | 
list(object({
    view_id        = string,
    description    = optional(string),
    query          = string,
    use_legacy_sql = bool,
    labels         = map(string),
  }))
| `[]` | no | From 6df0eeefb73efc88a754cceb61b75561aab3ee60 Mon Sep 17 00:00:00 2001 From: lord-skinner Date: Wed, 16 Oct 2024 12:17:18 -0500 Subject: [PATCH 8/9] updating version reqs as requested --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index 21d80fc4..4f533898 100644 --- a/versions.tf +++ b/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = ">= 5.39, < 7" + version = "> 5.39, < 7" } } From f754bff5ec623eae0d663521d694e09d4fb13a9f Mon Sep 17 00:00:00 2001 From: Matthew Skinner Date: Wed, 16 Oct 2024 13:25:17 -0500 Subject: [PATCH 9/9] Update versions.tf --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index 4f533898..21d80fc4 100644 --- a/versions.tf +++ b/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = "> 5.39, < 7" + version = ">= 5.39, < 7" } }