diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a909001..71c150ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.0.0] 2019-11-27 + +- Add support for `clustering` on a table basis [Issue #26](https://github.com/terraform-google-modules/terraform-google-bigquery/issues/26) +- `count` is replaced with for_each [Issue #39](https://github.com/terraform-google-modules/terraform-google-bigquery/issues/39) +- The `expiration` variable renamed to `default_table_expiration_ms` [#40] +- `expiration_time` can be specified on a table basis [#40] +- `partitioning` now can be specified on a table basis and started to be optional, also started supporting additional parameters +- Outputs updated to output the whole resources instead of attributes [#40] + ## [2.0.0] 2019-08-02 ### Changed @@ -34,11 +43,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - This is the initial release of the Big Query module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v2.0.0...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v3.0.0...HEAD +[3.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v2.0.0...v3.0.0 [2.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v1.0.0...v2.0.0 [1.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v0.1.0...v1.0.0 [0.1.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/releases/tag/v0.1.0/ +[#40]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/40 [#23]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/23 [#20]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/20 [#17]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/17 diff --git a/Makefile b/Makefile index 6a107952..5cbc3ec7 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.6.0 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/README.md b/README.md index d2516510..9e1d3bd8 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ Additional user accounts and permissions are necessary to begin querying the new This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, -the last released version intended for Terraform 0.11.x is -[1.0.0](https://registry.terraform.io/modules/terraform-google-modules/bigquery/google/1.0.0). +the last released version intended for Terraform 0.11.x is [1.0.0](https://registry.terraform.io/modules/terraform-google-modules/bigquery/google/1.0.0). ## Upgrading -The current version is 2.X. The following guide is available to assist with upgrades: +The current version is 3.X. The following guides are available to assist with upgrades: +- [2.0 -> 3.0](./docs/upgrading_to_bigquery_v3.0.md) - [1.0 -> 2.0](./docs/upgrading_to_bigquery_v2.0.md) - [0.1 -> 1.0](./docs/upgrading_to_bigquery_v1.0.md) @@ -25,30 +25,43 @@ Basic usage of this module is as follows: ```hcl module "bigquery" { source = "terraform-google-modules/bigquery/google" - version = "~> 2.0" - - dataset_id = "foo" - dataset_name = "foo" - description = "some description" - project_id = "" - location = "US" - time_partitioning = "DAY" + version = "~> 3.0" + + dataset_id = "foo" + dataset_name = "foo" + description = "some description" + project_id = "" + location = "US" + default_table_expiration_ms = 3600000 + tables = [ - { - table_id = "foo", - schema = "", - labels = { - env = "dev" - billable = "true" - }, + { + table_id = "foo", + schema = "", + time_partitioning = { + type = "DAY", + field = null, + require_partition_filter = false, + expiration_ms = null, }, - { - table_id = "bar", - schema = "", - labels = { - env = "devops" - billable = "true" - }, + expiration_time = null, + clustering = ["fullVisitorId", "visitId"], + labels = { + env = "dev" + billable = "true" + owner = "joedoe" + }, + }, + { + table_id = "bar", + schema = "", + time_partitioning = null, + expiration_time = 2524604400000, # 2050/01/01 + clustering = [], + labels = { + env = "devops" + billable = "true" + owner = "joedoe" } ] dataset_labels = { @@ -61,35 +74,51 @@ module "bigquery" { Functional examples are included in the [examples](./examples/) directory. +### Variable `tables` detailed description + +The `tables` variable should be provided as a list of object with the following keys: +```hcl +{ + table_id = "some_id" # Unique table id (will be used as ID and Freandly name for the table). + schema = "path/to/schema.json" # Path to the schema json file. + time_partitioning = { # Set it to `null` to omit partitioning configuration for the table. + type = "DAY", # The only type supported is DAY, which will generate one partition per day based on data loading time. + field = null, # The field used to determine how to create a time-based partition. If time-based partitioning is enabled without this value, the table is partitioned based on the load time. Set it to `null` to omit configuration. + require_partition_filter = false, # If set to true, queries over this table require a partition filter that can be used for partition elimination to be specified. Set it to `null` to omit configuration. + expiration_ms = null, # Number of milliseconds for which to keep the storage for a partition. + }, + clustering = ["fullVisitorId", "visitId"] # Specifies column names to use for data clustering. Up to four top-level columns are allowed, and should be specified in descending priority order. Partitioning should be configured in order to use clustering. + expiration_time = 2524604400000 # The time when this table expires, in milliseconds since the epoch. If set to `null`, the table will persist indefinitely. + dataset_labels = { # A mapping of labels to assign to the table. + env = "dev" + billable = "true" + } +} +``` + ## Features -This module provisions a dataset and a table with an associated JSON schema. +This module provisions a dataset and a list of tables with associated JSON schemas. ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| dataset\_id | Unique ID for the dataset being provisioned | string | n/a | yes | +| dataset\_id | Unique ID for the dataset being provisioned. | string | n/a | yes | | dataset\_labels | Key value pairs in a map for dataset labels | map(string) | n/a | yes | -| dataset\_name | Friendly name for the dataset being provisioned | string | n/a | yes | -| description | Dataset description | string | n/a | yes | -| expiration | TTL of tables using the dataset in MS | string | `"null"` | no | +| dataset\_name | Friendly name for the dataset being provisioned. | string | n/a | yes | +| default\_table\_expiration\_ms | TTL of tables using the dataset in MS | string | `"null"` | no | +| description | Dataset description. | string | n/a | yes | | location | The regional location for the dataset only US and EU are allowed in module | string | `"US"` | no | | project\_id | Project where the dataset and table are created | string | n/a | yes | -| tables | A list of objects which include table_id, schema, and labels. | object | `` | no | -| time\_partitioning | Configures time-based partitioning for this table | string | n/a | yes | +| tables | A list of objects which include table_id, schema, clustering, time_partitioning, expiration_time and labels. | object | `` | no | ## Outputs | Name | Description | |------|-------------| -| dataset\_id | Unique id for the dataset being provisioned | -| dataset\_labels | Key value pairs in a map for dataset labels | -| dataset\_name | Friendly name for the dataset being provisioned | -| dataset\_project | Project where the dataset and table are created | -| table\_id | Unique id for the table being provisioned | -| table\_labels | Key value pairs in a map for table labels | -| table\_name | Friendly name for the table being provisioned | +| bigquery\_dataset | Bigquery dataset resource. | +| bigquery\_tables | Map of bigquery table resources being provisioned. | @@ -102,7 +131,7 @@ These sections describe requirements for using this module. The following dependencies must be available: - [Terraform][terraform] v0.12 -- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.5 +- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.15 ### Service Account diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index acdecf83..0e60180a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -38,4 +38,4 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 779febf6..cb2cde0a 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -21,4 +21,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0' diff --git a/docs/upgrading_to_bigquery_v3.0.md b/docs/upgrading_to_bigquery_v3.0.md new file mode 100644 index 00000000..ada6ab98 --- /dev/null +++ b/docs/upgrading_to_bigquery_v3.0.md @@ -0,0 +1,83 @@ +# Upgrading to BigQuery v3.0 + +The v3.0 release of BigQuery is a backwards incompatible release. +- The `time_partitioning` variable has been inlined to the `table` object structure. +- The `clustering` and `expiration_time` keys were added to the `table` object structure. +- The `expiration` variable renamed to `default_table_expiration_ms` +- `count` replaced with `for_each`. +- Outputs changed to output raw resources. + +## Migration Instructions + +1. Rename `expiration` variable to `default_table_expiration_ms`. +```diff +module "bigquery" { + source = "terraform-google-modules/bigquery/google" +- version = "~> 2.0" +- version = "~> 3.0" + +- expiration = 3600000 ++ default_table_expiration_ms = 3600000 +.... +} +``` + +2. Delete `time_partitioning` variable and update `tables` variable. +```diff +- time_partitioning = "DAY" + tables = [ + { + table_id = "foo", + schema = "", ++ time_partitioning = { ++ type = "DAY", ++ field = null, ++ require_partition_filter = false, ++ expiration_ms = null, ++ }, ++ expiration_time = null, ++ clustering = [], + labels = { + env = "dev" + billable = "true" + }, + }, + { + table_id = "bar", + schema = "", ++ time_partitioning = { ++ type = "DAY", ++ field = null, ++ require_partition_filter = false, ++ expiration_ms = null, ++ }, ++ expiration_time = null, ++ clustering = [], + labels = { + env = "devops" + billable = "true" + }, + } + ] +... +} +``` + +3. Replacing `count` with `for_each` requires `removing/importing` resources from/to terraform state. +Make a backup of terraform state: +```bash +terraform init +terraform state pull > state-backup.json +``` +Remove table resources from the state: +```bash +terraform state rm 'module.bigquery.google_bigquery_table.main[0]' +terraform state rm 'module.bigquery.google_bigquery_table.main[1]' +... +``` + +Import table resources using table ids: +```bash +terraform import 'module.bigquery.google_bigquery_table.main["foo"]' ':.foo' +terraform import 'module.bigquery.google_bigquery_table.main["bar"]' ':.bar' +``` diff --git a/examples/basic_bq/README.md b/examples/basic_bq/README.md index 76d32846..3ef39b4a 100644 --- a/examples/basic_bq/README.md +++ b/examples/basic_bq/README.md @@ -6,23 +6,17 @@ The basic_bq example uses the root terraform-google-bigquery module to deploy a | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| dataset\_labels | A mapping of labels to assign to the table | map(string) | n/a | yes | -| expiration | TTL of tables using the dataset in MS | string | `"null"` | no | -| project\_id | Project where the dataset and table are created | string | n/a | yes | -| tables | A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset | object | `` | no | -| time\_partitioning | Configures time-based partitioning for this table | string | n/a | yes | +| dataset\_labels | A mapping of labels to assign to the table. | map(string) | n/a | yes | +| default\_table\_expiration\_ms | Default TTL of tables using the dataset in MS. | string | `"null"` | no | +| project\_id | Project where the dataset and table are created. | string | n/a | yes | +| tables | A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element. | object | `` | no | ## Outputs | Name | Description | |------|-------------| -| dataset\_id | Unique id for the dataset being provisioned | -| dataset\_labels | Key value pairs in a map for dataset labels | -| dataset\_name | Friendly name for the dataset being provisioned | -| dataset\_project | Project where the dataset and table are created | -| table\_id | Unique id for the table being provisioned | -| table\_labels | Key value pairs in a map for table labels | -| table\_name | Friendly name for the table being provisioned | +| bigquery\_dataset | Bigquery dataset resource. | +| bigquery\_tables | Map of bigquery table resources being provisioned. | diff --git a/examples/basic_bq/main.tf b/examples/basic_bq/main.tf index 3447be86..457b267e 100644 --- a/examples/basic_bq/main.tf +++ b/examples/basic_bq/main.tf @@ -14,23 +14,13 @@ * limitations under the License. */ -/****************************************** - Provider configuration - *****************************************/ -provider "google" { - version = "~> 2.5.0" -} - module "bigquery" { - source = "../.." - dataset_id = "foo" - dataset_name = "foo" - description = "some description" - expiration = var.expiration - project_id = var.project_id - location = "US" - tables = var.tables - time_partitioning = var.time_partitioning - dataset_labels = var.dataset_labels + source = "../.." + dataset_id = "foo" + dataset_name = "foo" + description = "some description" + project_id = var.project_id + location = "US" + tables = var.tables + dataset_labels = var.dataset_labels } - diff --git a/examples/basic_bq/outputs.tf b/examples/basic_bq/outputs.tf index 92312940..ced44bb2 100644 --- a/examples/basic_bq/outputs.tf +++ b/examples/basic_bq/outputs.tf @@ -14,38 +14,12 @@ * limitations under the License. */ -output "dataset_id" { - value = module.bigquery.dataset_id - description = "Unique id for the dataset being provisioned" +output "bigquery_dataset" { + value = module.bigquery.bigquery_dataset + description = "Bigquery dataset resource." } -output "dataset_name" { - value = module.bigquery.dataset_name - description = "Friendly name for the dataset being provisioned" +output "bigquery_tables" { + value = module.bigquery.bigquery_tables + description = "Map of bigquery table resources being provisioned." } - -output "dataset_project" { - value = module.bigquery.dataset_project - description = "Project where the dataset and table are created" -} - -output "table_id" { - value = module.bigquery.table_id - description = "Unique id for the table being provisioned" -} - -output "table_name" { - value = module.bigquery.table_name - description = "Friendly name for the table being provisioned" -} - -output "dataset_labels" { - value = module.bigquery.dataset_labels - description = "Key value pairs in a map for dataset labels" -} - -output "table_labels" { - value = module.bigquery.table_labels - description = "Key value pairs in a map for table labels" -} - diff --git a/examples/basic_bq/terraform.tfvars b/examples/basic_bq/terraform.tfvars index e48dfc2a..e2cab79b 100644 --- a/examples/basic_bq/terraform.tfvars +++ b/examples/basic_bq/terraform.tfvars @@ -1,5 +1,4 @@ -project_id = "example-project" -time_partitioning = "DAY" +project_id = "example-project" dataset_labels = { env = "dev" billable = "true" @@ -7,14 +6,17 @@ dataset_labels = { } tables = [ { - table_id = "foo", - schema = "sample_bq_schema.json", + table_id = "bar", + schema = "sample_bq_schema.json", + time_partitioning = null, + expiration_time = 2524604400000, # 2050/01/01 + clustering = [], labels = { - env = "dev" + env = "devops" billable = "true" owner = "joedoe" }, - }, + } ] diff --git a/examples/basic_bq/variables.tf b/examples/basic_bq/variables.tf index f0c9b33c..15059721 100644 --- a/examples/basic_bq/variables.tf +++ b/examples/basic_bq/variables.tf @@ -14,30 +14,34 @@ * limitations under the License. */ -variable "expiration" { - description = "TTL of tables using the dataset in MS" +variable "default_table_expiration_ms" { + description = "Default TTL of tables using the dataset in MS." default = null } variable "project_id" { - description = "Project where the dataset and table are created" -} - -variable "time_partitioning" { - description = "Configures time-based partitioning for this table" + description = "Project where the dataset and table are created." } variable "dataset_labels" { - description = "A mapping of labels to assign to the table" + description = "A mapping of labels to assign to the table." type = map(string) } variable "tables" { - description = "A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset" + description = "A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element." default = [] type = list(object({ - table_id = string, - schema = string, - labels = map(string), + table_id = string, + schema = string, + clustering = list(string), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, + require_partition_filter = bool, + }), + expiration_time = string, + labels = map(string), })) } diff --git a/examples/multiple_tables/README.md b/examples/multiple_tables/README.md index 18e5e67b..e5c1ecb5 100644 --- a/examples/multiple_tables/README.md +++ b/examples/multiple_tables/README.md @@ -8,23 +8,17 @@ This example is a good reference to understand and test the module usage. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| dataset\_labels | A mapping of labels to assign to the table | map(string) | n/a | yes | -| expiration | TTL of tables using the dataset in MS | string | `"null"` | no | -| project\_id | Project where the dataset and table are created | string | n/a | yes | -| tables | A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset | object | `` | no | -| time\_partitioning | Configures time-based partitioning for this table | string | n/a | yes | +| dataset\_labels | A mapping of labels to assign to the table. | map(string) | n/a | yes | +| default\_table\_expiration\_ms | Default TTL of tables using the dataset in MS. | string | `"null"` | no | +| project\_id | Project where the dataset and table are created. | string | n/a | yes | +| tables | A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element. | object | `` | no | ## Outputs | Name | Description | |------|-------------| -| dataset\_id | Unique id for the dataset being provisioned | -| dataset\_labels | Key value pairs in a map for dataset labels | -| dataset\_name | Friendly name for the dataset being provisioned | -| dataset\_project | Project where the dataset and table are created | -| table\_id | Unique id for the table being provisioned | -| table\_labels | Key value pairs in a map for table labels | -| table\_name | Friendly name for the table being provisioned | +| bigquery\_dataset | Bigquery dataset resource. | +| bigquery\_tables | Map of bigquery table resources being provisioned. | diff --git a/examples/multiple_tables/main.tf b/examples/multiple_tables/main.tf index 3447be86..1c790fca 100644 --- a/examples/multiple_tables/main.tf +++ b/examples/multiple_tables/main.tf @@ -14,23 +14,14 @@ * limitations under the License. */ -/****************************************** - Provider configuration - *****************************************/ -provider "google" { - version = "~> 2.5.0" -} - module "bigquery" { - source = "../.." - dataset_id = "foo" - dataset_name = "foo" - description = "some description" - expiration = var.expiration - project_id = var.project_id - location = "US" - tables = var.tables - time_partitioning = var.time_partitioning - dataset_labels = var.dataset_labels + source = "../.." + dataset_id = "foo" + dataset_name = "foo" + description = "some description" + default_table_expiration_ms = var.default_table_expiration_ms + project_id = var.project_id + location = "US" + tables = var.tables + dataset_labels = var.dataset_labels } - diff --git a/examples/multiple_tables/outputs.tf b/examples/multiple_tables/outputs.tf index 92312940..ced44bb2 100644 --- a/examples/multiple_tables/outputs.tf +++ b/examples/multiple_tables/outputs.tf @@ -14,38 +14,12 @@ * limitations under the License. */ -output "dataset_id" { - value = module.bigquery.dataset_id - description = "Unique id for the dataset being provisioned" +output "bigquery_dataset" { + value = module.bigquery.bigquery_dataset + description = "Bigquery dataset resource." } -output "dataset_name" { - value = module.bigquery.dataset_name - description = "Friendly name for the dataset being provisioned" +output "bigquery_tables" { + value = module.bigquery.bigquery_tables + description = "Map of bigquery table resources being provisioned." } - -output "dataset_project" { - value = module.bigquery.dataset_project - description = "Project where the dataset and table are created" -} - -output "table_id" { - value = module.bigquery.table_id - description = "Unique id for the table being provisioned" -} - -output "table_name" { - value = module.bigquery.table_name - description = "Friendly name for the table being provisioned" -} - -output "dataset_labels" { - value = module.bigquery.dataset_labels - description = "Key value pairs in a map for dataset labels" -} - -output "table_labels" { - value = module.bigquery.table_labels - description = "Key value pairs in a map for table labels" -} - diff --git a/examples/multiple_tables/terraform.tfvars b/examples/multiple_tables/terraform.tfvars index 12b34602..e93a26da 100644 --- a/examples/multiple_tables/terraform.tfvars +++ b/examples/multiple_tables/terraform.tfvars @@ -1,5 +1,5 @@ -project_id = "example-project" -time_partitioning = "DAY" +project_id = "example-project" +default_table_expiration_ms = 3600000 dataset_labels = { env = "dev" billable = "true" @@ -9,6 +9,14 @@ tables = [ { table_id = "foo", schema = "sample_bq_schema.json", + time_partitioning = { + type = "DAY", + field = null, + require_partition_filter = false, + expiration_ms = null, + }, + expiration_time = null, + clustering = ["fullVisitorId", "visitId"], labels = { env = "dev" billable = "true" @@ -16,8 +24,11 @@ tables = [ }, }, { - table_id = "bar", - schema = "sample_bq_schema.json", + table_id = "bar", + schema = "sample_bq_schema.json", + time_partitioning = null, + expiration_time = 2524604400000, # 2050/01/01 + clustering = [], labels = { env = "devops" billable = "true" diff --git a/examples/multiple_tables/variables.tf b/examples/multiple_tables/variables.tf index eccab3dd..15059721 100644 --- a/examples/multiple_tables/variables.tf +++ b/examples/multiple_tables/variables.tf @@ -14,31 +14,34 @@ * limitations under the License. */ -variable "expiration" { - description = "TTL of tables using the dataset in MS" +variable "default_table_expiration_ms" { + description = "Default TTL of tables using the dataset in MS." default = null } variable "project_id" { - description = "Project where the dataset and table are created" -} - -variable "time_partitioning" { - description = "Configures time-based partitioning for this table" + description = "Project where the dataset and table are created." } variable "dataset_labels" { - description = "A mapping of labels to assign to the table" + description = "A mapping of labels to assign to the table." type = map(string) } variable "tables" { - description = "A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset" + description = "A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element." default = [] type = list(object({ - table_id = string, - schema = string, - labels = map(string), + table_id = string, + schema = string, + clustering = list(string), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, + require_partition_filter = bool, + }), + expiration_time = string, + labels = map(string), })) } - diff --git a/main.tf b/main.tf index abcb0029..e245faf2 100644 --- a/main.tf +++ b/main.tf @@ -14,28 +14,38 @@ * limitations under the License. */ +locals { + tables = { for table in var.tables : table["table_id"] => table } +} + resource "google_bigquery_dataset" "main" { dataset_id = var.dataset_id friendly_name = var.dataset_name description = var.description location = var.location - default_table_expiration_ms = var.expiration + default_table_expiration_ms = var.default_table_expiration_ms project = var.project_id labels = var.dataset_labels } resource "google_bigquery_table" "main" { - count = length(var.tables) - dataset_id = google_bigquery_dataset.main.dataset_id - friendly_name = var.tables[count.index]["table_id"] - table_id = var.tables[count.index]["table_id"] - labels = var.tables[count.index]["labels"] - schema = file(var.tables[count.index]["schema"]) - project = var.project_id - - time_partitioning { - type = var.time_partitioning + for_each = local.tables + dataset_id = google_bigquery_dataset.main.dataset_id + friendly_name = each.key + table_id = each.key + labels = each.value["labels"] + schema = file(each.value["schema"]) + clustering = each.value["clustering"] + expiration_time = each.value["expiration_time"] + project = var.project_id + dynamic "time_partitioning" { + for_each = each.value["time_partitioning"] != null ? [each.value["time_partitioning"]] : [] + content { + type = time_partitioning.value["type"] + expiration_ms = time_partitioning.value["expiration_ms"] + field = time_partitioning.value["field"] + require_partition_filter = time_partitioning.value["require_partition_filter"] + } } } - diff --git a/outputs.tf b/outputs.tf index 5a574832..25b26e73 100644 --- a/outputs.tf +++ b/outputs.tf @@ -14,38 +14,12 @@ * limitations under the License. */ -output "dataset_id" { - value = google_bigquery_dataset.main.id - description = "Unique id for the dataset being provisioned" +output "bigquery_dataset" { + value = google_bigquery_dataset.main + description = "Bigquery dataset resource." } -output "dataset_name" { - value = google_bigquery_dataset.main.friendly_name - description = "Friendly name for the dataset being provisioned" +output "bigquery_tables" { + value = google_bigquery_table.main + description = "Map of bigquery table resources being provisioned." } - -output "dataset_project" { - value = google_bigquery_dataset.main.project - description = "Project where the dataset and table are created" -} - -output "table_id" { - value = google_bigquery_table.main.*.id - description = "Unique id for the table being provisioned" -} - -output "table_name" { - value = google_bigquery_table.main.*.friendly_name - description = "Friendly name for the table being provisioned" -} - -output "dataset_labels" { - value = google_bigquery_dataset.main.labels - description = "Key value pairs in a map for dataset labels" -} - -output "table_labels" { - value = google_bigquery_table.main.*.labels - description = "Key value pairs in a map for table labels" -} - diff --git a/test/fixtures/full/main.tf b/test/fixtures/full/main.tf index 2e9740f4..af8474d4 100644 --- a/test/fixtures/full/main.tf +++ b/test/fixtures/full/main.tf @@ -14,19 +14,10 @@ * limitations under the License. */ -/****************************************** - Provider configuration - *****************************************/ -provider "google" { - version = "~> 2.5.0" -} - module "example" { - source = "../../../examples/multiple_tables" - expiration = var.expiration - project_id = var.project_id - tables = var.tables - time_partitioning = var.time_partitioning - dataset_labels = var.dataset_labels + source = "../../../examples/multiple_tables" + default_table_expiration_ms = var.default_table_expiration_ms + project_id = var.project_id + tables = var.tables + dataset_labels = var.dataset_labels } - diff --git a/test/fixtures/full/outputs.tf b/test/fixtures/full/outputs.tf index b32a6a3d..05c574da 100644 --- a/test/fixtures/full/outputs.tf +++ b/test/fixtures/full/outputs.tf @@ -14,38 +14,12 @@ * limitations under the License. */ -output "dataset_id" { - value = module.example.dataset_id - description = "Unique id for the dataset being provisioned" +output "bigquery_dataset" { + value = module.example.bigquery_dataset + description = "Bigquery dataset resource." } -output "dataset_name" { - value = module.example.dataset_name - description = "Friendly name for the dataset being provisioned" +output "bigquery_tables" { + value = module.example.bigquery_tables + description = "Map of bigquery table resources being provisioned." } - -output "dataset_project" { - value = module.example.dataset_project - description = "Project where the dataset and table are created" -} - -output "table_id" { - value = module.example.table_id - description = "Unique id for the table being provisioned" -} - -output "table_name" { - value = module.example.table_name - description = "Friendly name for the table being provisioned" -} - -output "dataset_labels" { - value = module.example.dataset_labels - description = "Key value pairs in a map for dataset labels" -} - -output "table_labels" { - value = module.example.table_labels - description = "Key value pairs in a map for table labels" -} - diff --git a/test/fixtures/full/terraform.tfvars b/test/fixtures/full/terraform.tfvars index 5220dfcc..54664fdc 100644 --- a/test/fixtures/full/terraform.tfvars +++ b/test/fixtures/full/terraform.tfvars @@ -1,5 +1,4 @@ -expiration = 3600000 -time_partitioning = "DAY" +default_table_expiration_ms = 3600000 dataset_labels = { env = "dev" billable = "true" @@ -9,6 +8,14 @@ tables = [ { table_id = "foo", schema = "sample_bq_schema.json", + time_partitioning = { + type = "DAY", + field = null, + require_partition_filter = false, + expiration_ms = null, + }, + expiration_time = null, + clustering = ["fullVisitorId", "visitId"], labels = { env = "dev" billable = "true" @@ -16,8 +23,11 @@ tables = [ }, }, { - table_id = "bar", - schema = "sample_bq_schema.json", + table_id = "bar", + schema = "sample_bq_schema.json", + time_partitioning = null, + expiration_time = 2524604400000, # 2050/01/01 + clustering = [], labels = { env = "devops" billable = "true" diff --git a/test/fixtures/full/variables.tf b/test/fixtures/full/variables.tf index 89ed48f1..f0a71472 100644 --- a/test/fixtures/full/variables.tf +++ b/test/fixtures/full/variables.tf @@ -14,30 +14,33 @@ * limitations under the License. */ -variable "expiration" { - description = "TTL of tables using the dataset in MS" - default = null +variable "default_table_expiration_ms" { + description = "Default TTL of tables using the dataset in MS" } variable "project_id" { description = "Project where the dataset and table are created" } -variable "time_partitioning" { - description = "Configures time-based partitioning for this table" -} - variable "dataset_labels" { description = "Key value pairs in a map for dataset labels" type = map(string) } variable "tables" { - description = "A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset" + description = "A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element" default = [] type = list(object({ - table_id = string, - schema = string, - labels = map(string), + table_id = string, + schema = string, + clustering = list(string), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, + require_partition_filter = bool, + }), + expiration_time = string, + labels = map(string), })) } diff --git a/test/integration/full/controls/big_query.rb b/test/integration/full/controls/big_query.rb index f6c80fde..09f835bb 100644 --- a/test/integration/full/controls/big_query.rb +++ b/test/integration/full/controls/big_query.rb @@ -13,9 +13,9 @@ # limitations under the License. # Attributes can be used to create tests with as the mode becomes more complex -project_id = attribute('dataset_project') -dataset_name = attribute('dataset_name') -table_name = attribute('table_name') +project_id = attribute('bigquery_dataset')['project'] +dataset_name = attribute('bigquery_dataset')['friendly_name'] +tables = attribute('bigquery_tables') describe google_bigquery_dataset(project: "#{project_id}", name: "#{dataset_name}") do it { should exist } @@ -26,14 +26,16 @@ its('default_table_expiration_ms') { should cmp '3600000' } end -describe google_bigquery_table(project: "#{project_id}", dataset: "#{dataset_name}", name: "#{table_name[0]}") do +describe google_bigquery_table(project: "#{project_id}", dataset: "#{dataset_name}", name: "#{tables["foo"]["friendly_name"]}") do it { should exist } - its('friendly_name') { should eq "#{table_name[0]}" } + its('friendly_name') { should eq "#{tables["foo"]["friendly_name"]}" } its('time_partitioning.type') { should eq 'DAY' } + its('clustering') { should_not be nil } end -describe google_bigquery_table(project: "#{project_id}", dataset: "#{dataset_name}", name: "#{table_name[1]}") do +describe google_bigquery_table(project: "#{project_id}", dataset: "#{dataset_name}", name: "#{tables["bar"]["friendly_name"]}") do it { should exist } - its('friendly_name') { should eq "#{table_name[1]}" } - its('time_partitioning.type') { should eq 'DAY' } + its('friendly_name') { should eq "#{tables["bar"]["friendly_name"]}" } + its('time_partitioning.type') { should be nil } + its('clustering') { should be nil } end diff --git a/test/integration/full/inspec.yml b/test/integration/full/inspec.yml index 48174d27..f513bbcc 100644 --- a/test/integration/full/inspec.yml +++ b/test/integration/full/inspec.yml @@ -7,12 +7,9 @@ depends: supports: - platform: gcp attributes: - - name: dataset_project + - name: bigquery_dataset required: true - type: string - - name: dataset_name + type: hash + - name: bigquery_tables required: true - type: string - - name: table_name - required: true - type: array + type: hash diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh deleted file mode 100755 index ffdc48e1..00000000 --- a/test/setup/make_source.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "#!/usr/bin/env bash" > ../source.sh - -project_id=$(terraform output project_id) -echo "export TF_VAR_project_id='$project_id'" >> ../source.sh - -sa_json=$(terraform output sa_key) -# shellcheck disable=SC2086 -echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh diff --git a/variables.tf b/variables.tf index d4a0cdb5..491f8068 100644 --- a/variables.tf +++ b/variables.tf @@ -15,15 +15,15 @@ */ variable "dataset_id" { - description = "Unique ID for the dataset being provisioned" + description = "Unique ID for the dataset being provisioned." } variable "dataset_name" { - description = "Friendly name for the dataset being provisioned" + description = "Friendly name for the dataset being provisioned." } variable "description" { - description = "Dataset description" + description = "Dataset description." } variable "location" { @@ -31,7 +31,7 @@ variable "location" { default = "US" } -variable "expiration" { +variable "default_table_expiration_ms" { description = "TTL of tables using the dataset in MS" default = null } @@ -40,22 +40,25 @@ variable "project_id" { description = "Project where the dataset and table are created" } -variable "time_partitioning" { - description = "Configures time-based partitioning for this table" -} - variable "dataset_labels" { description = "Key value pairs in a map for dataset labels" type = map(string) } variable "tables" { - description = "A list of objects which include table_id, schema, and labels." + description = "A list of objects which include table_id, schema, clustering, time_partitioning, expiration_time and labels." default = [] type = list(object({ - table_id = string, - schema = string, - labels = map(string), + table_id = string, + schema = string, + clustering = list(string), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, + require_partition_filter = bool, + }), + expiration_time = string, + labels = map(string), })) } - diff --git a/versions.tf b/versions.tf new file mode 100644 index 00000000..b44c744c --- /dev/null +++ b/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.6" + required_providers { + google = "~> 2.15" + } +}