Merge pull request #40 from averbuks/averbuks-for-each

Use for_each and make module more flex V3.0

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.0.0] 2019-11-27
+
+-  Add support for `clustering` on a table basis [Issue #26](https://github.com/terraform-google-modules/terraform-google-bigquery/issues/26)
+-  `count` is replaced with for_each [Issue #39](https://github.com/terraform-google-modules/terraform-google-bigquery/issues/39)
+-  The `expiration` variable renamed to `default_table_expiration_ms` [#40]
+-  `expiration_time` can be specified on a table basis [#40]
+-  `partitioning` now can be specified on a table basis and started to be optional, also started supporting additional parameters
+-  Outputs updated to output the whole resources instead of attributes [#40]
+
 ## [2.0.0] 2019-08-02
 
 ### Changed
@@ -34,11 +43,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - This is the initial release of the Big Query module.
 
-[Unreleased]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v2.0.0...HEAD
+[Unreleased]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v3.0.0...HEAD
+[3.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v2.0.0...v3.0.0
 [2.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v1.0.0...v2.0.0
 [1.0.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/compare/v0.1.0...v1.0.0
 [0.1.0]: https://github.com/terraform-google-modules/terraform-google-bigquery/releases/tag/v0.1.0/
 
+[#40]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/40
 [#23]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/23
 [#20]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/20
 [#17]: https://github.com/terraform-google-modules/terraform-google-bigquery/pulls/17

Makefile

@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.6.0
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 

README.md

@@ -8,13 +8,13 @@ Additional user accounts and permissions are necessary to begin querying the new
 
 This module is meant for use with Terraform 0.12. If you haven't
 [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module,
-the last released version intended for Terraform 0.11.x is
-[1.0.0](https://registry.terraform.io/modules/terraform-google-modules/bigquery/google/1.0.0).
+the last released version intended for Terraform 0.11.x is [1.0.0](https://registry.terraform.io/modules/terraform-google-modules/bigquery/google/1.0.0).
 
 ## Upgrading
 
-The current version is 2.X. The following guide is available to assist with upgrades:
+The current version is 3.X. The following guides are available to assist with upgrades:
 
+- [2.0 -> 3.0](./docs/upgrading_to_bigquery_v3.0.md)
 - [1.0 -> 2.0](./docs/upgrading_to_bigquery_v2.0.md)
 - [0.1 -> 1.0](./docs/upgrading_to_bigquery_v1.0.md)
 
@@ -25,30 +25,43 @@ Basic usage of this module is as follows:
 ```hcl
 module "bigquery" {
   source  = "terraform-google-modules/bigquery/google"
-  version = "~> 2.0"
-
-  dataset_id        = "foo"
-  dataset_name      = "foo"
-  description       = "some description"
-  project_id        = "<PROJECT ID>"
-  location          = "US"
-  time_partitioning = "DAY"
+  version = "~> 3.0"
+
+  dataset_id                  = "foo"
+  dataset_name                = "foo"
+  description                 = "some description"
+  project_id                  = "<PROJECT ID>"
+  location                    = "US"
+  default_table_expiration_ms = 3600000
+
   tables = [
-    {
-      table_id = "foo",
-      schema   = "<PATH TO THE SCHEMA JSON FILE>",
-      labels = {
-        env      = "dev"
-        billable = "true"
-      },
+  {
+    table_id          = "foo",
+    schema            =  "<PATH TO THE SCHEMA JSON FILE>",
+    time_partitioning = {
+      type                     = "DAY",
+      field                    = null,
+      require_partition_filter = false,
+      expiration_ms            = null,
     },
-    {
-      table_id = "bar",
-      schema   = "<PATH TO THE SCHEMA JSON FILE>",
-      labels = {
-        env      = "devops"
-        billable = "true"
-      },
+    expiration_time = null,
+    clustering      = ["fullVisitorId", "visitId"],
+    labels          = {
+      env      = "dev"
+      billable = "true"
+      owner    = "joedoe"
+    },
+  },
+  {
+    table_id          = "bar",
+    schema            =  "<PATH TO THE SCHEMA JSON FILE>",
+    time_partitioning = null,
+    expiration_time   = 2524604400000, # 2050/01/01
+    clustering        = [],
+    labels = {
+      env      = "devops"
+      billable = "true"
+      owner    = "joedoe"
     }
   ]
   dataset_labels = {
@@ -61,35 +74,51 @@ module "bigquery" {
 Functional examples are included in the
 [examples](./examples/) directory.
 
+### Variable `tables` detailed description
+
+The `tables` variable should be provided as a list of object with the following keys:
+```hcl
+{
+  table_id = "some_id"                        # Unique table id (will be used as ID and Freandly name for the table).
+  schema = "path/to/schema.json"              # Path to the schema json file.
+  time_partitioning = {                       # Set it to `null` to omit partitioning configuration for the table.
+        type                     = "DAY",     # The only type supported is DAY, which will generate one partition per day based on data loading time.
+        field                    = null,      # The field used to determine how to create a time-based partition. If time-based partitioning is enabled without this value, the table is partitioned based on the load time. Set it to `null` to omit configuration.
+        require_partition_filter = false,     # If set to true, queries over this table require a partition filter that can be used for partition elimination to be specified. Set it to `null` to omit configuration.
+        expiration_ms            = null,      # Number of milliseconds for which to keep the storage for a partition.
+      },
+  clustering = ["fullVisitorId", "visitId"]   # Specifies column names to use for data clustering. Up to four top-level columns are allowed, and should be specified in descending priority order. Partitioning should be configured in order to use clustering.
+  expiration_time = 2524604400000             # The time when this table expires, in milliseconds since the epoch. If set to `null`, the table will persist indefinitely.
+  dataset_labels = {                          # A mapping of labels to assign to the table.
+      env      = "dev"
+      billable = "true"
+    }
+}
+```
+
 ## Features
-This module provisions a dataset and a table with an associated JSON schema.
+This module provisions a dataset and a list of tables with associated JSON schemas.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| dataset\_id | Unique ID for the dataset being provisioned | string | n/a | yes |
+| dataset\_id | Unique ID for the dataset being provisioned. | string | n/a | yes |
 | dataset\_labels | Key value pairs in a map for dataset labels | map(string) | n/a | yes |
-| dataset\_name | Friendly name for the dataset being provisioned | string | n/a | yes |
-| description | Dataset description | string | n/a | yes |
-| expiration | TTL of tables using the dataset in MS | string | `"null"` | no |
+| dataset\_name | Friendly name for the dataset being provisioned. | string | n/a | yes |
+| default\_table\_expiration\_ms | TTL of tables using the dataset in MS | string | `"null"` | no |
+| description | Dataset description. | string | n/a | yes |
 | location | The regional location for the dataset only US and EU are allowed in module | string | `"US"` | no |
 | project\_id | Project where the dataset and table are created | string | n/a | yes |
-| tables | A list of objects which include table_id, schema, and labels. | object | `<list>` | no |
-| time\_partitioning | Configures time-based partitioning for this table | string | n/a | yes |
+| tables | A list of objects which include table_id, schema, clustering, time_partitioning, expiration_time and labels. | object | `<list>` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| dataset\_id | Unique id for the dataset being provisioned |
-| dataset\_labels | Key value pairs in a map for dataset labels |
-| dataset\_name | Friendly name for the dataset being provisioned |
-| dataset\_project | Project where the dataset and table are created |
-| table\_id | Unique id for the table being provisioned |
-| table\_labels | Key value pairs in a map for table labels |
-| table\_name | Friendly name for the table being provisioned |
+| bigquery\_dataset | Bigquery dataset resource. |
+| bigquery\_tables | Map of bigquery table resources being provisioned. |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
@@ -102,7 +131,7 @@ These sections describe requirements for using this module.
 The following dependencies must be available:
 
 - [Terraform][terraform] v0.12
-- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.5
+- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.15
 
 ### Service Account
 

build/int.cloudbuild.yaml

@@ -38,4 +38,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0'

build/lint.cloudbuild.yaml

@@ -21,4 +21,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0'

docs/upgrading_to_bigquery_v3.0.md

@@ -0,0 +1,83 @@
+# Upgrading to BigQuery v3.0
+
+The v3.0 release of BigQuery is a backwards incompatible release.
+- The `time_partitioning` variable has been inlined to the `table` object structure.
+- The `clustering` and `expiration_time` keys were added to the `table` object structure.
+- The `expiration` variable renamed to `default_table_expiration_ms`
+- `count` replaced with `for_each`.
+- Outputs changed to output raw resources.
+
+## Migration Instructions
+
+1. Rename `expiration` variable to `default_table_expiration_ms`.
+```diff
+module "bigquery" {
+  source  = "terraform-google-modules/bigquery/google"
+-  version = "~> 2.0"
+-  version = "~> 3.0"
+
+- expiration = 3600000
++ default_table_expiration_ms = 3600000
+....
+}
+```
+
+2. Delete `time_partitioning` variable and update `tables` variable.
+```diff
+- time_partitioning = "DAY"
+  tables = [
+    {
+      table_id = "foo",
+      schema   = "<PATH TO THE SCHEMA JSON FILE>",
++     time_partitioning = {
++       type                     = "DAY",
++       field                    = null,
++       require_partition_filter = false,
++       expiration_ms            = null,
++     },
++     expiration_time = null,
++     clustering      = [],
+      labels = {
+        env      = "dev"
+        billable = "true"
+      },
+    },
+    {
+      table_id = "bar",
+      schema   = "<PATH TO THE SCHEMA JSON FILE>",
++     time_partitioning = {
++       type                     = "DAY",
++       field                    = null,
++       require_partition_filter = false,
++       expiration_ms            = null,
++     },
++     expiration_time = null,
++     clustering      = [],
+      labels = {
+        env      = "devops"
+        billable = "true"
+      },
+    }
+  ]
+...
+}
+```
+
+3. Replacing `count` with `for_each` requires `removing/importing` resources from/to terraform state.
+Make a backup of terraform state:
+```bash
+terraform init
+terraform state pull > state-backup.json
+```
+Remove table resources from the state:
+```bash
+terraform state rm 'module.bigquery.google_bigquery_table.main[0]'
+terraform state rm 'module.bigquery.google_bigquery_table.main[1]'
+...
+```
+
+Import table resources using table ids:
+```bash
+terraform import 'module.bigquery.google_bigquery_table.main["foo"]' '<PROJECT ID>:<DATASET ID>.foo'
+terraform import 'module.bigquery.google_bigquery_table.main["bar"]' '<PROJECT ID>:<DATASET ID>.bar'
+```

examples/basic_bq/README.md

@@ -6,23 +6,17 @@ The basic_bq example uses the root terraform-google-bigquery module to deploy a
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| dataset\_labels | A mapping of labels to assign to the table | map(string) | n/a | yes |
-| expiration | TTL of tables using the dataset in MS | string | `"null"` | no |
-| project\_id | Project where the dataset and table are created | string | n/a | yes |
-| tables | A list of maps that includes both table_id and schema in each element, the table(s) will be created on the single dataset | object | `<list>` | no |
-| time\_partitioning | Configures time-based partitioning for this table | string | n/a | yes |
+| dataset\_labels | A mapping of labels to assign to the table. | map(string) | n/a | yes |
+| default\_table\_expiration\_ms | Default TTL of tables using the dataset in MS. | string | `"null"` | no |
+| project\_id | Project where the dataset and table are created. | string | n/a | yes |
+| tables | A list of maps that includes table_id, schema, clustering, time_partitioning, expiration_time, labels in each element. | object | `<list>` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| dataset\_id | Unique id for the dataset being provisioned |
-| dataset\_labels | Key value pairs in a map for dataset labels |
-| dataset\_name | Friendly name for the dataset being provisioned |
-| dataset\_project | Project where the dataset and table are created |
-| table\_id | Unique id for the table being provisioned |
-| table\_labels | Key value pairs in a map for table labels |
-| table\_name | Friendly name for the table being provisioned |
+| bigquery\_dataset | Bigquery dataset resource. |
+| bigquery\_tables | Map of bigquery table resources being provisioned. |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

examples/basic_bq/main.tf

@@ -14,23 +14,13 @@
  * limitations under the License.
  */
 
-/******************************************
-   Provider configuration
-  *****************************************/
-provider "google" {
-  version = "~> 2.5.0"
-}
-
 module "bigquery" {
-  source            = "../.."
-  dataset_id        = "foo"
-  dataset_name      = "foo"
-  description       = "some description"
-  expiration        = var.expiration
-  project_id        = var.project_id
-  location          = "US"
-  tables            = var.tables
-  time_partitioning = var.time_partitioning
-  dataset_labels    = var.dataset_labels
+  source         = "../.."
+  dataset_id     = "foo"
+  dataset_name   = "foo"
+  description    = "some description"
+  project_id     = var.project_id
+  location       = "US"
+  tables         = var.tables
+  dataset_labels = var.dataset_labels
 }
-