From 7d5b02b4d627c388fed3146f36062827927231fa Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 29 Apr 2019 23:51:44 +0000 Subject: [PATCH 01/13] added subnetwork and network arguments to module --- examples/simple_example/main.tf | 12 ++++++------ main.tf | 2 ++ modules/dataflow_bucket/outputs.tf | 2 +- variables.tf | 10 ++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index e7a91de..6eb66a9 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -28,16 +28,16 @@ locals { } module "dataflow-bucket" { - source = "../../modules/dataflow_bucket" - name = "${local.gcs_bucket_name}" - region = "${var.region}" - project_id = "${var.project_id}" + source = "../../modules/dataflow_bucket" + name = "${local.gcs_bucket_name}" + region = "${var.region}" + project_id = "${var.project_id}" } module "dataflow-job" { source = "../../" project_id = "${var.project_id}" - name = "wordcount-terraform-example" + name = "wordcount-terraform-example" on_delete = "cancel" zone = "${var.region}-a" max_workers = 1 @@ -54,7 +54,7 @@ module "dataflow-job" { module "dataflow-job-2" { source = "../../" project_id = "${var.project_id}" - name = "wordcount-terraform-example-2" + name = "wordcount-terraform-example-2" on_delete = "cancel" zone = "${var.region}-a" max_workers = 1 diff --git a/main.tf b/main.tf index e0c4e39..22052d5 100644 --- a/main.tf +++ b/main.tf @@ -25,4 +25,6 @@ resource "google_dataflow_job" "dataflow_job" { temp_gcs_location = "gs://${var.temp_gcs_location}/tmp_dir" parameters = "${var.parameters}" service_account_email = "${var.service_account_email}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" } diff --git a/modules/dataflow_bucket/outputs.tf b/modules/dataflow_bucket/outputs.tf index fd94c16..44b184e 100644 --- a/modules/dataflow_bucket/outputs.tf +++ b/modules/dataflow_bucket/outputs.tf @@ -6,4 +6,4 @@ output "name" { output "region" { description = "The bucket's region location" value = "${var.region}" -} \ No newline at end of file +} diff --git a/variables.tf b/variables.tf index bd09066..83bbc8b 100644 --- a/variables.tf +++ b/variables.tf @@ -58,3 +58,13 @@ variable "region" { description = "The bucket's region location" default = "us-central1" } + +variable "subnetwork" { + description = "The subnetwork to which VMs will be assigned. Should be of the form \"regions/REGION/subnetworks/SUBNETWORK\"." + default = "" +} + +variable "network" { + description = "The network to which VMs will be assigned. If it is not provided, \"default\" will be used." + default = "" +} From d0204c2e6770a28698874855934a647550f831ec Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 29 Apr 2019 23:53:11 +0000 Subject: [PATCH 02/13] make generate_docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8090dc0..4f595e5 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,13 @@ Then perform the following commands on the root folder: |------|-------------|:----:|:-----:|:-----:| | max\_workers | The number of workers permitted to work on the job. More workers may improve processing speed at additional cost. | string | `"1"` | no | | name | The name of the dataflow job | string | n/a | yes | +| network | The network to which VMs will be assigned. If it is not provided, "default" will be used. | string | `""` | no | | on\_delete | One of drain or cancel. Specifies behavior of deletion during terraform destroy. The default is cancel. | string | `"cancel"` | no | | parameters | Key/Value pairs to be passed to the Dataflow job (as used in the template). | map | `` | no | | project\_id | The project in which the resource belongs. If it is not provided, the provider project is used. | string | n/a | yes | | region | The bucket's region location | string | `"us-central1"` | no | | service\_account\_email | The Service Account email that will be used to identify the VMs in which the jobs are running | string | `""` | no | +| subnetwork | The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". | string | `""` | no | | temp\_gcs\_location | A writeable location on GCS for the Dataflow job to dump its temporary data. | string | n/a | yes | | template\_gcs\_path | The GCS path to the Dataflow job template. | string | n/a | yes | | zone | The zone in which the created job should run. If it is not provided, the provider zone is used. | string | `"us-central1-a"` | no | From fa133080433599672631e9c1142620aa7c4bd8f5 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 31 May 2019 00:40:28 -0400 Subject: [PATCH 03/13] added network and subnetwork vars to module and inside example --- examples/simple_example/main.tf | 35 ++++++++++++++++++++++++---- examples/simple_example/variables.tf | 5 ++++ modules/dataflow_bucket/main.tf | 1 + modules/dataflow_bucket/variables.tf | 5 ++++ test/fixtures/simple_example/main.tf | 1 + 5 files changed, 43 insertions(+), 4 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 6eb66a9..ed4404c 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -27,11 +27,34 @@ locals { gcs_bucket_name = "tmp-dir-bucket-${random_id.random_suffix.hex}" } +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 0.4.0" + project_id = "${var.project_id}" + network_name = "dataflow-network" + + subnets = [ + { + subnet_name = "dataflow-subnetwork" + subnet_ip = "10.1.3.0/24" + subnet_region = "us-central1" + }, + ] + + secondary_ranges = { + dataflow-subnetwork = [{ + range_name = "my-secondary-range" + ip_cidr_range = "192.168.64.0/24" + }] + } +} + module "dataflow-bucket" { - source = "../../modules/dataflow_bucket" - name = "${local.gcs_bucket_name}" - region = "${var.region}" - project_id = "${var.project_id}" + source = "../../modules/dataflow_bucket" + name = "${local.gcs_bucket_name}" + region = "${var.region}" + project_id = "${var.project_id}" + force_destroy = "${var.force_destroy}" } module "dataflow-job" { @@ -44,6 +67,8 @@ module "dataflow-job" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" + network = "${module.vpc.network_name}" + subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" @@ -61,6 +86,8 @@ module "dataflow-job-2" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" + network = "${module.vpc.network_name}" + subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" diff --git a/examples/simple_example/variables.tf b/examples/simple_example/variables.tf index 532ecee..06df9db 100644 --- a/examples/simple_example/variables.tf +++ b/examples/simple_example/variables.tf @@ -25,3 +25,8 @@ variable "region" { variable "service_account_email" { description = "The Service Account email used to create the job." } + +variable "force_destroy" { + description = "When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run." + default = "false" +} diff --git a/modules/dataflow_bucket/main.tf b/modules/dataflow_bucket/main.tf index 0db3446..b051cee 100644 --- a/modules/dataflow_bucket/main.tf +++ b/modules/dataflow_bucket/main.tf @@ -20,4 +20,5 @@ resource "google_storage_bucket" "tmp_dir_bucket" { location = "${var.region}" storage_class = "REGIONAL" project = "${var.project_id}" + force_destroy = "${var.force_destroy}" } diff --git a/modules/dataflow_bucket/variables.tf b/modules/dataflow_bucket/variables.tf index f231cae..5faaa70 100644 --- a/modules/dataflow_bucket/variables.tf +++ b/modules/dataflow_bucket/variables.tf @@ -10,3 +10,8 @@ variable "region" { variable "name" { description = "The name of the bucket." } + +variable "force_destroy" { + description = "When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run." + default = "false" +} diff --git a/test/fixtures/simple_example/main.tf b/test/fixtures/simple_example/main.tf index afc278a..293b8db 100644 --- a/test/fixtures/simple_example/main.tf +++ b/test/fixtures/simple_example/main.tf @@ -19,4 +19,5 @@ module "example" { project_id = "${var.project_id}" region = "${var.region}" service_account_email = "${var.service_account_email}" + force_destroy = "true" } From 215de47479fca3192526cf2b9df740302a7d5bdc Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 31 May 2019 01:21:19 -0400 Subject: [PATCH 04/13] modified readme --- README.md | 3 ++- examples/simple_example/README.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f595e5..1a81e4e 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ If you want to use the service_account_email input to specify a service account ### Enable APIs In order to launch a Dataflow Job, the Dataflow API must be enabled: -- Dataflow API - dataflow.googleapis.com +- Dataflow API - `dataflow.googleapis.com` +- Compute Engine API: `compute.googleapis.com` ## Install diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index 8087dec..7e755a5 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -9,6 +9,12 @@ This example illustrates how to use the Dataflow module to start multiple jobs w As featured in this example, using a single regional bucket for storing your jobs' temporary data is recommended to optimize cost. Also, to optimize your jobs performance, this bucket should always in the corresponding region of the zones in which your jobs are running. +## Running the example +Make sure you grant the addtional permissions below to the service account execute the module: + +- roles/compute.networkAdmin + + ### Controller Service Account This example features the use of a controller service accoun which is specified with the `service_account_email` input variables. We recommend using a custome service account with fine-grained access control to mitigate security risks. See more about controller service accounts [here](https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#controller_service_account) From b59cb1f0da4fe79437e16dd5194d50c588cf121b Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 31 May 2019 02:13:46 -0400 Subject: [PATCH 05/13] README --- examples/simple_example/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index 7e755a5..a705909 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -1,6 +1,7 @@ # Simple Example This example illustrates how to use the Dataflow module to start multiple jobs with a common bucket for temporary job data. +A network and subnetwork are created as well to demonstrate how Dataflow instance can be created in a specific network and subnetwork. ## Best practices From 65b330cf729fa4b7741339d2f83094ba3a570dc1 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 10 Jun 2019 16:28:13 -0500 Subject: [PATCH 06/13] added machine_type argument to root module --- examples/simple_example/main.tf | 2 ++ main.tf | 1 + variables.tf | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index ed4404c..c4a009c 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -69,6 +69,7 @@ module "dataflow-job" { service_account_email = "${var.service_account_email}" network = "${module.vpc.network_name}" subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" + machine_type = "n1-standard-1" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" @@ -88,6 +89,7 @@ module "dataflow-job-2" { service_account_email = "${var.service_account_email}" network = "${module.vpc.network_name}" subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" + machine_type = "n1-standard-2" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" diff --git a/main.tf b/main.tf index 22052d5..abe8de8 100644 --- a/main.tf +++ b/main.tf @@ -27,4 +27,5 @@ resource "google_dataflow_job" "dataflow_job" { service_account_email = "${var.service_account_email}" network = "${var.network}" subnetwork = "${var.subnetwork}" + machine_type = "${var.machine_type}" } diff --git a/variables.tf b/variables.tf index 83bbc8b..2c2711c 100644 --- a/variables.tf +++ b/variables.tf @@ -68,3 +68,8 @@ variable "network" { description = "The network to which VMs will be assigned. If it is not provided, \"default\" will be used." default = "" } + +variable "machine_type" { + description = "The machine type to use for the job." + default = "" +} From cf5829ab3b9407dbfbac16814b17d744f87c59ba Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 14 Jun 2019 11:49:02 -0500 Subject: [PATCH 07/13] fixed subnetwork argument to take self link. Added machine type argument. --- examples/simple_example/main.tf | 8 ++++---- main.tf | 2 +- variables.tf | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index c4a009c..3e6e0bb 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -29,7 +29,7 @@ locals { module "vpc" { source = "terraform-google-modules/network/google" - version = "~> 0.4.0" + version = "~> 0.8.0" project_id = "${var.project_id}" network_name = "dataflow-network" @@ -68,8 +68,8 @@ module "dataflow-job" { temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" network = "${module.vpc.network_name}" - subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" - machine_type = "n1-standard-1" + subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" + machine_type = "n1-standard-1" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" @@ -88,7 +88,7 @@ module "dataflow-job-2" { temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" network = "${module.vpc.network_name}" - subnetwork = "regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}" + subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-2" parameters = { diff --git a/main.tf b/main.tf index abe8de8..d2e2cb5 100644 --- a/main.tf +++ b/main.tf @@ -26,6 +26,6 @@ resource "google_dataflow_job" "dataflow_job" { parameters = "${var.parameters}" service_account_email = "${var.service_account_email}" network = "${var.network}" - subnetwork = "${var.subnetwork}" + subnetwork = "${replace(var.subnetwork_self_link, "/(.*)/regions/(.*)/", "regions/$2")}" machine_type = "${var.machine_type}" } diff --git a/variables.tf b/variables.tf index 2c2711c..35a9ce2 100644 --- a/variables.tf +++ b/variables.tf @@ -59,14 +59,14 @@ variable "region" { default = "us-central1" } -variable "subnetwork" { - description = "The subnetwork to which VMs will be assigned. Should be of the form \"regions/REGION/subnetworks/SUBNETWORK\"." +variable "subnetwork_self_link" { + description = "The subnetwork self link to which VMs will be assigned." default = "" } variable "network" { - description = "The network to which VMs will be assigned. If it is not provided, \"default\" will be used." - default = "" + description = "The network to which VMs will be assigned." + default = "default" } variable "machine_type" { From 5b9f2cbc01e77e023675bff2142c077a9a69d71f Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 14 Jun 2019 14:07:36 -0500 Subject: [PATCH 08/13] replaced network variable to network_self_link --- examples/simple_example/main.tf | 10 +++++----- main.tf | 2 +- variables.tf | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 3e6e0bb..51c6ae9 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -67,9 +67,9 @@ module "dataflow-job" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network = "${module.vpc.network_name}" - subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" - machine_type = "n1-standard-1" + network_self_link = "${module.vpc.network_self_link}" + subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" + machine_type = "n1-standard-1" parameters = { inputFile = "gs://dataflow-samples/shakespeare/kinglear.txt" @@ -87,8 +87,8 @@ module "dataflow-job-2" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network = "${module.vpc.network_name}" - subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" + network_self_link = "${module.vpc.network_self_link}" + subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-2" parameters = { diff --git a/main.tf b/main.tf index d2e2cb5..84eb0c7 100644 --- a/main.tf +++ b/main.tf @@ -25,7 +25,7 @@ resource "google_dataflow_job" "dataflow_job" { temp_gcs_location = "gs://${var.temp_gcs_location}/tmp_dir" parameters = "${var.parameters}" service_account_email = "${var.service_account_email}" - network = "${var.network}" + network = "${replace(var.network_self_link, "/(.*)/networks/(.*)/", "$2")}" subnetwork = "${replace(var.subnetwork_self_link, "/(.*)/regions/(.*)/", "regions/$2")}" machine_type = "${var.machine_type}" } diff --git a/variables.tf b/variables.tf index 35a9ce2..046572b 100644 --- a/variables.tf +++ b/variables.tf @@ -64,8 +64,8 @@ variable "subnetwork_self_link" { default = "" } -variable "network" { - description = "The network to which VMs will be assigned." +variable "network_self_link" { + description = "The network self link to which VMs will be assigned." default = "default" } From b4daa74a022be51eceba88d055be57c6ffa7f0e2 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 14 Jun 2019 16:10:49 -0500 Subject: [PATCH 09/13] make generate_docs --- README.md | 5 +++-- examples/simple_example/README.md | 1 + examples/simple_example/main.tf | 2 +- modules/dataflow_bucket/README.md | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1a81e4e..d0273b2 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,16 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| machine\_type | The machine type to use for the job. | string | `""` | no | | max\_workers | The number of workers permitted to work on the job. More workers may improve processing speed at additional cost. | string | `"1"` | no | | name | The name of the dataflow job | string | n/a | yes | -| network | The network to which VMs will be assigned. If it is not provided, "default" will be used. | string | `""` | no | +| network\_self\_link | The network self link to which VMs will be assigned. | string | `"default"` | no | | on\_delete | One of drain or cancel. Specifies behavior of deletion during terraform destroy. The default is cancel. | string | `"cancel"` | no | | parameters | Key/Value pairs to be passed to the Dataflow job (as used in the template). | map | `` | no | | project\_id | The project in which the resource belongs. If it is not provided, the provider project is used. | string | n/a | yes | | region | The bucket's region location | string | `"us-central1"` | no | | service\_account\_email | The Service Account email that will be used to identify the VMs in which the jobs are running | string | `""` | no | -| subnetwork | The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". | string | `""` | no | +| subnetwork\_self\_link | The subnetwork self link to which VMs will be assigned. | string | `""` | no | | temp\_gcs\_location | A writeable location on GCS for the Dataflow job to dump its temporary data. | string | n/a | yes | | template\_gcs\_path | The GCS path to the Dataflow job template. | string | n/a | yes | | zone | The zone in which the created job should run. If it is not provided, the provider zone is used. | string | `"us-central1-a"` | no | diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index a705909..06dbe5c 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -27,6 +27,7 @@ We recommend using a custome service account with fine-grained access control to | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | project\_id | The project ID to deploy to | string | n/a | yes | | region | The region in which the bucket and the dataflow job will be deployed | string | n/a | yes | | service\_account\_email | The Service Account email used to create the job. | string | n/a | yes | diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 51c6ae9..539cb6e 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.0" + version = "~> 2.8.0" region = "${var.region}" } diff --git a/modules/dataflow_bucket/README.md b/modules/dataflow_bucket/README.md index fd7872c..5ee693f 100644 --- a/modules/dataflow_bucket/README.md +++ b/modules/dataflow_bucket/README.md @@ -20,6 +20,7 @@ See [here](../example/simple_example) for a multi jobs example. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | name | The name of the bucket. | string | n/a | yes | | project\_id | The project_id to deploy the example instance into. (e.g. "simple-sample-project-1234") | string | n/a | yes | | region | The GCS bucket region. This should be the same as your dataflow job's zone ot optimize performance. | string | `"us-central1"` | no | From a601f1099d405a315560493b7cf39a0a809283e2 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Fri, 14 Jun 2019 16:33:47 -0500 Subject: [PATCH 10/13] reverted changes for network argument --- examples/simple_example/main.tf | 4 ++-- main.tf | 2 +- variables.tf | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 539cb6e..20611a9 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -67,7 +67,7 @@ module "dataflow-job" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network_self_link = "${module.vpc.network_self_link}" + network_name = "${module.vpc.network_name}" subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-1" @@ -87,7 +87,7 @@ module "dataflow-job-2" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network_self_link = "${module.vpc.network_self_link}" + network_name = "${module.vpc.network_name}" subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-2" diff --git a/main.tf b/main.tf index 84eb0c7..dc9d6f1 100644 --- a/main.tf +++ b/main.tf @@ -25,7 +25,7 @@ resource "google_dataflow_job" "dataflow_job" { temp_gcs_location = "gs://${var.temp_gcs_location}/tmp_dir" parameters = "${var.parameters}" service_account_email = "${var.service_account_email}" - network = "${replace(var.network_self_link, "/(.*)/networks/(.*)/", "$2")}" + network = "${var.network_name}" subnetwork = "${replace(var.subnetwork_self_link, "/(.*)/regions/(.*)/", "regions/$2")}" machine_type = "${var.machine_type}" } diff --git a/variables.tf b/variables.tf index 046572b..612aaee 100644 --- a/variables.tf +++ b/variables.tf @@ -64,8 +64,8 @@ variable "subnetwork_self_link" { default = "" } -variable "network_self_link" { - description = "The network self link to which VMs will be assigned." +variable "network_name" { + description = "The name of the network to which VMs will be assigned." default = "default" } From d5b29a091cc7da89821c46eb1cf07adc7d643dd2 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 17 Jun 2019 17:40:15 -0500 Subject: [PATCH 11/13] Revert "make generate_docs" This reverts commit b4daa74a022be51eceba88d055be57c6ffa7f0e2. --- README.md | 5 ++--- examples/simple_example/README.md | 1 - examples/simple_example/main.tf | 2 +- modules/dataflow_bucket/README.md | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d0273b2..1a81e4e 100644 --- a/README.md +++ b/README.md @@ -47,16 +47,15 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| machine\_type | The machine type to use for the job. | string | `""` | no | | max\_workers | The number of workers permitted to work on the job. More workers may improve processing speed at additional cost. | string | `"1"` | no | | name | The name of the dataflow job | string | n/a | yes | -| network\_self\_link | The network self link to which VMs will be assigned. | string | `"default"` | no | +| network | The network to which VMs will be assigned. If it is not provided, "default" will be used. | string | `""` | no | | on\_delete | One of drain or cancel. Specifies behavior of deletion during terraform destroy. The default is cancel. | string | `"cancel"` | no | | parameters | Key/Value pairs to be passed to the Dataflow job (as used in the template). | map | `` | no | | project\_id | The project in which the resource belongs. If it is not provided, the provider project is used. | string | n/a | yes | | region | The bucket's region location | string | `"us-central1"` | no | | service\_account\_email | The Service Account email that will be used to identify the VMs in which the jobs are running | string | `""` | no | -| subnetwork\_self\_link | The subnetwork self link to which VMs will be assigned. | string | `""` | no | +| subnetwork | The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". | string | `""` | no | | temp\_gcs\_location | A writeable location on GCS for the Dataflow job to dump its temporary data. | string | n/a | yes | | template\_gcs\_path | The GCS path to the Dataflow job template. | string | n/a | yes | | zone | The zone in which the created job should run. If it is not provided, the provider zone is used. | string | `"us-central1-a"` | no | diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index 06dbe5c..a705909 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -27,7 +27,6 @@ We recommend using a custome service account with fine-grained access control to | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | project\_id | The project ID to deploy to | string | n/a | yes | | region | The region in which the bucket and the dataflow job will be deployed | string | n/a | yes | | service\_account\_email | The Service Account email used to create the job. | string | n/a | yes | diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 20611a9..36c6d01 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.8.0" + version = "~> 2.0" region = "${var.region}" } diff --git a/modules/dataflow_bucket/README.md b/modules/dataflow_bucket/README.md index 5ee693f..fd7872c 100644 --- a/modules/dataflow_bucket/README.md +++ b/modules/dataflow_bucket/README.md @@ -20,7 +20,6 @@ See [here](../example/simple_example) for a multi jobs example. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | name | The name of the bucket. | string | n/a | yes | | project\_id | The project_id to deploy the example instance into. (e.g. "simple-sample-project-1234") | string | n/a | yes | | region | The GCS bucket region. This should be the same as your dataflow job's zone ot optimize performance. | string | `"us-central1"` | no | From 214b70f3bb0ea04b2fab29ffc9850d86b4587b74 Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 17 Jun 2019 17:44:06 -0500 Subject: [PATCH 12/13] Revert "Revert "make generate_docs"" This reverts commit d5b29a091cc7da89821c46eb1cf07adc7d643dd2. --- README.md | 5 +++-- examples/simple_example/README.md | 1 + examples/simple_example/main.tf | 2 +- modules/dataflow_bucket/README.md | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1a81e4e..d0273b2 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,16 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| machine\_type | The machine type to use for the job. | string | `""` | no | | max\_workers | The number of workers permitted to work on the job. More workers may improve processing speed at additional cost. | string | `"1"` | no | | name | The name of the dataflow job | string | n/a | yes | -| network | The network to which VMs will be assigned. If it is not provided, "default" will be used. | string | `""` | no | +| network\_self\_link | The network self link to which VMs will be assigned. | string | `"default"` | no | | on\_delete | One of drain or cancel. Specifies behavior of deletion during terraform destroy. The default is cancel. | string | `"cancel"` | no | | parameters | Key/Value pairs to be passed to the Dataflow job (as used in the template). | map | `` | no | | project\_id | The project in which the resource belongs. If it is not provided, the provider project is used. | string | n/a | yes | | region | The bucket's region location | string | `"us-central1"` | no | | service\_account\_email | The Service Account email that will be used to identify the VMs in which the jobs are running | string | `""` | no | -| subnetwork | The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". | string | `""` | no | +| subnetwork\_self\_link | The subnetwork self link to which VMs will be assigned. | string | `""` | no | | temp\_gcs\_location | A writeable location on GCS for the Dataflow job to dump its temporary data. | string | n/a | yes | | template\_gcs\_path | The GCS path to the Dataflow job template. | string | n/a | yes | | zone | The zone in which the created job should run. If it is not provided, the provider zone is used. | string | `"us-central1-a"` | no | diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index a705909..06dbe5c 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -27,6 +27,7 @@ We recommend using a custome service account with fine-grained access control to | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | project\_id | The project ID to deploy to | string | n/a | yes | | region | The region in which the bucket and the dataflow job will be deployed | string | n/a | yes | | service\_account\_email | The Service Account email used to create the job. | string | n/a | yes | diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 36c6d01..20611a9 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.0" + version = "~> 2.8.0" region = "${var.region}" } diff --git a/modules/dataflow_bucket/README.md b/modules/dataflow_bucket/README.md index fd7872c..5ee693f 100644 --- a/modules/dataflow_bucket/README.md +++ b/modules/dataflow_bucket/README.md @@ -20,6 +20,7 @@ See [here](../example/simple_example) for a multi jobs example. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | string | `"false"` | no | | name | The name of the bucket. | string | n/a | yes | | project\_id | The project_id to deploy the example instance into. (e.g. "simple-sample-project-1234") | string | n/a | yes | | region | The GCS bucket region. This should be the same as your dataflow job's zone ot optimize performance. | string | `"us-central1"` | no | From ffa3bfde26b32a78d8a98ccf6f1880e6e059e90a Mon Sep 17 00:00:00 2001 From: tfmenard Date: Mon, 17 Jun 2019 17:44:51 -0500 Subject: [PATCH 13/13] Revert "reverted changes for network argument" This reverts commit a601f1099d405a315560493b7cf39a0a809283e2. --- examples/simple_example/main.tf | 4 ++-- main.tf | 2 +- variables.tf | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 20611a9..539cb6e 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -67,7 +67,7 @@ module "dataflow-job" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network_name = "${module.vpc.network_name}" + network_self_link = "${module.vpc.network_self_link}" subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-1" @@ -87,7 +87,7 @@ module "dataflow-job-2" { template_gcs_path = "gs://dataflow-templates/latest/Word_Count" temp_gcs_location = "${module.dataflow-bucket.name}" service_account_email = "${var.service_account_email}" - network_name = "${module.vpc.network_name}" + network_self_link = "${module.vpc.network_self_link}" subnetwork_self_link = "${module.vpc.subnets_self_links[0]}" machine_type = "n1-standard-2" diff --git a/main.tf b/main.tf index dc9d6f1..84eb0c7 100644 --- a/main.tf +++ b/main.tf @@ -25,7 +25,7 @@ resource "google_dataflow_job" "dataflow_job" { temp_gcs_location = "gs://${var.temp_gcs_location}/tmp_dir" parameters = "${var.parameters}" service_account_email = "${var.service_account_email}" - network = "${var.network_name}" + network = "${replace(var.network_self_link, "/(.*)/networks/(.*)/", "$2")}" subnetwork = "${replace(var.subnetwork_self_link, "/(.*)/regions/(.*)/", "regions/$2")}" machine_type = "${var.machine_type}" } diff --git a/variables.tf b/variables.tf index 612aaee..046572b 100644 --- a/variables.tf +++ b/variables.tf @@ -64,8 +64,8 @@ variable "subnetwork_self_link" { default = "" } -variable "network_name" { - description = "The name of the network to which VMs will be assigned." +variable "network_self_link" { + description = "The network self link to which VMs will be assigned." default = "default" }