From 8a79411a2e97ea55a3cf16c2f916c625da7a0b73 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Thu, 19 Sep 2019 14:10:47 -0400 Subject: [PATCH 01/78] fix: updated subnets to use for_each rather then count I also updated the outputs as well. This will be large state change however and probably needs to be a major release --- main.tf | 32 +++++++++++++++----------------- outputs.tf | 15 +++++++-------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/main.tf b/main.tf index 59e436df..78b2a837 100644 --- a/main.tf +++ b/main.tf @@ -14,6 +14,13 @@ * limitations under the License. */ +locals { + subnets = { + for x in var.subnets : + "${x.subnet_region}/${x.subnet_name}" => x + } +} + /****************************************** VPC configuration *****************************************/ @@ -38,25 +45,16 @@ resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { Subnet configuration *****************************************/ resource "google_compute_subnetwork" "subnetwork" { - count = length(var.subnets) - - name = var.subnets[count.index]["subnet_name"] - ip_cidr_range = var.subnets[count.index]["subnet_ip"] - region = var.subnets[count.index]["subnet_region"] - private_ip_google_access = lookup(var.subnets[count.index], "subnet_private_access", "false") - enable_flow_logs = lookup(var.subnets[count.index], "subnet_flow_logs", "false") + for_each = local.subnets + name = each.value.subnet_name + ip_cidr_range = each.value.subnet_ip + region = each.value.subnet_region + private_ip_google_access = lookup(each.value, "subnet_private_access", "false") + enable_flow_logs = lookup(each.value, "subnet_flow_logs", "false") network = google_compute_network.network.name project = var.project_id - secondary_ip_range = [for i in range(length(contains(keys(var.secondary_ranges), var.subnets[count.index]["subnet_name"]) == true ? var.secondary_ranges[var.subnets[count.index]["subnet_name"]] : [])) : var.secondary_ranges[var.subnets[count.index]["subnet_name"]][i]] - description = lookup(var.subnets[count.index], "description", null) -} - -data "google_compute_subnetwork" "created_subnets" { - count = length(var.subnets) - name = element(google_compute_subnetwork.subnetwork.*.name, count.index) - region = element(google_compute_subnetwork.subnetwork.*.region, count.index) - project = var.project_id - depends_on = [google_compute_subnetwork.subnetwork] + secondary_ip_range = [for i in range(length(contains(keys(var.secondary_ranges), each.value.subnet_name) == true ? var.secondary_ranges[each.value.subnet_name] : [])) : var.secondary_ranges[each.value.subnet_name][i]] + description = lookup(each.value, "description", null) } /****************************************** diff --git a/outputs.tf b/outputs.tf index 09591656..07610d90 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,37 +30,37 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = google_compute_subnetwork.subnetwork.*.name + value = values(google_compute_subnetwork.subnetwork)[*].name description = "The names of the subnets being created" } output "subnets_ips" { - value = google_compute_subnetwork.subnetwork.*.ip_cidr_range + value = values(google_compute_subnetwork.subnetwork)[*].ip_cidr_range description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = google_compute_subnetwork.subnetwork.*.self_link + value = values(google_compute_subnetwork.subnetwork)[*].self_link description = "The self-links of subnets being created" } output "subnets_regions" { - value = google_compute_subnetwork.subnetwork.*.region + value = values(google_compute_subnetwork.subnetwork)[*].region description = "The region where the subnets will be created" } output "subnets_private_access" { - value = google_compute_subnetwork.subnetwork.*.private_ip_google_access + value = values(google_compute_subnetwork.subnetwork)[*].private_ip_google_access description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = google_compute_subnetwork.subnetwork.*.enable_flow_logs + value = values(google_compute_subnetwork.subnetwork)[*].enable_flow_logs description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = data.google_compute_subnetwork.created_subnets.*.secondary_ip_range + value = values(google_compute_subnetwork.subnetwork)[*].secondary_ip_range description = "The secondary ranges associated with these subnets" } @@ -68,4 +68,3 @@ output "routes" { value = google_compute_route.route.*.name description = "The routes associated with this VPC" } - From f1590db1a746c1910f535f64f2b43266f82aad27 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Thu, 19 Sep 2019 16:05:42 -0400 Subject: [PATCH 02/78] fix: fixing incorrect outputs --- outputs.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/outputs.tf b/outputs.tf index 07610d90..877ca861 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,37 +30,37 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = values(google_compute_subnetwork.subnetwork)[*].name + value = google_compute_subnetwork.subnetwork[*].name description = "The names of the subnets being created" } output "subnets_ips" { - value = values(google_compute_subnetwork.subnetwork)[*].ip_cidr_range + value = google_compute_subnetwork.subnetwork[*].ip_cidr_range description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = values(google_compute_subnetwork.subnetwork)[*].self_link + value = google_compute_subnetwork.subnetwork[*].self_link description = "The self-links of subnets being created" } output "subnets_regions" { - value = values(google_compute_subnetwork.subnetwork)[*].region + value = google_compute_subnetwork.subnetwork[*].region description = "The region where the subnets will be created" } output "subnets_private_access" { - value = values(google_compute_subnetwork.subnetwork)[*].private_ip_google_access + value = google_compute_subnetwork.subnetwork[*].private_ip_google_access description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = values(google_compute_subnetwork.subnetwork)[*].enable_flow_logs + value = google_compute_subnetwork.subnetwork[*].enable_flow_logs description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = values(google_compute_subnetwork.subnetwork)[*].secondary_ip_range + value = google_compute_subnetwork.subnetwork[*].secondary_ip_range description = "The secondary ranges associated with these subnets" } From f2839e8385d8f7404538fe0e4570a7a29fd90a5d Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 12:36:33 -0400 Subject: [PATCH 03/78] wip: tried chaging to list, didnt' work. Going to do more debugging --- outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs.tf b/outputs.tf index 877ca861..29b16c4b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -35,7 +35,7 @@ output "subnets_names" { } output "subnets_ips" { - value = google_compute_subnetwork.subnetwork[*].ip_cidr_range + value = tolist(google_compute_subnetwork.subnetwork[*].ip_cidr_range) description = "The IPs and CIDRs of the subnets being created" } From 1392cf04d8d35c4f55af2a8bb7cb17a16eb778b1 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 13:50:14 -0400 Subject: [PATCH 04/78] fix: fixed outputs so they would match the outputs of splat I also added some debug output to tests to make tracking this down in the future a bit easier --- outputs.tf | 15 +++++----- test/fixtures/shared/outputs.tf | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/outputs.tf b/outputs.tf index 29b16c4b..f1af3c96 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,37 +30,38 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = google_compute_subnetwork.subnetwork[*].name + value = [ for network in google_compute_subnetwork.subnetwork: network.name ] #google_compute_subnetwork.subnetwork[*].name description = "The names of the subnets being created" } output "subnets_ips" { - value = tolist(google_compute_subnetwork.subnetwork[*].ip_cidr_range) + #value = values(google_compute_subnetwork.subnetwork[*]).ip_cidr_range + value = [ for network in google_compute_subnetwork.subnetwork: network.ip_cidr_range ] description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = google_compute_subnetwork.subnetwork[*].self_link + value = [ for network in google_compute_subnetwork.subnetwork: network.self_link ] #google_compute_subnetwork.subnetwork[*].self_link description = "The self-links of subnets being created" } output "subnets_regions" { - value = google_compute_subnetwork.subnetwork[*].region + value = [ for network in google_compute_subnetwork.subnetwork: network.region ] #google_compute_subnetwork.subnetwork[*].region description = "The region where the subnets will be created" } output "subnets_private_access" { - value = google_compute_subnetwork.subnetwork[*].private_ip_google_access + value = [ for network in google_compute_subnetwork.subnetwork: network.private_ip_google_access ] #google_compute_subnetwork.subnetwork[*].private_ip_google_access description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = google_compute_subnetwork.subnetwork[*].enable_flow_logs + value = [ for network in google_compute_subnetwork.subnetwork: network.enable_flow_logs ] #google_compute_subnetwork.subnetwork[*].enable_flow_logs description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = google_compute_subnetwork.subnetwork[*].secondary_ip_range + value = [ for network in google_compute_subnetwork.subnetwork: network.secondary_ip_range ] #google_compute_subnetwork.subnetwork[*].secondary_ip_range description = "The secondary ranges associated with these subnets" } diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 68e9e076..fba7438b 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -23,3 +23,54 @@ output "network_name" { value = module.example.network_name description = "The name of the VPC being created" } + +output "output_network_name" { + value = module.example.network_name + description = "The name of the VPC being created" +} + +output "output_network_self_link" { + value = module.example.network_self_link + description = "The URI of the VPC being created" +} + +output "output_svpc_host_project_id" { + value = module.example.svpc_host_project_id + description = "Shared VPC host project id." +} + +output "output_subnets_names" { + value = module.example.subnets_names + description = "The names of the subnets being created" +} + +output "output_subnets_ips" { + value = module.example.subnets_ips + description = "The IP and cidrs of the subnets being created" +} + + +output "output_subnets_regions" { + value = module.example.subnets_regions + description = "The region where subnets will be created" +} + +output "output_subnets_private_access" { + value = module.example.subnets_private_access + description = "Whether the subnets will have access to Google API's without a public IP" +} + +output "output_subnets_flow_logs" { + value = module.example.subnets_flow_logs + description = "Whether the subnets will have VPC flow logs enabled" +} + +output "output_subnets_secondary_ranges" { + value = module.example.subnets_secondary_ranges + description = "The secondary ranges associated with these subnets" +} + +output "output_routes" { + value = module.example.routes + description = "The routes associated with this VPC" +} From 264b2e0417a14dabe0a4e5a404f0454bd71d8bed Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 13:54:31 -0400 Subject: [PATCH 05/78] fix: formatting! From hells heart I curse thee! --- outputs.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/outputs.tf b/outputs.tf index f1af3c96..34ce0789 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,38 +30,38 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = [ for network in google_compute_subnetwork.subnetwork: network.name ] #google_compute_subnetwork.subnetwork[*].name + value = [for network in google_compute_subnetwork.subnetwork : network.name] #google_compute_subnetwork.subnetwork[*].name description = "The names of the subnets being created" } output "subnets_ips" { #value = values(google_compute_subnetwork.subnetwork[*]).ip_cidr_range - value = [ for network in google_compute_subnetwork.subnetwork: network.ip_cidr_range ] + value = [for network in google_compute_subnetwork.subnetwork : network.ip_cidr_range] description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = [ for network in google_compute_subnetwork.subnetwork: network.self_link ] #google_compute_subnetwork.subnetwork[*].self_link + value = [for network in google_compute_subnetwork.subnetwork : network.self_link] #google_compute_subnetwork.subnetwork[*].self_link description = "The self-links of subnets being created" } output "subnets_regions" { - value = [ for network in google_compute_subnetwork.subnetwork: network.region ] #google_compute_subnetwork.subnetwork[*].region + value = [for network in google_compute_subnetwork.subnetwork : network.region] #google_compute_subnetwork.subnetwork[*].region description = "The region where the subnets will be created" } output "subnets_private_access" { - value = [ for network in google_compute_subnetwork.subnetwork: network.private_ip_google_access ] #google_compute_subnetwork.subnetwork[*].private_ip_google_access + value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] #google_compute_subnetwork.subnetwork[*].private_ip_google_access description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = [ for network in google_compute_subnetwork.subnetwork: network.enable_flow_logs ] #google_compute_subnetwork.subnetwork[*].enable_flow_logs + value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] #google_compute_subnetwork.subnetwork[*].enable_flow_logs description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = [ for network in google_compute_subnetwork.subnetwork: network.secondary_ip_range ] #google_compute_subnetwork.subnetwork[*].secondary_ip_range + value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] #google_compute_subnetwork.subnetwork[*].secondary_ip_range description = "The secondary ranges associated with these subnets" } From 71939d22ca72f6dc0598c4d79e1707cc9acde262 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 14:19:22 -0400 Subject: [PATCH 06/78] fix: cleaned up example outputs declarations --- examples/secondary_ranges/outputs.tf | 5 +++ examples/simple_project/outputs.tf | 5 +++ .../outputs.tf | 5 +++ examples/submodule_firewall/outputs.tf | 45 +++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/examples/secondary_ranges/outputs.tf b/examples/secondary_ranges/outputs.tf index 2ccc964a..95d5b63e 100644 --- a/examples/secondary_ranges/outputs.tf +++ b/examples/secondary_ranges/outputs.tf @@ -24,6 +24,11 @@ output "network_self_link" { description = "The URI of the VPC being created" } +output "svpc_host_project_id" { + value = module.vpc-secondary-ranges.svpc_host_project_id + description = "Shared VPC host project id." +} + output "subnets_names" { value = module.vpc-secondary-ranges.subnets_names description = "The names of the subnets being created" diff --git a/examples/simple_project/outputs.tf b/examples/simple_project/outputs.tf index 429e0b97..b5f631cc 100644 --- a/examples/simple_project/outputs.tf +++ b/examples/simple_project/outputs.tf @@ -24,6 +24,11 @@ output "network_self_link" { description = "The URI of the VPC being created" } +output "svpc_host_project_id" { + value = module.test-vpc-module.svpc_host_project_id + description = "Shared VPC host project id." +} + output "subnets_names" { value = module.test-vpc-module.subnets_names description = "The names of the subnets being created" diff --git a/examples/simple_project_with_regional_network/outputs.tf b/examples/simple_project_with_regional_network/outputs.tf index 429e0b97..b5f631cc 100644 --- a/examples/simple_project_with_regional_network/outputs.tf +++ b/examples/simple_project_with_regional_network/outputs.tf @@ -24,6 +24,11 @@ output "network_self_link" { description = "The URI of the VPC being created" } +output "svpc_host_project_id" { + value = module.test-vpc-module.svpc_host_project_id + description = "Shared VPC host project id." +} + output "subnets_names" { value = module.test-vpc-module.subnets_names description = "The names of the subnets being created" diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index f687da74..0586334f 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -28,3 +28,48 @@ output "admin_ranges" { description = "Firewall attributes for admin ranges." value = module.test-firewall-submodule.admin_ranges } + +output "network_self_link" { + value = module.test-vpc-module.network_self_link + description = "The URI of the VPC being created" +} + +output "svpc_host_project_id" { + value = module.test-vpc-module.svpc_host_project_id + description = "Shared VPC host project id." +} + +output "subnets_names" { + value = module.test-vpc-module.subnets_names + description = "The names of the subnets being created" +} + +output "subnets_ips" { + value = module.test-vpc-module.subnets_ips + description = "The IP and cidrs of the subnets being created" +} + +output "subnets_regions" { + value = module.test-vpc-module.subnets_regions + description = "The region where subnets will be created" +} + +output "subnets_private_access" { + value = module.test-vpc-module.subnets_private_access + description = "Whether the subnets will have access to Google API's without a public IP" +} + +output "subnets_flow_logs" { + value = module.test-vpc-module.subnets_flow_logs + description = "Whether the subnets will have VPC flow logs enabled" +} + +output "subnets_secondary_ranges" { + value = module.test-vpc-module.subnets_secondary_ranges + description = "The secondary ranges associated with these subnets" +} + +output "routes" { + value = module.test-vpc-module.routes + description = "The routes associated with this VPC" +} From b17126ecfdbc7c5ded3cdc9f5582bef741915a57 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 15:50:05 -0400 Subject: [PATCH 07/78] feat: adding some tests to validate outputs --- .kitchen.yml | 150 +++++++++--------- .../controls/inspec_attributes.rb | 16 ++ test/integration/secondary_ranges/inspec.yml | 6 + 3 files changed, 99 insertions(+), 73 deletions(-) create mode 100644 test/integration/secondary_ranges/controls/inspec_attributes.rb diff --git a/.kitchen.yml b/.kitchen.yml index 1617025d..a487be7f 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -24,36 +24,36 @@ platforms: - name: local suites: - - name: "simple_project" - driver: - name: "terraform" - command_timeout: 1800 - root_module_directory: test/fixtures/simple_project/ - verifier: - name: terraform - color: true - systems: - - name: inspec-gcp - backend: gcp - controls: - - gcp - - name: local - backend: local - controls: - - gcloud - - name: "simple_project_with_regional_network" - driver: - name: "terraform" - command_timeout: 1800 - root_module_directory: test/fixtures/simple_project_with_regional_network/ - verifier: - name: terraform - color: true - systems: - - name: inspec-gcp - backend: gcp - controls: - - gcp + # - name: "simple_project" + # driver: + # name: "terraform" + # command_timeout: 1800 + # root_module_directory: test/fixtures/simple_project/ + # verifier: + # name: terraform + # color: true + # systems: + # - name: inspec-gcp + # backend: gcp + # controls: + # - gcp + # - name: local + # backend: local + # controls: + # - gcloud + # - name: "simple_project_with_regional_network" + # driver: + # name: "terraform" + # command_timeout: 1800 + # root_module_directory: test/fixtures/simple_project_with_regional_network/ + # verifier: + # name: terraform + # color: true + # systems: + # - name: inspec-gcp + # backend: gcp + # controls: + # - gcp - name: "secondary_ranges" driver: name: "terraform" @@ -64,49 +64,53 @@ suites: color: true systems: - name: local + attrs_outputs: + customized_inspec_attribute: output_network_name + customized_inspec_attribute: output_network_self_link backend: local controls: - gcloud - - name: "multi_vpc" - driver: - name: "terraform" - command_timeout: 1800 - root_module_directory: test/fixtures/multi_vpc/ - verifier: - name: terraform - color: true - systems: - - name: local - backend: local - controls: - - gcloud - - name: "delete_default_gateway_routes" - driver: - name: "terraform" - command_timeout: 1800 - root_module_directory: test/fixtures/delete_default_gateway_routes/ - verifier: - name: terraform - color: true - systems: - - name: local - backend: local - controls: - - gcloud - - name: "submodule_firewall" - driver: - name: "terraform" - command_timeout: 1800 - root_module_directory: test/fixtures/submodule_firewall/ - verifier: - name: terraform - color: true - systems: - - name: inspec-gcp - backend: gcp - controls: - - gcp - - name: local - backend: local - controls: - - gcloud + - inspec_attributes + # - name: "multi_vpc" + # driver: + # name: "terraform" + # command_timeout: 1800 + # root_module_directory: test/fixtures/multi_vpc/ + # verifier: + # name: terraform + # color: true + # systems: + # - name: local + # backend: local + # controls: + # - gcloud + # - name: "delete_default_gateway_routes" + # driver: + # name: "terraform" + # command_timeout: 1800 + # root_module_directory: test/fixtures/delete_default_gateway_routes/ + # verifier: + # name: terraform + # color: true + # systems: + # - name: local + # backend: local + # controls: + # - gcloud + # - name: "submodule_firewall" + # driver: + # name: "terraform" + # command_timeout: 1800 + # root_module_directory: test/fixtures/submodule_firewall/ + # verifier: + # name: terraform + # color: true + # systems: + # - name: inspec-gcp + # backend: gcp + # controls: + # - gcp + # - name: local + # backend: local + # controls: + # - gcloud diff --git a/test/integration/secondary_ranges/controls/inspec_attributes.rb b/test/integration/secondary_ranges/controls/inspec_attributes.rb new file mode 100644 index 00000000..9d0e9232 --- /dev/null +++ b/test/integration/secondary_ranges/controls/inspec_attributes.rb @@ -0,0 +1,16 @@ +project_id = attribute('project_id') +network_name = attribute('network_name') + +control "inspec_attributes" do + title "Test Terraform Outputs" + desc "Test Terraform outputs" + + describe attribute("output_network_name") do + it { should eq "#{network_name}" } + end + + describe attribute("output_network_self_link") do + it { should eq "https://www.googleapis.com/compute/v1/projects/#{project_id}/global/networks/#{network_name}" } + end + +end diff --git a/test/integration/secondary_ranges/inspec.yml b/test/integration/secondary_ranges/inspec.yml index 94f1cbc6..c0dff54b 100644 --- a/test/integration/secondary_ranges/inspec.yml +++ b/test/integration/secondary_ranges/inspec.yml @@ -6,3 +6,9 @@ attributes: - name: network_name required: true type: string + - name: output_network_name + required: true + type: string + - name: output_network_self_link + required: true + type: string From 188b777c05cb0d317ad8feb98357fdf0a367b4b3 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 15:50:57 -0400 Subject: [PATCH 08/78] fix: doh, don't comment out the tests --- .kitchen.yml | 146 +++++++++++++++++++++++++-------------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index a487be7f..176e3898 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -24,36 +24,36 @@ platforms: - name: local suites: - # - name: "simple_project" - # driver: - # name: "terraform" - # command_timeout: 1800 - # root_module_directory: test/fixtures/simple_project/ - # verifier: - # name: terraform - # color: true - # systems: - # - name: inspec-gcp - # backend: gcp - # controls: - # - gcp - # - name: local - # backend: local - # controls: - # - gcloud - # - name: "simple_project_with_regional_network" - # driver: - # name: "terraform" - # command_timeout: 1800 - # root_module_directory: test/fixtures/simple_project_with_regional_network/ - # verifier: - # name: terraform - # color: true - # systems: - # - name: inspec-gcp - # backend: gcp - # controls: - # - gcp + - name: "simple_project" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/simple_project/ + verifier: + name: terraform + color: true + systems: + - name: inspec-gcp + backend: gcp + controls: + - gcp + - name: local + backend: local + controls: + - gcloud + - name: "simple_project_with_regional_network" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/simple_project_with_regional_network/ + verifier: + name: terraform + color: true + systems: + - name: inspec-gcp + backend: gcp + controls: + - gcp - name: "secondary_ranges" driver: name: "terraform" @@ -71,46 +71,46 @@ suites: controls: - gcloud - inspec_attributes - # - name: "multi_vpc" - # driver: - # name: "terraform" - # command_timeout: 1800 - # root_module_directory: test/fixtures/multi_vpc/ - # verifier: - # name: terraform - # color: true - # systems: - # - name: local - # backend: local - # controls: - # - gcloud - # - name: "delete_default_gateway_routes" - # driver: - # name: "terraform" - # command_timeout: 1800 - # root_module_directory: test/fixtures/delete_default_gateway_routes/ - # verifier: - # name: terraform - # color: true - # systems: - # - name: local - # backend: local - # controls: - # - gcloud - # - name: "submodule_firewall" - # driver: - # name: "terraform" - # command_timeout: 1800 - # root_module_directory: test/fixtures/submodule_firewall/ - # verifier: - # name: terraform - # color: true - # systems: - # - name: inspec-gcp - # backend: gcp - # controls: - # - gcp - # - name: local - # backend: local - # controls: - # - gcloud + - name: "multi_vpc" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/multi_vpc/ + verifier: + name: terraform + color: true + systems: + - name: local + backend: local + controls: + - gcloud + - name: "delete_default_gateway_routes" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/delete_default_gateway_routes/ + verifier: + name: terraform + color: true + systems: + - name: local + backend: local + controls: + - gcloud + - name: "submodule_firewall" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/submodule_firewall/ + verifier: + name: terraform + color: true + systems: + - name: inspec-gcp + backend: gcp + controls: + - gcp + - name: local + backend: local + controls: + - gcloud From 17e12e5df275d920c5197bbd61040bf33eda4422 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 20 Sep 2019 21:27:51 -0400 Subject: [PATCH 09/78] feat: and low. I have created tests for outputs --- .kitchen.yml | 10 ++++++ .../controls/inspec_attributes.rb | 35 +++++++++++++++++-- test/integration/secondary_ranges/inspec.yml | 16 +++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 176e3898..7f175b52 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -67,6 +67,16 @@ suites: attrs_outputs: customized_inspec_attribute: output_network_name customized_inspec_attribute: output_network_self_link + customized_inspec_attribute: output_subnets_ips + customized_inspec_attribute: output_routes + customized_inspec_attribute: output_subnets_flow_logs + customized_inspec_attribute: output_subnets_names + customized_inspec_attribute: output_subnets_private_access + customized_inspec_attribute: output_subnets_regions + customized_inspec_attribute: output_subnets_secondary_ranges + customized_inspec_attribute: output_svpc_host_project_id + + backend: local controls: - gcloud diff --git a/test/integration/secondary_ranges/controls/inspec_attributes.rb b/test/integration/secondary_ranges/controls/inspec_attributes.rb index 9d0e9232..4875b2c4 100644 --- a/test/integration/secondary_ranges/controls/inspec_attributes.rb +++ b/test/integration/secondary_ranges/controls/inspec_attributes.rb @@ -2,8 +2,8 @@ network_name = attribute('network_name') control "inspec_attributes" do - title "Test Terraform Outputs" - desc "Test Terraform outputs" + title "Terraform Outputs" + desc "Terraform Outputs" describe attribute("output_network_name") do it { should eq "#{network_name}" } @@ -13,4 +13,35 @@ it { should eq "https://www.googleapis.com/compute/v1/projects/#{project_id}/global/networks/#{network_name}" } end + describe attribute("output_subnets_ips") do + it { should eq ["10.10.10.0/24", "10.10.20.0/24", "10.10.30.0/24", "10.10.40.0/24"] } + end + + describe attribute("output_routes") do + it { should eq [] } + end + + describe attribute("output_subnets_flow_logs") do + it { should eq [false, true, false, false] } + end + + describe attribute("output_subnets_names") do + it { should eq ["#{network_name}-subnet-01", "#{network_name}-subnet-02", "#{network_name}-subnet-03", "#{network_name}-subnet-04"] } + end + + describe attribute("output_subnets_private_access") do + it { should eq [false, true, false, false] } + end + + describe attribute("output_subnets_regions") do + it { should eq ["us-west1", "us-west1", "us-west1", "us-west1"] } + end + + describe attribute("output_subnets_secondary_ranges") do + it { should eq [{"ip_cidr_range"=>"192.168.64.0/24", "range_name"=>"#{network_name}-subnet-01-01"}, {"ip_cidr_range"=>"192.168.65.0/24", "range_name"=>"#{network_name}-subnet-01-02"}, {"ip_cidr_range"=>"192.168.66.0/24", "range_name"=>"#{network_name}-subnet-03-01"}] } + end + + describe attribute("output_svpc_host_project_id") do + it { should eq "" } + end end diff --git a/test/integration/secondary_ranges/inspec.yml b/test/integration/secondary_ranges/inspec.yml index c0dff54b..ea9c3b05 100644 --- a/test/integration/secondary_ranges/inspec.yml +++ b/test/integration/secondary_ranges/inspec.yml @@ -12,3 +12,19 @@ attributes: - name: output_network_self_link required: true type: string + - name: output_subnets_ips + required: true + - name: output_routes + required: true + - name: output_subnets_flow_logs + required: true + - name: output_subnets_names + required: true + - name: output_subnets_private_access + required: true + - name: output_subnets_regions + required: true + - name: output_subnets_secondary_ranges + required: true + - name: output_svpc_host_project_id + required: true From e89d26bc8d8960cead61f26f140612b54ddd9a01 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Tue, 24 Sep 2019 10:19:23 -0400 Subject: [PATCH 10/78] fix: removing spurious comments Whoops, missed these --- outputs.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/outputs.tf b/outputs.tf index 34ce0789..11e707a9 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,38 +30,37 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = [for network in google_compute_subnetwork.subnetwork : network.name] #google_compute_subnetwork.subnetwork[*].name + value = [for network in google_compute_subnetwork.subnetwork : network.name] description = "The names of the subnets being created" } output "subnets_ips" { - #value = values(google_compute_subnetwork.subnetwork[*]).ip_cidr_range value = [for network in google_compute_subnetwork.subnetwork : network.ip_cidr_range] description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = [for network in google_compute_subnetwork.subnetwork : network.self_link] #google_compute_subnetwork.subnetwork[*].self_link + value = [for network in google_compute_subnetwork.subnetwork : network.self_link] description = "The self-links of subnets being created" } output "subnets_regions" { - value = [for network in google_compute_subnetwork.subnetwork : network.region] #google_compute_subnetwork.subnetwork[*].region + value = [for network in google_compute_subnetwork.subnetwork : network.region] description = "The region where the subnets will be created" } output "subnets_private_access" { - value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] #google_compute_subnetwork.subnetwork[*].private_ip_google_access + value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] #google_compute_subnetwork.subnetwork[*].enable_flow_logs + value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] #google_compute_subnetwork.subnetwork[*].secondary_ip_range + value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] description = "The secondary ranges associated with these subnets" } From a57883c5daf17df33230660f18945807291d652c Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 25 Sep 2019 10:13:25 -0400 Subject: [PATCH 11/78] fix: removed outputs from the firewall example --- examples/submodule_firewall/outputs.tf | 45 -------------------------- 1 file changed, 45 deletions(-) diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index 0586334f..f687da74 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -28,48 +28,3 @@ output "admin_ranges" { description = "Firewall attributes for admin ranges." value = module.test-firewall-submodule.admin_ranges } - -output "network_self_link" { - value = module.test-vpc-module.network_self_link - description = "The URI of the VPC being created" -} - -output "svpc_host_project_id" { - value = module.test-vpc-module.svpc_host_project_id - description = "Shared VPC host project id." -} - -output "subnets_names" { - value = module.test-vpc-module.subnets_names - description = "The names of the subnets being created" -} - -output "subnets_ips" { - value = module.test-vpc-module.subnets_ips - description = "The IP and cidrs of the subnets being created" -} - -output "subnets_regions" { - value = module.test-vpc-module.subnets_regions - description = "The region where subnets will be created" -} - -output "subnets_private_access" { - value = module.test-vpc-module.subnets_private_access - description = "Whether the subnets will have access to Google API's without a public IP" -} - -output "subnets_flow_logs" { - value = module.test-vpc-module.subnets_flow_logs - description = "Whether the subnets will have VPC flow logs enabled" -} - -output "subnets_secondary_ranges" { - value = module.test-vpc-module.subnets_secondary_ranges - description = "The secondary ranges associated with these subnets" -} - -output "routes" { - value = module.test-vpc-module.routes - description = "The routes associated with this VPC" -} From 7c43cf0ead5d5a4432a834d4baf8b5f1414833ba Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 25 Sep 2019 10:37:44 -0400 Subject: [PATCH 12/78] Revert "fix: removed outputs from the firewall example" This reverts commit 1dd8c4e76a8040765ff46b174ee138dae60e2da9. --- examples/submodule_firewall/outputs.tf | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index f687da74..0586334f 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -28,3 +28,48 @@ output "admin_ranges" { description = "Firewall attributes for admin ranges." value = module.test-firewall-submodule.admin_ranges } + +output "network_self_link" { + value = module.test-vpc-module.network_self_link + description = "The URI of the VPC being created" +} + +output "svpc_host_project_id" { + value = module.test-vpc-module.svpc_host_project_id + description = "Shared VPC host project id." +} + +output "subnets_names" { + value = module.test-vpc-module.subnets_names + description = "The names of the subnets being created" +} + +output "subnets_ips" { + value = module.test-vpc-module.subnets_ips + description = "The IP and cidrs of the subnets being created" +} + +output "subnets_regions" { + value = module.test-vpc-module.subnets_regions + description = "The region where subnets will be created" +} + +output "subnets_private_access" { + value = module.test-vpc-module.subnets_private_access + description = "Whether the subnets will have access to Google API's without a public IP" +} + +output "subnets_flow_logs" { + value = module.test-vpc-module.subnets_flow_logs + description = "Whether the subnets will have VPC flow logs enabled" +} + +output "subnets_secondary_ranges" { + value = module.test-vpc-module.subnets_secondary_ranges + description = "The secondary ranges associated with these subnets" +} + +output "routes" { + value = module.test-vpc-module.routes + description = "The routes associated with this VPC" +} From d451d320c4f84eb5ef36a87d8a9d04850233a86f Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 25 Sep 2019 10:47:00 -0400 Subject: [PATCH 13/78] docs: fixing header warning This warning was being thrown by the linter ``` Checking file headers 1 files have incorrect boilerplate headers: test/integration/secondary_ranges/controls/inspec_attributes.rb ``` --- .../secondary_ranges/controls/inspec_attributes.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/integration/secondary_ranges/controls/inspec_attributes.rb b/test/integration/secondary_ranges/controls/inspec_attributes.rb index 4875b2c4..9e11b6cd 100644 --- a/test/integration/secondary_ranges/controls/inspec_attributes.rb +++ b/test/integration/secondary_ranges/controls/inspec_attributes.rb @@ -1,3 +1,17 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + project_id = attribute('project_id') network_name = attribute('network_name') From ff274e9e8c7a20aa3241c1f458221fbe669a61e2 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 25 Sep 2019 10:48:15 -0400 Subject: [PATCH 14/78] fix: ran `make docker_generate_docs` --- examples/secondary_ranges/README.md | 1 + examples/simple_project/README.md | 1 + examples/simple_project_with_regional_network/README.md | 1 + examples/submodule_firewall/README.md | 9 +++++++++ 4 files changed, 12 insertions(+) diff --git a/examples/secondary_ranges/README.md b/examples/secondary_ranges/README.md index b31e8e3c..3ffc4ec3 100644 --- a/examples/secondary_ranges/README.md +++ b/examples/secondary_ranges/README.md @@ -26,5 +26,6 @@ ranges and the third being given a single secondary range. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | +| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/simple_project/README.md b/examples/simple_project/README.md index 583a5c48..3f6cb802 100644 --- a/examples/simple_project/README.md +++ b/examples/simple_project/README.md @@ -25,5 +25,6 @@ This VPC has two subnets, with no secondary ranges. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | +| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/simple_project_with_regional_network/README.md b/examples/simple_project_with_regional_network/README.md index 898c9e96..445f3111 100644 --- a/examples/simple_project_with_regional_network/README.md +++ b/examples/simple_project_with_regional_network/README.md @@ -25,5 +25,6 @@ This VPC has two subnets, with no secondary ranges. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | +| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/submodule_firewall/README.md b/examples/submodule_firewall/README.md index 1cb360d8..faf52212 100644 --- a/examples/submodule_firewall/README.md +++ b/examples/submodule_firewall/README.md @@ -19,5 +19,14 @@ This VPC has two subnets, with no secondary ranges. | admin\_ranges | Firewall attributes for admin ranges. | | internal\_ranges | Firewall attributes for internal ranges. | | network\_name | The name of the VPC being created | +| network\_self\_link | The URI of the VPC being created | +| routes | The routes associated with this VPC | +| subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | +| subnets\_ips | The IP and cidrs of the subnets being created | +| subnets\_names | The names of the subnets being created | +| subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | +| subnets\_regions | The region where subnets will be created | +| subnets\_secondary\_ranges | The secondary ranges associated with these subnets | +| svpc\_host\_project\_id | Shared VPC host project id. | From a440fce052c8ca710ac61af5c860c1aa2087be17 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Thu, 26 Sep 2019 11:09:23 -0400 Subject: [PATCH 15/78] wip: adding tests for outputs in the firewall submodule --- .kitchen.yml | 12 ++++ .../controls/inspec_attributes.rb | 61 +++++++++++++++++++ .../integration/submodule_firewall/inspec.yml | 22 +++++++ 3 files changed, 95 insertions(+) create mode 100644 test/integration/submodule_firewall/controls/inspec_attributes.rb diff --git a/.kitchen.yml b/.kitchen.yml index 7f175b52..ed06b86b 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -121,6 +121,18 @@ suites: controls: - gcp - name: local + attrs_outputs: + customized_inspec_attribute: output_network_name + customized_inspec_attribute: output_network_self_link + customized_inspec_attribute: output_subnets_ips + customized_inspec_attribute: output_routes + customized_inspec_attribute: output_subnets_flow_logs + customized_inspec_attribute: output_subnets_names + customized_inspec_attribute: output_subnets_private_access + customized_inspec_attribute: output_subnets_regions + customized_inspec_attribute: output_subnets_secondary_ranges + customized_inspec_attribute: output_svpc_host_project_id backend: local controls: - gcloud + - inspec_attributes diff --git a/test/integration/submodule_firewall/controls/inspec_attributes.rb b/test/integration/submodule_firewall/controls/inspec_attributes.rb new file mode 100644 index 00000000..05ca71cb --- /dev/null +++ b/test/integration/submodule_firewall/controls/inspec_attributes.rb @@ -0,0 +1,61 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +network_name = attribute('network_name') + +control "inspec_attributes" do + title "Terraform Outputs" + desc "Terraform Outputs" + + describe attribute("output_network_name") do + it { should eq "#{network_name}" } + end + + describe attribute("output_network_self_link") do + it { should eq "https://www.googleapis.com/compute/v1/projects/#{project_id}/global/networks/#{network_name}" } + end + + describe attribute("output_subnets_ips") do + it { should eq ["10.10.10.0/24", "10.10.20.0/24"] } + end + + describe attribute("output_routes") do + it { should eq [] } + end + + describe attribute("output_subnets_flow_logs") do + it { should eq [false, true] } + end + + describe attribute("output_subnets_names") do + it { should eq ["#{network_name}-subnet-01", "#{network_name}-subnet-02"] } + end + + describe attribute("output_subnets_private_access") do + it { should eq [false, true] } + end + + describe attribute("output_subnets_regions") do + it { should eq ["us-west1", "us-west1"] } + end + + describe attribute("output_subnets_secondary_ranges") do + it { should eq [[],[]] } + end + + describe attribute("output_svpc_host_project_id") do + it { should eq "" } + end +end diff --git a/test/integration/submodule_firewall/inspec.yml b/test/integration/submodule_firewall/inspec.yml index 7e69b529..67efdd29 100644 --- a/test/integration/submodule_firewall/inspec.yml +++ b/test/integration/submodule_firewall/inspec.yml @@ -10,3 +10,25 @@ attributes: - name: network_name required: true type: string + - name: output_network_name + required: true + type: string + - name: output_network_self_link + required: true + type: string + - name: output_subnets_ips + required: true + - name: output_routes + required: true + - name: output_subnets_flow_logs + required: true + - name: output_subnets_names + required: true + - name: output_subnets_private_access + required: true + - name: output_subnets_regions + required: true + - name: output_subnets_secondary_ranges + required: true + - name: output_svpc_host_project_id + required: true From c19fd95ada312a6f4ab53c3a70043b9f8ae698a9 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 2 Oct 2019 10:48:58 -0400 Subject: [PATCH 16/78] fix: syntax update to make the for loop more readable --- main.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 78b2a837..3be7e062 100644 --- a/main.tf +++ b/main.tf @@ -53,8 +53,17 @@ resource "google_compute_subnetwork" "subnetwork" { enable_flow_logs = lookup(each.value, "subnet_flow_logs", "false") network = google_compute_network.network.name project = var.project_id - secondary_ip_range = [for i in range(length(contains(keys(var.secondary_ranges), each.value.subnet_name) == true ? var.secondary_ranges[each.value.subnet_name] : [])) : var.secondary_ranges[each.value.subnet_name][i]] description = lookup(each.value, "description", null) + secondary_ip_range = [ + for i in range( + length( + contains( + keys(var.secondary_ranges), each.value.subnet_name) == true + ? var.secondary_ranges[each.value.subnet_name] + : [] + )) : + var.secondary_ranges[each.value.subnet_name][i] + ] } /****************************************** From f886fd79ab850bd045cc502a2383589af63a9742 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 2 Oct 2019 11:39:31 -0400 Subject: [PATCH 17/78] docs: WIP on migration docs --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a72194..fa092fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog][keepachangelog-site], and this project adheres to [Semantic Versioning][semver-site]. ## [Unreleased] +v2.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v2.0.md). + +### Fixed + +- Fixes subnet recreation when a subnet is updated. [#73] + ## [1.3.0] - 2019-10-09 From 42330545cc1ce8aa684c6f365e654a7c140ecb24 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 2 Oct 2019 14:42:02 -0400 Subject: [PATCH 18/78] feat: added migrations docs and a migration script --- docs/upgrading_to_v2.0.md | 151 ++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 16 ++++ 2 files changed, 167 insertions(+) create mode 100644 docs/upgrading_to_v2.0.md create mode 100755 scripts/migrate.sh diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md new file mode 100644 index 00000000..17711e6b --- /dev/null +++ b/docs/upgrading_to_v2.0.md @@ -0,0 +1,151 @@ +# Upgrading to v2.x + +The v2.x release of _google-network_ is a backwards incompatible +release. + +Because v2.x changed how the subnet resource is iterated on, resources in Terraform state need to be migrated in order to avoid the resources from getting destroyed and recreated. + +## Migration Instructions + +- Upgrade to the new version of this module. + +if you run `terraform plan` at this point. Terraform will inform you that it will attempt to delete and recreate your existing subnets. This is almost certainly not the behavior you want. For example: + +```Shell +Terraform will perform the following actions: + + # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork will be destroyed + - resource "google_compute_subnetwork" "subnetwork" { + - creation_timestamp = "2019-10-02T08:40:26.282-07:00" -> null + - enable_flow_logs = false -> null + - fingerprint = "f8LZx006zY4=" -> null + - gateway_address = "10.10.10.1" -> null + - id = "us-west1/simple-project-timh-subnet-01" -> null + - ip_cidr_range = "10.10.10.0/24" -> null + - name = "simple-project-timh-subnet-01" -> null + - network = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/global/networks/simple-project-timh" -> null + - private_ip_google_access = false -> null + - project = "dev-xpn-networking" -> null + - region = "us-west1" -> null + - secondary_ip_range = [] -> null + - self_link = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/regions/us-west1/subnetworks/simple-project-timh-subnet-01" -> null + } + + # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] will be destroyed + - resource "google_compute_subnetwork" "subnetwork" { + - creation_timestamp = "2019-10-02T08:40:26.292-07:00" -> null + - enable_flow_logs = true -> null + - fingerprint = "wOwStN9lK-Q=" -> null + - gateway_address = "10.10.20.1" -> null + - id = "us-west1/simple-project-timh-subnet-02" -> null + - ip_cidr_range = "10.10.20.0/24" -> null + - name = "simple-project-timh-subnet-02" -> null + - network = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/global/networks/simple-project-timh" -> null + - private_ip_google_access = true -> null + - project = "dev-xpn-networking" -> null + - region = "us-west1" -> null + - secondary_ip_range = [] -> null + - self_link = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/regions/us-west1/subnetworks/simple-project-timh-subnet-02" -> null + } + + # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-01"] will be created + + resource "google_compute_subnetwork" "subnetwork" { + + creation_timestamp = (known after apply) + + enable_flow_logs = false + + fingerprint = (known after apply) + + gateway_address = (known after apply) + + id = (known after apply) + + ip_cidr_range = "10.10.10.0/24" + + name = "simple-project-timh-subnet-01" + + network = "simple-project-timh" + + private_ip_google_access = false + + project = "dev-xpn-networking" + + region = "us-west1" + + secondary_ip_range = [] + + self_link = (known after apply) + } + + # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"] will be created + + resource "google_compute_subnetwork" "subnetwork" { + + creation_timestamp = (known after apply) + + enable_flow_logs = true + + fingerprint = (known after apply) + + gateway_address = (known after apply) + + id = (known after apply) + + ip_cidr_range = "10.10.20.0/24" + + name = "simple-project-timh-subnet-02" + + network = "simple-project-timh" + + private_ip_google_access = true + + project = "dev-xpn-networking" + + region = "us-west1" + + secondary_ip_range = [] + + self_link = (known after apply) + } + +Plan: 2 to add, 0 to change, 2 to destroy. + +------------------------------------------------------------------------ + +Note: You didn't specify an "-out" parameter to save this plan, so Terraform +can't guarantee that exactly these actions will be performed if +"terraform apply" is subsequently run. +``` + +### Manual Migration Steps + +In this example here are the two commands used migrate the subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. + +- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` + +- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]` + +`terraform plan` should now return a no-op and show no new changes. + +```Shell +$ terraform plan +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +module.example.module.test-vpc-module.google_compute_network.network: Refreshing state... [id=simple-project-timh] +module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-02] +module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-01"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-01] + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +``` + +### Migration Script + +1. Download the script + + ```sh + curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/scripts/migrate.sh + chmod +x migrate.sh + ``` + +2. Run the script to output the migration commands: + + ```sh + $ MODULE_NAME="test-vpc-module" ./migrate.sh + terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] + terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] + ``` + +3. Execute the migration command + + ```sh + $ terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] + Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]" + Successfully moved 1 object(s). + $ terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] + Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]" + Successfully moved 1 object(s). + ``` + +4. Run `terraform plan` to confirm no changes are expected. diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 00000000..69658434 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Output Terraform Commands to migrate to new subnet config +set -e +set -o pipefail + +if [[ "$MODULE_NAME" ]]; then + NAME=$(terraform state list | grep ${MODULE_NAME}.google_compute_network.network | sed 's/.google_compute_network.network//') + for x in $(terraform state list | grep ${NAME}.google_compute_subnetwork.subnetwork); do + ID=$(terraform state show $x | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') + echo "terraform state mv $x ${NAME}.google_compute_subnetwork.subnetwork[\\\"${ID}\\\"]" + done +else + echo "MISSING MODULE_NAME: MODULE_NAME env var is required" + exit 1 +fi From a1d5d044e8e72f2b1b7839ee2d76e7f9194768ea Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Wed, 2 Oct 2019 14:54:19 -0400 Subject: [PATCH 19/78] lint: Fixing docs and linting errors --- docs/upgrading_to_v2.0.md | 2 +- scripts/migrate.sh | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index 17711e6b..ea7bd714 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -7,7 +7,7 @@ Because v2.x changed how the subnet resource is iterated on, resources in Terraf ## Migration Instructions -- Upgrade to the new version of this module. +- Upgrade to the new version of this module. if you run `terraform plan` at this point. Terraform will inform you that it will attempt to delete and recreate your existing subnets. This is almost certainly not the behavior you want. For example: diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 69658434..a36e2d8c 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -1,13 +1,26 @@ #!/usr/bin/env bash +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # shellcheck shell=bash # Output Terraform Commands to migrate to new subnet config set -e set -o pipefail if [[ "$MODULE_NAME" ]]; then - NAME=$(terraform state list | grep ${MODULE_NAME}.google_compute_network.network | sed 's/.google_compute_network.network//') - for x in $(terraform state list | grep ${NAME}.google_compute_subnetwork.subnetwork); do - ID=$(terraform state show $x | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') + NAME=$(terraform state list | grep "${MODULE_NAME}".google_compute_network.network | sed 's/.google_compute_network.network//') + for x in $(terraform state list | grep "${NAME}".google_compute_subnetwork.subnetwork); do + ID=$(terraform state show "$x" | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') echo "terraform state mv $x ${NAME}.google_compute_subnetwork.subnetwork[\\\"${ID}\\\"]" done else From 0e771c960725755e024c9dd3d916c0df4a5c7e25 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 4 Oct 2019 09:43:46 -0400 Subject: [PATCH 20/78] wip: updated migration script to run migration commands --- docs/upgrading_to_v2.0.md | 7 +++---- scripts/migrate.sh | 42 +++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index ea7bd714..efa5285f 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -93,7 +93,7 @@ can't guarantee that exactly these actions will be performed if ### Manual Migration Steps -In this example here are the two commands used migrate the subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. +In this example here are the two commands used migrate the subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. You may also use the migration script. - `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` @@ -132,7 +132,7 @@ actions need to be performed. 2. Run the script to output the migration commands: ```sh - $ MODULE_NAME="test-vpc-module" ./migrate.sh + $ MODULE_NAME="test-vpc-module" ./migrate.sh --dry-run terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] ``` @@ -140,10 +140,9 @@ actions need to be performed. 3. Execute the migration command ```sh - $ terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] + $ MODULE_NAME="test-vpc-module" ./migrate.sh Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]" Successfully moved 1 object(s). - $ terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]" Successfully moved 1 object(s). ``` diff --git a/scripts/migrate.sh b/scripts/migrate.sh index a36e2d8c..ac4fb434 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -16,12 +16,46 @@ # Output Terraform Commands to migrate to new subnet config set -e set -o pipefail +CMD="terraform state" + +while (( "$#" )); do + # shellcheck disable=SC2221,SC2222 + case "$1" in + -d|--dry-run) + DRY_RUN=true + shift 1 + ;; + --) # end argument parsing + shift + break + ;; + -*|--*=) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +eval set -- "$PARAMS" + +if [ ! -e "$(command -v terraform)" ]; then + echo "can not find terraform" + exit 1 +fi if [[ "$MODULE_NAME" ]]; then - NAME=$(terraform state list | grep "${MODULE_NAME}".google_compute_network.network | sed 's/.google_compute_network.network//') - for x in $(terraform state list | grep "${NAME}".google_compute_subnetwork.subnetwork); do - ID=$(terraform state show "$x" | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') - echo "terraform state mv $x ${NAME}.google_compute_subnetwork.subnetwork[\\\"${ID}\\\"]" + NAME=$(${CMD} list | grep "${MODULE_NAME}".google_compute_network.network | sed 's/.google_compute_network.network//') + for x in $($CMD list | grep "${NAME}".google_compute_subnetwork.subnetwork); do + ID=$(${CMD} show "$x" | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') + if [[ $DRY_RUN ]]; then + echo "${CMD} mv $x ${NAME}.google_compute_subnetwork.subnetwork[\\\"${ID}\\\"]" + else + ${CMD} mv "$x" "${NAME}".google_compute_subnetwork.subnetwork[\""${ID}"\"] + fi done else echo "MISSING MODULE_NAME: MODULE_NAME env var is required" From aee0cc0b9afb8892050b8abbccb5ef2cecfc93be Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 11 Oct 2019 11:45:33 -0400 Subject: [PATCH 21/78] rebase: add back depends_on for subnets after a rebase Making sure `depends_on = [google_compute_subnetwork.subnetwork]` is correctly added in after a rebase as per https://github.com/terraform-google-modules/terraform-google-network/pull/81 --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 3be7e062..cf725ad6 100644 --- a/main.tf +++ b/main.tf @@ -54,6 +54,7 @@ resource "google_compute_subnetwork" "subnetwork" { network = google_compute_network.network.name project = var.project_id description = lookup(each.value, "description", null) + depends_on = [google_compute_subnetwork.subnetwork] secondary_ip_range = [ for i in range( length( From 14f3be38c23d8fbe615132542a7e77d93e4fa093 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Fri, 11 Oct 2019 11:51:04 -0400 Subject: [PATCH 22/78] Revert "rebase: add back depends_on for subnets after a rebase" This reverts commit aee0cc0b9afb8892050b8abbccb5ef2cecfc93be. --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index cf725ad6..3be7e062 100644 --- a/main.tf +++ b/main.tf @@ -54,7 +54,6 @@ resource "google_compute_subnetwork" "subnetwork" { network = google_compute_network.network.name project = var.project_id description = lookup(each.value, "description", null) - depends_on = [google_compute_subnetwork.subnetwork] secondary_ip_range = [ for i in range( length( From f3bebfc8612d101dd0993bd07707ab1a55fcf985 Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Tue, 15 Oct 2019 17:36:43 -0400 Subject: [PATCH 23/78] moved migrate.sh into the helpers dir --- {scripts => helpers}/migrate.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {scripts => helpers}/migrate.sh (100%) diff --git a/scripts/migrate.sh b/helpers/migrate.sh similarity index 100% rename from scripts/migrate.sh rename to helpers/migrate.sh From 1c99d287145e223157b2ed5fc6e375ad8b307f3d Mon Sep 17 00:00:00 2001 From: Tim Hartmann Date: Tue, 15 Oct 2019 18:06:10 -0400 Subject: [PATCH 24/78] feat: made the scipt work across multiple modules The migrate script will now migrate all the modules --- docs/upgrading_to_v2.0.md | 6 +++--- helpers/migrate.sh | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index efa5285f..cb397a0c 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -125,14 +125,14 @@ actions need to be performed. 1. Download the script ```sh - curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/scripts/migrate.sh + curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/helpers/migrate.sh chmod +x migrate.sh ``` 2. Run the script to output the migration commands: ```sh - $ MODULE_NAME="test-vpc-module" ./migrate.sh --dry-run + $ ./migrate.sh --dry-run terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] ``` @@ -140,7 +140,7 @@ actions need to be performed. 3. Execute the migration command ```sh - $ MODULE_NAME="test-vpc-module" ./migrate.sh + $ ./migrate.sh Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]" Successfully moved 1 object(s). Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]" diff --git a/helpers/migrate.sh b/helpers/migrate.sh index ac4fb434..752fc06a 100755 --- a/helpers/migrate.sh +++ b/helpers/migrate.sh @@ -47,8 +47,9 @@ if [ ! -e "$(command -v terraform)" ]; then exit 1 fi -if [[ "$MODULE_NAME" ]]; then - NAME=$(${CMD} list | grep "${MODULE_NAME}".google_compute_network.network | sed 's/.google_compute_network.network//') +MODULES=$(${CMD} list | grep google_compute_network.network) +for module in $MODULES; do + NAME=$(sed 's/.google_compute_network.network//' <<<"${module}") for x in $($CMD list | grep "${NAME}".google_compute_subnetwork.subnetwork); do ID=$(${CMD} show "$x" | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') if [[ $DRY_RUN ]]; then @@ -57,7 +58,4 @@ if [[ "$MODULE_NAME" ]]; then ${CMD} mv "$x" "${NAME}".google_compute_subnetwork.subnetwork[\""${ID}"\"] fi done -else - echo "MISSING MODULE_NAME: MODULE_NAME env var is required" - exit 1 -fi +done From 2d1b821e9ab84b89def340185c41f87abd939e4d Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:48:54 +0000 Subject: [PATCH 25/78] Updates gcp.rb test for simple_project Adds a test for the new subnetwork for ip_cidr_range and private_ip_google_access --- test/integration/simple_project/controls/gcp.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/integration/simple_project/controls/gcp.rb b/test/integration/simple_project/controls/gcp.rb index b62b3f32..d48c79da 100644 --- a/test/integration/simple_project/controls/gcp.rb +++ b/test/integration/simple_project/controls/gcp.rb @@ -44,4 +44,14 @@ its('ip_cidr_range') { should eq "10.10.20.0/24" } its('private_ip_google_access') { should be true } end + + describe google_compute_subnetwork( + project: project_id, + name: "#{network_name}-subnet-03", + region: "us-west1" + ) do + it { should exist } + its('ip_cidr_range') { should eq "10.10.30.0/24" } + its('private_ip_google_access') { should be false } + end end From 7224c7dcba99e3cbde62d361407976a6450d93c0 Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:51:22 +0000 Subject: [PATCH 26/78] Updates gcloud.rb for simple-project Updates the test for enableFlowLogs as behaviour has changed slightly. Adds a test for logConfig to ensure it has correct values. --- .../simple_project/controls/gcloud.rb | 65 +++++++++++++++---- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/test/integration/simple_project/controls/gcloud.rb b/test/integration/simple_project/controls/gcloud.rb index a22b2d80..73770483 100644 --- a/test/integration/simple_project/controls/gcloud.rb +++ b/test/integration/simple_project/controls/gcloud.rb @@ -30,12 +30,16 @@ end end - describe "enableFlowLogs" do - it "should be false" do - expect(data).to include( - "enableFlowLogs" => false - ) - end + it "enableFlowLogs should not exist" do + expect(data).to_not include( + "enableFlowLogs" + ) + end + + it "logConfig should not exist" do + expect(data).to_not include( + "logConfig" + ) end end @@ -51,12 +55,51 @@ end end - describe "enableFlowLogs" do - it "should be true" do - expect(data).to include( - "enableFlowLogs" => true - ) + it "enableFlowLogs should be true" do + expect(data).to include( + "enableFlowLogs" => true + ) + end + + it "Log config should be correct" do + expect(data).to include( + "logConfig" => { + "aggregationInterval" => "INTERVAL_5_SEC", + "enable" => true, + "flowSampling" => 0.5, + "metadata" => "INCLUDE_ALL_METADATA" + } + ) + end + end + + describe command("gcloud compute networks subnets describe #{network_name}-subnet-03 --project=#{project_id} --region=us-west1 --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} end end + + it "enableFlowLogs should be true" do + expect(data).to include( + "enableFlowLogs" => true + ) + end + + it "Log config should be correct" do + expect(data).to include( + "logConfig" => { + "aggregationInterval" => "INTERVAL_10_MIN", + "enable" => true, + "flowSampling" => 0.7, + "metadata" => "INCLUDE_ALL_METADATA" + } + ) + end end end From 2ac57e2976e5f4c3c2cb6ca98298c1a7d535774b Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:52:30 +0000 Subject: [PATCH 27/78] Updates simple-project example Updates google provider to 2.19.0. Adds a subnet with flow logs log config set. --- examples/simple_project/main.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index 13c4a716..849e1eb5 100644 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { @@ -25,6 +25,7 @@ provider "null" { locals { subnet_01 = "${var.network_name}-subnet-01" subnet_02 = "${var.network_name}-subnet-02" + subnet_03 = "${var.network_name}-subnet-03" } module "test-vpc-module" { @@ -45,5 +46,13 @@ module "test-vpc-module" { subnet_private_access = "true" subnet_flow_logs = "true" }, + { + subnet_name = "${local.subnet_03}" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1" + subnet_flow_logs_interval = "INTERVAL_10_MIN" + subnet_flow_logs_sampling = 0.7 + subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + } ] } From 5a8f0ee397a5a469f1628da82a767ae45905c07a Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:55:33 +0000 Subject: [PATCH 28/78] Updates main.tf Adds a dynamic log_config block. This checks if subnet_flow_logs, or any of the log config settings are set. If they are set, log_config exists and therefore flow logs are enabled. --- main.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 3be7e062..2ff00ea9 100644 --- a/main.tf +++ b/main.tf @@ -50,7 +50,18 @@ resource "google_compute_subnetwork" "subnetwork" { ip_cidr_range = each.value.subnet_ip region = each.value.subnet_region private_ip_google_access = lookup(each.value, "subnet_private_access", "false") - enable_flow_logs = lookup(each.value, "subnet_flow_logs", "false") + dynamic "log_config" { + for_each = lookup(each.value, "subnet_flow_logs", false) || lookup(each.value, "subnet_flow_logs_interval", null) != null || lookup(each.value, "subnet_flow_logs_sampling", null) != null || lookup(each.value, "subnet_flow_logs_metadata", null) != null ? [{ + aggregation_interval = lookup(each.value, "subnet_flow_logs_interval", null) + flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) + metadata = lookup(each.value, "subnet_flow_logs_metadata", null) + }] : [] + content { + aggregation_interval = log_config.value.aggregation_interval + flow_sampling = log_config.value.flow_sampling + metadata = log_config.value.metadata + } + } network = google_compute_network.network.name project = var.project_id description = lookup(each.value, "description", null) From d5482242a26ff4edcc221d17f86e23b24072a6e9 Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:57:34 +0000 Subject: [PATCH 29/78] Formats terraform --- examples/simple_project/main.tf | 6 +++--- main.tf | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index 849e1eb5..19df39cb 100644 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -47,9 +47,9 @@ module "test-vpc-module" { subnet_flow_logs = "true" }, { - subnet_name = "${local.subnet_03}" - subnet_ip = "10.10.30.0/24" - subnet_region = "us-west1" + subnet_name = "${local.subnet_03}" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1" subnet_flow_logs_interval = "INTERVAL_10_MIN" subnet_flow_logs_sampling = 0.7 subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" diff --git a/main.tf b/main.tf index 2ff00ea9..0ffd0242 100644 --- a/main.tf +++ b/main.tf @@ -51,20 +51,20 @@ resource "google_compute_subnetwork" "subnetwork" { region = each.value.subnet_region private_ip_google_access = lookup(each.value, "subnet_private_access", "false") dynamic "log_config" { - for_each = lookup(each.value, "subnet_flow_logs", false) || lookup(each.value, "subnet_flow_logs_interval", null) != null || lookup(each.value, "subnet_flow_logs_sampling", null) != null || lookup(each.value, "subnet_flow_logs_metadata", null) != null ? [{ + for_each = lookup(each.value, "subnet_flow_logs", false) || lookup(each.value, "subnet_flow_logs_interval", null) != null || lookup(each.value, "subnet_flow_logs_sampling", null) != null || lookup(each.value, "subnet_flow_logs_metadata", null) != null ? [{ aggregation_interval = lookup(each.value, "subnet_flow_logs_interval", null) - flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) - metadata = lookup(each.value, "subnet_flow_logs_metadata", null) + flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) + metadata = lookup(each.value, "subnet_flow_logs_metadata", null) }] : [] content { aggregation_interval = log_config.value.aggregation_interval - flow_sampling = log_config.value.flow_sampling - metadata = log_config.value.metadata + flow_sampling = log_config.value.flow_sampling + metadata = log_config.value.metadata } } - network = google_compute_network.network.name - project = var.project_id - description = lookup(each.value, "description", null) + network = google_compute_network.network.name + project = var.project_id + description = lookup(each.value, "description", null) secondary_ip_range = [ for i in range( length( From e8063621042893983afb3ee65257610bb0e6e85c Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Thu, 14 Nov 2019 17:59:38 +0000 Subject: [PATCH 30/78] Updates README example --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index f219b6a7..42947153 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,14 @@ module "vpc" { subnet_flow_logs = "true" description = "This subnet has a description" }, + { + subnet_name = "subnet-03" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1", + subnet_flow_logs_interval = "INTERVAL_10_MIN", + subnet_flow_logs_sampling = 0.5, + subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + } ] secondary_ranges = { From 9983e144828e698b020c39a19374e3ba13ea63e0 Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Fri, 15 Nov 2019 15:17:44 +0000 Subject: [PATCH 31/78] Reduces complexity of log_config conditional The previous conditional was very complex and so this can be reduced by requiring `subnet_flow_logs` to be set if `log_config` is to be set. --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 0ffd0242..e6057d9e 100644 --- a/main.tf +++ b/main.tf @@ -51,7 +51,7 @@ resource "google_compute_subnetwork" "subnetwork" { region = each.value.subnet_region private_ip_google_access = lookup(each.value, "subnet_private_access", "false") dynamic "log_config" { - for_each = lookup(each.value, "subnet_flow_logs", false) || lookup(each.value, "subnet_flow_logs_interval", null) != null || lookup(each.value, "subnet_flow_logs_sampling", null) != null || lookup(each.value, "subnet_flow_logs_metadata", null) != null ? [{ + for_each = lookup(each.value, "subnet_flow_logs", false) ? [{ aggregation_interval = lookup(each.value, "subnet_flow_logs_interval", null) flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) metadata = lookup(each.value, "subnet_flow_logs_metadata", null) From 035a1b4d253f157c9a09b01538f96c79ede9a630 Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Fri, 15 Nov 2019 15:18:30 +0000 Subject: [PATCH 32/78] Removes `enable flow logs` tests These tests are no longer required as this setting is being deprecated on the API. --- .../simple_project/controls/gcloud.rb | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/integration/simple_project/controls/gcloud.rb b/test/integration/simple_project/controls/gcloud.rb index 73770483..3e7fc768 100644 --- a/test/integration/simple_project/controls/gcloud.rb +++ b/test/integration/simple_project/controls/gcloud.rb @@ -30,12 +30,6 @@ end end - it "enableFlowLogs should not exist" do - expect(data).to_not include( - "enableFlowLogs" - ) - end - it "logConfig should not exist" do expect(data).to_not include( "logConfig" @@ -55,12 +49,6 @@ end end - it "enableFlowLogs should be true" do - expect(data).to include( - "enableFlowLogs" => true - ) - end - it "Log config should be correct" do expect(data).to include( "logConfig" => { @@ -85,12 +73,6 @@ end end - it "enableFlowLogs should be true" do - expect(data).to include( - "enableFlowLogs" => true - ) - end - it "Log config should be correct" do expect(data).to include( "logConfig" => { From a9d2dbf3d00c92481dde86de83c6f08e35119c2b Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Fri, 15 Nov 2019 15:48:00 +0000 Subject: [PATCH 33/78] Updates simple-project example Adds `subnet_flow_logs` to subnet 3 due to new condition. --- examples/simple_project/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index 19df39cb..40519adc 100644 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -50,6 +50,7 @@ module "test-vpc-module" { subnet_name = "${local.subnet_03}" subnet_ip = "10.10.30.0/24" subnet_region = "us-west1" + subnet_flow_logs = "true" subnet_flow_logs_interval = "INTERVAL_10_MIN" subnet_flow_logs_sampling = 0.7 subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" From b3607cb6debd6695fb49f06a546029b9c5399f64 Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Fri, 15 Nov 2019 15:50:59 +0000 Subject: [PATCH 34/78] Updates README.md Updates the example to show the subnet_flow_logs behaviour. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 42947153..1559a0eb 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,10 @@ module "vpc" { { subnet_name = "subnet-03" subnet_ip = "10.10.30.0/24" - subnet_region = "us-west1", - subnet_flow_logs_interval = "INTERVAL_10_MIN", - subnet_flow_logs_sampling = 0.5, + subnet_region = "us-west1" + subnet_flow_logs = "true" + subnet_flow_logs_interval = "INTERVAL_10_MIN" + subnet_flow_logs_sampling = 0.7 subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" } ] From 8ec4e16a8c8ed53107f779d26e2442f9a25b1abb Mon Sep 17 00:00:00 2001 From: Josh Hill Date: Fri, 15 Nov 2019 16:35:56 +0000 Subject: [PATCH 35/78] Updates examples provider version Upgrades to 2.19.0 to introduce log_config so tests can run. --- README.md | 2 +- examples/delete_default_gateway_routes/main.tf | 2 +- examples/multi_vpc/main.tf | 2 +- examples/secondary_ranges/main.tf | 2 +- examples/simple_project_with_regional_network/main.tf | 2 +- examples/submodule_firewall/main.tf | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1559a0eb..e6beea3f 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ The routes list contains maps, where each object represents a route. For the nex ## Requirements ### Installed Software - [Terraform](https://www.terraform.io/downloads.html) ~> 0.12.0 -- [Terraform Provider for GCP][terraform-provider-google] ~> 2.10.0 +- [Terraform Provider for GCP][terraform-provider-google] ~> 2.19.0 - [gcloud](https://cloud.google.com/sdk/gcloud/) >243.0.0 ### Configure a Service Account diff --git a/examples/delete_default_gateway_routes/main.tf b/examples/delete_default_gateway_routes/main.tf index 7b48d150..1269b0e9 100644 --- a/examples/delete_default_gateway_routes/main.tf +++ b/examples/delete_default_gateway_routes/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { diff --git a/examples/multi_vpc/main.tf b/examples/multi_vpc/main.tf index 7c039a5b..1d0b7247 100644 --- a/examples/multi_vpc/main.tf +++ b/examples/multi_vpc/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { diff --git a/examples/secondary_ranges/main.tf b/examples/secondary_ranges/main.tf index 1ddcb460..576ff002 100644 --- a/examples/secondary_ranges/main.tf +++ b/examples/secondary_ranges/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { diff --git a/examples/simple_project_with_regional_network/main.tf b/examples/simple_project_with_regional_network/main.tf index 58b0ba4e..583a21db 100644 --- a/examples/simple_project_with_regional_network/main.tf +++ b/examples/simple_project_with_regional_network/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { diff --git a/examples/submodule_firewall/main.tf b/examples/submodule_firewall/main.tf index 8c1ed331..4319be80 100644 --- a/examples/submodule_firewall/main.tf +++ b/examples/submodule_firewall/main.tf @@ -15,7 +15,7 @@ */ provider "google" { - version = "~> 2.10.0" + version = "~> 2.19.0" } provider "null" { From dedcf4c0bce237c0e480b4d2389e044acb63862b Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 11:03:42 -0800 Subject: [PATCH 36/78] fix makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 50d30874..69ee80e1 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ docker_test_prepare: # Clean up test environment within the docker container .PHONY: docker_test_cleanup -docker_test_prepare: +docker_test_cleanup: docker run --rm -it \ -e SERVICE_ACCOUNT_JSON \ -e TF_VAR_org_id \ From 0a135c0189d3d8eee68c7c594e5dab9cc2819191 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 11:04:37 -0800 Subject: [PATCH 37/78] move vpc creation to its own module --- main.tf | 23 +++++++------------- modules/vpc/main.tf | 35 ++++++++++++++++++++++++++++++ modules/vpc/outputs.tf | 35 ++++++++++++++++++++++++++++++ modules/vpc/variables.tf | 47 ++++++++++++++++++++++++++++++++++++++++ outputs.tf | 6 ++--- 5 files changed, 128 insertions(+), 18 deletions(-) create mode 100644 modules/vpc/main.tf create mode 100644 modules/vpc/outputs.tf create mode 100644 modules/vpc/variables.tf diff --git a/main.tf b/main.tf index e6057d9e..e3dd6253 100644 --- a/main.tf +++ b/main.tf @@ -24,21 +24,14 @@ locals { /****************************************** VPC configuration *****************************************/ -resource "google_compute_network" "network" { - name = var.network_name +module "vpc" { + source = "./modules/vpc" + network_name = var.network_name auto_create_subnetworks = var.auto_create_subnetworks routing_mode = var.routing_mode - project = var.project_id + project_id = var.project_id description = var.description -} - -/****************************************** - Shared VPC - *****************************************/ -resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { - count = var.shared_vpc_host == "true" ? 1 : 0 - project = var.project_id - depends_on = [google_compute_network.network] + shared_vpc_host = var.shared_vpc_host } /****************************************** @@ -62,7 +55,7 @@ resource "google_compute_subnetwork" "subnetwork" { metadata = log_config.value.metadata } } - network = google_compute_network.network.name + network = module.vpc.network_name project = var.project_id description = lookup(each.value, "description", null) secondary_ip_range = [ @@ -96,7 +89,7 @@ resource "google_compute_route" "route" { priority = lookup(var.routes[count.index], "priority", "1000") depends_on = [ - google_compute_network.network, + module.vpc.network, google_compute_subnetwork.subnetwork, ] } @@ -113,7 +106,7 @@ resource "null_resource" "delete_default_internet_gateway_routes" { } depends_on = [ - google_compute_network.network, + module.vpc.network, google_compute_subnetwork.subnetwork, google_compute_route.route, ] diff --git a/modules/vpc/main.tf b/modules/vpc/main.tf new file mode 100644 index 00000000..a1b249ae --- /dev/null +++ b/modules/vpc/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + VPC configuration + *****************************************/ +resource "google_compute_network" "network" { + name = var.network_name + auto_create_subnetworks = var.auto_create_subnetworks + routing_mode = var.routing_mode + project = var.project_id + description = var.description +} + +/****************************************** + Shared VPC + *****************************************/ +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + count = var.shared_vpc_host == "true" ? 1 : 0 + project = var.project_id + depends_on = [google_compute_network.network] +} diff --git a/modules/vpc/outputs.tf b/modules/vpc/outputs.tf new file mode 100644 index 00000000..e5238214 --- /dev/null +++ b/modules/vpc/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network" { + value = google_compute_network.network + description = "The VPC resource being created" +} + +output "network_name" { + value = google_compute_network.network.name + description = "The name of the VPC being created" +} + +output "network_self_link" { + value = google_compute_network.network.self_link + description = "The URI of the VPC being created" +} + +output "svpc_host_project_id" { + value = element(concat(google_compute_shared_vpc_host_project.shared_vpc_host.*.project, list("")), 0) + description = "Shared VPC host project id." +} diff --git a/modules/vpc/variables.tf b/modules/vpc/variables.tf new file mode 100644 index 00000000..80089284 --- /dev/null +++ b/modules/vpc/variables.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project where this VPC will be created" +} + +variable "network_name" { + description = "The name of the network being created" +} + +variable "routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default 'GLOBAL')" +} + +variable "shared_vpc_host" { + type = string + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = "false" +} + +variable "description" { + type = string + description = "An optional description of this resource. The resource must be recreated to modify this field." + default = "" +} + +variable "auto_create_subnetworks" { + type = bool + description = "When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources." + default = false +} diff --git a/outputs.tf b/outputs.tf index 11e707a9..a5fb462c 100644 --- a/outputs.tf +++ b/outputs.tf @@ -15,17 +15,17 @@ */ output "network_name" { - value = google_compute_network.network.name + value = module.vpc.network_name description = "The name of the VPC being created" } output "network_self_link" { - value = google_compute_network.network.self_link + value = module.vpc.network_self_link description = "The URI of the VPC being created" } output "svpc_host_project_id" { - value = element(concat(google_compute_shared_vpc_host_project.shared_vpc_host.*.project, list("")), 0) + value = module.vpc.svpc_host_project_id description = "Shared VPC host project id." } From 987b4e4298443d48adb7698b41bdd4c8a104d3bf Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 11:21:50 -0800 Subject: [PATCH 38/78] move subnets to own module --- main.tf | 49 ++++++------------------------ modules/subnets/main.tf | 59 ++++++++++++++++++++++++++++++++++++ modules/subnets/outputs.tf | 55 +++++++++++++++++++++++++++++++++ modules/subnets/variables.tf | 40 ++++++++++++++++++++++++ outputs.tf | 14 ++++----- 5 files changed, 170 insertions(+), 47 deletions(-) create mode 100644 modules/subnets/main.tf create mode 100644 modules/subnets/outputs.tf create mode 100644 modules/subnets/variables.tf diff --git a/main.tf b/main.tf index e3dd6253..c044018b 100644 --- a/main.tf +++ b/main.tf @@ -14,13 +14,6 @@ * limitations under the License. */ -locals { - subnets = { - for x in var.subnets : - "${x.subnet_region}/${x.subnet_name}" => x - } -} - /****************************************** VPC configuration *****************************************/ @@ -37,37 +30,13 @@ module "vpc" { /****************************************** Subnet configuration *****************************************/ -resource "google_compute_subnetwork" "subnetwork" { - for_each = local.subnets - name = each.value.subnet_name - ip_cidr_range = each.value.subnet_ip - region = each.value.subnet_region - private_ip_google_access = lookup(each.value, "subnet_private_access", "false") - dynamic "log_config" { - for_each = lookup(each.value, "subnet_flow_logs", false) ? [{ - aggregation_interval = lookup(each.value, "subnet_flow_logs_interval", null) - flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) - metadata = lookup(each.value, "subnet_flow_logs_metadata", null) - }] : [] - content { - aggregation_interval = log_config.value.aggregation_interval - flow_sampling = log_config.value.flow_sampling - metadata = log_config.value.metadata - } - } - network = module.vpc.network_name - project = var.project_id - description = lookup(each.value, "description", null) - secondary_ip_range = [ - for i in range( - length( - contains( - keys(var.secondary_ranges), each.value.subnet_name) == true - ? var.secondary_ranges[each.value.subnet_name] - : [] - )) : - var.secondary_ranges[each.value.subnet_name][i] - ] +module "subnets" { + source = "./modules/subnets" + project_id = var.project_id + network_name = module.vpc.network_name + subnets = var.subnets + description = var.description + secondary_ranges = var.secondary_ranges } /****************************************** @@ -90,7 +59,7 @@ resource "google_compute_route" "route" { depends_on = [ module.vpc.network, - google_compute_subnetwork.subnetwork, + module.subnets.subnets, ] } @@ -107,7 +76,7 @@ resource "null_resource" "delete_default_internet_gateway_routes" { depends_on = [ module.vpc.network, - google_compute_subnetwork.subnetwork, + module.subnets.subnets, google_compute_route.route, ] } diff --git a/modules/subnets/main.tf b/modules/subnets/main.tf new file mode 100644 index 00000000..1f175c09 --- /dev/null +++ b/modules/subnets/main.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + subnets = { + for x in var.subnets : + "${x.subnet_region}/${x.subnet_name}" => x + } +} + + +/****************************************** + Subnet configuration + *****************************************/ +resource "google_compute_subnetwork" "subnetwork" { + for_each = local.subnets + name = each.value.subnet_name + ip_cidr_range = each.value.subnet_ip + region = each.value.subnet_region + private_ip_google_access = lookup(each.value, "subnet_private_access", "false") + dynamic "log_config" { + for_each = lookup(each.value, "subnet_flow_logs", false) ? [{ + aggregation_interval = lookup(each.value, "subnet_flow_logs_interval", null) + flow_sampling = lookup(each.value, "subnet_flow_logs_sampling", null) + metadata = lookup(each.value, "subnet_flow_logs_metadata", null) + }] : [] + content { + aggregation_interval = log_config.value.aggregation_interval + flow_sampling = log_config.value.flow_sampling + metadata = log_config.value.metadata + } + } + network = var.network_name + project = var.project_id + description = lookup(each.value, "description", null) + secondary_ip_range = [ + for i in range( + length( + contains( + keys(var.secondary_ranges), each.value.subnet_name) == true + ? var.secondary_ranges[each.value.subnet_name] + : [] + )) : + var.secondary_ranges[each.value.subnet_name][i] + ] +} diff --git a/modules/subnets/outputs.tf b/modules/subnets/outputs.tf new file mode 100644 index 00000000..950d814f --- /dev/null +++ b/modules/subnets/outputs.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "subnets" { + value = google_compute_subnetwork.subnetwork + description = "The subnet resources" +} + +output "subnets_names" { + value = [for network in google_compute_subnetwork.subnetwork : network.name] + description = "The names of the subnets being created" +} + +output "subnets_ips" { + value = [for network in google_compute_subnetwork.subnetwork : network.ip_cidr_range] + description = "The IPs and CIDRs of the subnets being created" +} + +output "subnets_self_links" { + value = [for network in google_compute_subnetwork.subnetwork : network.self_link] + description = "The self-links of subnets being created" +} + +output "subnets_regions" { + value = [for network in google_compute_subnetwork.subnetwork : network.region] + description = "The region where the subnets will be created" +} + +output "subnets_private_access" { + value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] + description = "Whether the subnets will have access to Google API's without a public IP" +} + +output "subnets_flow_logs" { + value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] + description = "Whether the subnets will have VPC flow logs enabled" +} + +output "subnets_secondary_ranges" { + value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] + description = "The secondary ranges associated with these subnets" +} diff --git a/modules/subnets/variables.tf b/modules/subnets/variables.tf new file mode 100644 index 00000000..29f7b0d2 --- /dev/null +++ b/modules/subnets/variables.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project where this VPC will be created" +} + +variable "network_name" { + description = "The name of the network being created" +} + +variable "subnets" { + type = list(map(string)) + description = "The list of subnets being created" +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = "Secondary ranges that will be used in some of the subnets" + default = {} +} + +variable "description" { + type = string + description = "An optional description of this resource. The resource must be recreated to modify this field." + default = "" +} diff --git a/outputs.tf b/outputs.tf index a5fb462c..c870df76 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,37 +30,37 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = [for network in google_compute_subnetwork.subnetwork : network.name] + value = module.subnets.subnets_names description = "The names of the subnets being created" } output "subnets_ips" { - value = [for network in google_compute_subnetwork.subnetwork : network.ip_cidr_range] + value = module.subnets.subnets_ips description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = [for network in google_compute_subnetwork.subnetwork : network.self_link] + value = module.subnets.subnets_self_links description = "The self-links of subnets being created" } output "subnets_regions" { - value = [for network in google_compute_subnetwork.subnetwork : network.region] + value = module.subnets.subnets_regions description = "The region where the subnets will be created" } output "subnets_private_access" { - value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] + value = module.subnets.subnets_private_access description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] + value = module.subnets.subnets_flow_logs description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] + value = module.subnets.subnets_secondary_ranges description = "The secondary ranges associated with these subnets" } From 28a26a29cddfbf629527e1dd0f878ba4b668575b Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 11:56:00 -0800 Subject: [PATCH 39/78] move routes to its own submodule --- main.tf | 45 +++------------ modules/routes/main.tf | 57 +++++++++++++++++++ modules/routes/outputs.tf | 20 +++++++ .../scripts}/delete-default-gateway-routes.sh | 0 modules/routes/variables.tf | 44 ++++++++++++++ outputs.tf | 2 +- 6 files changed, 130 insertions(+), 38 deletions(-) create mode 100644 modules/routes/main.tf create mode 100644 modules/routes/outputs.tf rename {scripts => modules/routes/scripts}/delete-default-gateway-routes.sh (100%) create mode 100644 modules/routes/variables.tf diff --git a/main.tf b/main.tf index c044018b..cece7f54 100644 --- a/main.tf +++ b/main.tf @@ -42,41 +42,12 @@ module "subnets" { /****************************************** Routes *****************************************/ -resource "google_compute_route" "route" { - count = length(var.routes) - project = var.project_id - network = var.network_name - name = lookup(var.routes[count.index], "name", format("%s-%s-%d", lower(var.network_name), "route", count.index)) - description = lookup(var.routes[count.index], "description", "") - tags = compact(split(",", lookup(var.routes[count.index], "tags", ""))) - dest_range = lookup(var.routes[count.index], "destination_range", "") - next_hop_gateway = lookup(var.routes[count.index], "next_hop_internet", "false") == "true" ? "default-internet-gateway" : "" - next_hop_ip = lookup(var.routes[count.index], "next_hop_ip", "") - next_hop_instance = lookup(var.routes[count.index], "next_hop_instance", "") - next_hop_instance_zone = lookup(var.routes[count.index], "next_hop_instance_zone", "") - next_hop_vpn_tunnel = lookup(var.routes[count.index], "next_hop_vpn_tunnel", "") - priority = lookup(var.routes[count.index], "priority", "1000") - - depends_on = [ - module.vpc.network, - module.subnets.subnets, - ] -} - -resource "null_resource" "delete_default_internet_gateway_routes" { - count = var.delete_default_internet_gateway_routes ? 1 : 0 - - provisioner "local-exec" { - command = "${path.module}/scripts/delete-default-gateway-routes.sh ${var.project_id} ${var.network_name}" - } - - triggers = { - number_of_routes = length(var.routes) - } - - depends_on = [ - module.vpc.network, - module.subnets.subnets, - google_compute_route.route, - ] +module "routes" { + source = "./modules/routes" + project_id = var.project_id + network_name = module.vpc.network_name + routes = var.routes + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + network = module.vpc.network + subnets = module.subnets.subnets } diff --git a/modules/routes/main.tf b/modules/routes/main.tf new file mode 100644 index 00000000..987ce5c1 --- /dev/null +++ b/modules/routes/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + Routes + *****************************************/ +resource "google_compute_route" "route" { + count = length(var.routes) + project = var.project_id + network = var.network_name + name = lookup(var.routes[count.index], "name", format("%s-%s-%d", lower(var.network_name), "route", count.index)) + description = lookup(var.routes[count.index], "description", "") + tags = compact(split(",", lookup(var.routes[count.index], "tags", ""))) + dest_range = lookup(var.routes[count.index], "destination_range", "") + next_hop_gateway = lookup(var.routes[count.index], "next_hop_internet", "false") == "true" ? "default-internet-gateway" : "" + next_hop_ip = lookup(var.routes[count.index], "next_hop_ip", "") + next_hop_instance = lookup(var.routes[count.index], "next_hop_instance", "") + next_hop_instance_zone = lookup(var.routes[count.index], "next_hop_instance_zone", "") + next_hop_vpn_tunnel = lookup(var.routes[count.index], "next_hop_vpn_tunnel", "") + priority = lookup(var.routes[count.index], "priority", "1000") + + depends_on = [ + var.network, + var.subnets, + ] +} + +resource "null_resource" "delete_default_internet_gateway_routes" { + count = var.delete_default_internet_gateway_routes ? 1 : 0 + + provisioner "local-exec" { + command = "${path.module}/scripts/delete-default-gateway-routes.sh ${var.project_id} ${var.network_name}" + } + + triggers = { + number_of_routes = length(var.routes) + } + + depends_on = [ + var.network, + var.subnets, + google_compute_route.route, + ] +} diff --git a/modules/routes/outputs.tf b/modules/routes/outputs.tf new file mode 100644 index 00000000..83797436 --- /dev/null +++ b/modules/routes/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "routes" { + value = google_compute_route.route.*.name + description = "The routes associated with this VPC" +} diff --git a/scripts/delete-default-gateway-routes.sh b/modules/routes/scripts/delete-default-gateway-routes.sh similarity index 100% rename from scripts/delete-default-gateway-routes.sh rename to modules/routes/scripts/delete-default-gateway-routes.sh diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf new file mode 100644 index 00000000..1a2c494b --- /dev/null +++ b/modules/routes/variables.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project where this VPC will be created" +} + +variable "network_name" { + description = "The name of the network being created" +} + +variable "network" { + description = "The network resource that we depend on being created first" + default = null +} + +variable "subnets" { + description = "The subnet resources that we depend on being created first" + default = [] +} + +variable "routes" { + type = list(map(string)) + description = "List of routes being created in this VPC" + default = [] +} + +variable "delete_default_internet_gateway_routes" { + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = "false" +} diff --git a/outputs.tf b/outputs.tf index c870df76..fce8efaa 100644 --- a/outputs.tf +++ b/outputs.tf @@ -65,6 +65,6 @@ output "subnets_secondary_ranges" { } output "routes" { - value = google_compute_route.route.*.name + value = module.routes.routes description = "The routes associated with this VPC" } From 211a5164178bdf61e3972cdfa4a004e9d11dc6c2 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 13:24:06 -0800 Subject: [PATCH 40/78] use for_each for route creation --- examples/multi_vpc/main.tf | 23 ++++++++++------------- modules/routes/main.tf | 24 +++++++++++++----------- modules/routes/outputs.tf | 2 +- modules/routes/variables.tf | 6 +++--- variables.tf | 6 +++--- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/examples/multi_vpc/main.tf b/examples/multi_vpc/main.tf index 1d0b7247..500eda11 100644 --- a/examples/multi_vpc/main.tf +++ b/examples/multi_vpc/main.tf @@ -29,32 +29,29 @@ locals { network_02_subnet_01 = "${var.network_02_name}-subnet-01" network_02_subnet_02 = "${var.network_02_name}-subnet-02" - network_01_routes = [ - { - name = "${var.network_01_name}-egress-inet" + network_01_routes = { + "${var.network_01_name}-egress-inet" = { description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - }, - ] + } + } - network_02_routes = [ - { - name = "${var.network_02_name}-egress-inet" + network_02_routes = { + "${var.network_02_name}-egress-inet" = { description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - }, - { - name = "${var.network_02_name}-testapp-proxy" + } + "${var.network_02_name}-testapp-proxy" = { description = "route through proxy to reach app" destination_range = "10.50.10.0/24" tags = "app-proxy" next_hop_ip = "10.10.40.10" - }, - ] + } + } } module "test-vpc-module-01" { diff --git a/modules/routes/main.tf b/modules/routes/main.tf index 987ce5c1..bae4bc07 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -18,19 +18,21 @@ Routes *****************************************/ resource "google_compute_route" "route" { - count = length(var.routes) + for_each = var.routes + project = var.project_id network = var.network_name - name = lookup(var.routes[count.index], "name", format("%s-%s-%d", lower(var.network_name), "route", count.index)) - description = lookup(var.routes[count.index], "description", "") - tags = compact(split(",", lookup(var.routes[count.index], "tags", ""))) - dest_range = lookup(var.routes[count.index], "destination_range", "") - next_hop_gateway = lookup(var.routes[count.index], "next_hop_internet", "false") == "true" ? "default-internet-gateway" : "" - next_hop_ip = lookup(var.routes[count.index], "next_hop_ip", "") - next_hop_instance = lookup(var.routes[count.index], "next_hop_instance", "") - next_hop_instance_zone = lookup(var.routes[count.index], "next_hop_instance_zone", "") - next_hop_vpn_tunnel = lookup(var.routes[count.index], "next_hop_vpn_tunnel", "") - priority = lookup(var.routes[count.index], "priority", "1000") + + name = each.key + description = lookup(each.value, "description", "") + tags = compact(split(",", lookup(each.value, "tags", ""))) + dest_range = lookup(each.value, "destination_range", "") + next_hop_gateway = lookup(each.value, "next_hop_internet", "false") == "true" ? "default-internet-gateway" : "" + next_hop_ip = lookup(each.value, "next_hop_ip", "") + next_hop_instance = lookup(each.value, "next_hop_instance", "") + next_hop_instance_zone = lookup(each.value, "next_hop_instance_zone", "") + next_hop_vpn_tunnel = lookup(each.value, "next_hop_vpn_tunnel", "") + priority = lookup(each.value, "priority", "1000") depends_on = [ var.network, diff --git a/modules/routes/outputs.tf b/modules/routes/outputs.tf index 83797436..e45fbdf8 100644 --- a/modules/routes/outputs.tf +++ b/modules/routes/outputs.tf @@ -15,6 +15,6 @@ */ output "routes" { - value = google_compute_route.route.*.name + value = [for route in google_compute_route.route : route.name] description = "The routes associated with this VPC" } diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf index 1a2c494b..020ceb9a 100644 --- a/modules/routes/variables.tf +++ b/modules/routes/variables.tf @@ -33,9 +33,9 @@ variable "subnets" { } variable "routes" { - type = list(map(string)) - description = "List of routes being created in this VPC" - default = [] + type = map(any) + description = "Map of routes being created in this VPC" + default = {} } variable "delete_default_internet_gateway_routes" { diff --git a/variables.tf b/variables.tf index 51d1d78b..a08c78fa 100644 --- a/variables.tf +++ b/variables.tf @@ -46,9 +46,9 @@ variable "secondary_ranges" { } variable "routes" { - type = list(map(string)) - description = "List of routes being created in this VPC" - default = [] + type = map(any) + description = "Map of routes being created in this VPC" + default = {} } variable "delete_default_internet_gateway_routes" { From 2dd1fadcafda4d207380abb42e8185ba1e18c2b3 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 13:24:46 -0800 Subject: [PATCH 41/78] update developer tools version to fix generate doc errors --- Makefile | 2 +- build/int.cloudbuild.yaml | 2 +- build/lint.cloudbuild.yaml | 2 +- test/setup/make_source.sh | 24 ------------------------ 4 files changed, 3 insertions(+), 27 deletions(-) delete mode 100755 test/setup/make_source.sh diff --git a/Makefile b/Makefile index 69ee80e1..fd4c9220 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.6.0 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 299ca3e8..b76f495d 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -38,4 +38,4 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index b2d4c569..3f3923fb 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -21,4 +21,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0' diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh deleted file mode 100755 index ffdc48e1..00000000 --- a/test/setup/make_source.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "#!/usr/bin/env bash" > ../source.sh - -project_id=$(terraform output project_id) -echo "export TF_VAR_project_id='$project_id'" >> ../source.sh - -sa_json=$(terraform output sa_key) -# shellcheck disable=SC2086 -echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh From 6e0faef656abd3ad87f743b9f8dbec7d1bafd971 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 13:26:37 -0800 Subject: [PATCH 42/78] generate docs --- README.md | 20 +++++++++----------- modules/fabric-net-firewall/README.md | 6 +++++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e6beea3f..8ee79c11 100644 --- a/README.md +++ b/README.md @@ -60,23 +60,21 @@ module "vpc" { subnet-02 = [] } - routes = [ - { - name = "egress-internet" + routes = { + "egress-internet" = { description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - }, - { - name = "app-proxy" + } + "app-proxy" = { description = "route through proxy to reach app" destination_range = "10.50.10.0/24" tags = "app-proxy" next_hop_instance = "app-proxy-instance" next_hop_instance_zone = "us-west1-a" - }, - ] + } + } } ``` @@ -97,7 +95,7 @@ Then perform the following commands on the root folder: | description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | | network\_name | The name of the network being created | string | n/a | yes | | project\_id | The ID of the project where this VPC will be created | string | n/a | yes | -| routes | List of routes being created in this VPC | list(map(string)) | `` | no | +| routes | Map of routes being created in this VPC | map(any) | `` | no | | routing\_mode | The network routing mode (default 'GLOBAL') | string | `"GLOBAL"` | no | | secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | | shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | string | `"false"` | no | @@ -133,11 +131,11 @@ The subnets list contains maps, where each object represents a subnet. Each map | subnet\_flow\_logs | Whether the subnet will record and send flow log data to logging | string | `"false"` | no | ### Route Inputs -The routes list contains maps, where each object represents a route. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each map has the following inputs (please see examples folder for additional references): +The routes map contains objects, where each object represents a route. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| name | The name of the route being created | string | - | no | +| key | The name of the route being created | string | - | yes | | description | The description of the route being created | string | - | no | | tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | | destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes diff --git a/modules/fabric-net-firewall/README.md b/modules/fabric-net-firewall/README.md index f34555ef..3f0228cd 100644 --- a/modules/fabric-net-firewall/README.md +++ b/modules/fabric-net-firewall/README.md @@ -74,7 +74,7 @@ module "net-firewall" { |------|-------------|:----:|:-----:|:-----:| | admin\_ranges | IP CIDR ranges that have complete access to all subnets. | list | `` | no | | admin\_ranges\_enabled | Enable admin ranges-based rules. | string | `"false"` | no | -| custom\_rules | List of custom rule definitions (refer to variables file for syntax). | map | `` | no | +| custom\_rules | List of custom rule definitions (refer to variables file for syntax). | object | `` | no | | http\_source\_ranges | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | list | `` | no | | https\_source\_ranges | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | list | `` | no | | internal\_allow | Allow rules for internal ranges. | list | `` | no | @@ -89,6 +89,10 @@ module "net-firewall" { | Name | Description | |------|-------------| | admin\_ranges | Admin ranges data. | +| custom\_egress\_allow\_rules | Custom egress rules with allow blocks. | +| custom\_egress\_deny\_rules | Custom egress rules with allow blocks. | +| custom\_ingress\_allow\_rules | Custom ingress rules with allow blocks. | +| custom\_ingress\_deny\_rules | Custom ingress rules with deny blocks. | | internal\_ranges | Internal ranges. | From 2c8ed098db288c2c3c7656b7aceaa4088f89a086 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 13:57:04 -0800 Subject: [PATCH 43/78] add readme to submodules --- README.md | 6 ++- main.tf | 2 +- modules/routes/README.md | 82 ++++++++++++++++++++++++++++++++ modules/subnets/README.md | 99 +++++++++++++++++++++++++++++++++++++++ modules/vpc/README.md | 50 ++++++++++++++++++++ 5 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 modules/routes/README.md create mode 100644 modules/subnets/README.md create mode 100644 modules/vpc/README.md diff --git a/README.md b/README.md index 8ee79c11..89ca4526 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ It supports creating: - Subnets within the VPC - Secondary ranges for the subnets (if applicable) +Sub modules are provided for creating individual vpc, subnets, and routes. See the modules directory for the various sub modules usage. + ## Compatibility This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). @@ -120,6 +122,7 @@ Then perform the following commands on the root folder: ### Subnet Inputs + The subnets list contains maps, where each object represents a subnet. Each map has the following inputs (please see examples folder for additional references): | Name | Description | Type | Default | Required | @@ -131,7 +134,8 @@ The subnets list contains maps, where each object represents a subnet. Each map | subnet\_flow\_logs | Whether the subnet will record and send flow log data to logging | string | `"false"` | no | ### Route Inputs -The routes map contains objects, where each object represents a route. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): + +The `routes` map `key` is the unique route `name` and the `value` object represents the route input options. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| diff --git a/main.tf b/main.tf index cece7f54..ca5c20b3 100644 --- a/main.tf +++ b/main.tf @@ -34,8 +34,8 @@ module "subnets" { source = "./modules/subnets" project_id = var.project_id network_name = module.vpc.network_name - subnets = var.subnets description = var.description + subnets = var.subnets secondary_ranges = var.secondary_ranges } diff --git a/modules/routes/README.md b/modules/routes/README.md new file mode 100644 index 00000000..b840c00b --- /dev/null +++ b/modules/routes/README.md @@ -0,0 +1,82 @@ +# Terraform Network Module + +This submodule is part of the the `terraform-google-network` module. It creates the individual vpc routes and optionally deletes the default internet gateway routes. + +It supports creating: + +- Routes within vpc network. +- Optionally deletes the default internet gateway routes. + +## Compatibility + +This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). + +## Usage + +Basic usage of this submodule is as follows: + +```hcl +module "vpc" { + source = "terraform-google-modules/network/google//modules/routes" + version = "~> 2.0.0" + + project_id = "" + network_name = "example-vpc" + + delete_default_internet_gateway_routes = false + + routes = { + "egress-internet" = { + description = "route through IGW to access internet" + destination_range = "0.0.0.0/0" + tags = "egress-inet" + next_hop_internet = "true" + } + "app-proxy" = { + description = "route through proxy to reach app" + destination_range = "10.50.10.0/24" + tags = "app-proxy" + next_hop_instance = "app-proxy-instance" + next_hop_instance_zone = "us-west1-a" + } + } +} +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| delete\_default\_internet\_gateway\_routes | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | string | `"false"` | no | +| network | The network resource that we depend on being created first | string | `"null"` | no | +| network\_name | The name of the network being created | string | n/a | yes | +| project\_id | The ID of the project where this VPC will be created | string | n/a | yes | +| routes | Map of routes being created in this VPC | map(any) | `` | no | +| subnets | The subnet resources that we depend on being created first | list | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| routes | The routes associated with this VPC | + + + + +### Routes Input + +The `routes` map `key` is the unique route `name` and the `value` object represents the route input options. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| key | The name of the route being created | string | - | yes | +| description | The description of the route being created | string | - | no | +| tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | +| destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes +| next\_hop\_internet | Whether the next hop to this route will the default internet gateway. Use "true" to enable this as next hop | string | `"false"` | yes | +| next\_hop\_ip | Network IP address of an instance that should handle matching packets | string | - | yes | +| next\_hop\_instance | URL or name of an instance that should handle matching packets. If just name is specified "next\_hop\_instance\_zone" is required | string | - | yes | +| next\_hop\_instance\_zone | The zone of the instance specified in next\_hop\_instance. Only required if next\_hop\_instance is specified as a name | string | - | no | +| next\_hop\_vpn\_tunnel | URL to a VpnTunnel that should handle matching packets | string | - | yes | +| priority | The priority of this route. Priority is used to break ties in cases where there is more than one matching route of equal prefix length. In the case of two routes with equal prefix length, the one with the lowest-numbered priority value wins | string | `"1000"` | yes | diff --git a/modules/subnets/README.md b/modules/subnets/README.md new file mode 100644 index 00000000..ac47948b --- /dev/null +++ b/modules/subnets/README.md @@ -0,0 +1,99 @@ +# Terraform Network Module + +This submodule is part of the the `terraform-google-network` module. It creates the individual vpc subnets. + +It supports creating: + +- Subnets within vpc network. + +## Compatibility + +This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). + +## Usage + +Basic usage of this submodule is as follows: + +```hcl +module "vpc" { + source = "terraform-google-modules/network/google//modules/routes" + version = "~> 2.0.0" + + project_id = "" + network_name = "example-vpc" + + subnets = [ + { + subnet_name = "subnet-01" + subnet_ip = "10.10.10.0/24" + subnet_region = "us-west1" + }, + { + subnet_name = "subnet-02" + subnet_ip = "10.10.20.0/24" + subnet_region = "us-west1" + subnet_private_access = "true" + subnet_flow_logs = "true" + description = "This subnet has a description" + }, + { + subnet_name = "subnet-03" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1" + subnet_flow_logs = "true" + subnet_flow_logs_interval = "INTERVAL_10_MIN" + subnet_flow_logs_sampling = 0.7 + subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + } + ] + + secondary_ranges = { + subnet-01 = [ + { + range_name = "subnet-01-secondary-01" + ip_cidr_range = "192.168.64.0/24" + }, + ] + + subnet-02 = [] + } +} +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | +| network\_name | The name of the network being created | string | n/a | yes | +| project\_id | The ID of the project where this VPC will be created | string | n/a | yes | +| secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | +| subnets | The list of subnets being created | list(map(string)) | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| subnets | The subnet resources | +| subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | +| subnets\_ips | The IPs and CIDRs of the subnets being created | +| subnets\_names | The names of the subnets being created | +| subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | +| subnets\_regions | The region where the subnets will be created | +| subnets\_secondary\_ranges | The secondary ranges associated with these subnets | +| subnets\_self\_links | The self-links of subnets being created | + + + +### Subnet Inputs + +The subnets list contains maps, where each object represents a subnet. Each map has the following inputs (please see examples folder for additional references): + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| subnet\_name | The name of the subnet being created | string | - | yes | +| subnet\_ip | The IP and CIDR range of the subnet being created | string | - | yes | +| subnet\_region | The region where the subnet will be created | string | - | yes | +| subnet\_private\_access | Whether this subnet will have private Google access enabled | string | `"false"` | no | +| subnet\_flow\_logs | Whether the subnet will record and send flow log data to logging | string | `"false"` | no | diff --git a/modules/vpc/README.md b/modules/vpc/README.md new file mode 100644 index 00000000..67ff2c8d --- /dev/null +++ b/modules/vpc/README.md @@ -0,0 +1,50 @@ +# Terraform Network Module + +This submodule is part of the the `terraform-google-network` module. It creates a vpc network and optionally enables it as a Shared VPC host project. + +It supports creating: + +- A VPC Network +- Optionally enabling the network as a Shared VPC host + +## Compatibility + +This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). + +## Usage + +Basic usage of this submodule is as follows: + +```hcl +module "vpc" { + source = "terraform-google-modules/network/google//modules/routes" + version = "~> 2.0.0" + + project_id = "" + network_name = "example-vpc" + + shared_vpc_host = false +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| auto\_create\_subnetworks | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | `"false"` | no | +| description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | +| network\_name | The name of the network being created | string | n/a | yes | +| project\_id | The ID of the project where this VPC will be created | string | n/a | yes | +| routing\_mode | The network routing mode (default 'GLOBAL') | string | `"GLOBAL"` | no | +| shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | string | `"false"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| network | The VPC resource being created | +| network\_name | The name of the VPC being created | +| network\_self\_link | The URI of the VPC being created | +| svpc\_host\_project\_id | Shared VPC host project id. | + + From f566110ae92da9805e14d7538db8514f3518bf57 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 14:08:43 -0800 Subject: [PATCH 44/78] formatting fix --- modules/routes/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/routes/main.tf b/modules/routes/main.tf index bae4bc07..9b00994e 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -20,8 +20,8 @@ resource "google_compute_route" "route" { for_each = var.routes - project = var.project_id - network = var.network_name + project = var.project_id + network = var.network_name name = each.key description = lookup(each.value, "description", "") From f3e5e993729a22c32e4ad9dd98296fbc9edbb9ee Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 14:09:57 -0800 Subject: [PATCH 45/78] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ce66c7..7e41d76a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning][semver-site]. ## [Unreleased] v2.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v2.0.md). +### Added + +- Split main module up into vpc, subnets, and routes submodules. Changed `routes` input to be comptiable with `for_each` `maps` [#103] + ### Fixed - Fixes subnet recreation when a subnet is updated. [#73] From 605269af7a2c86fb6a6a4abe82ca60e01847046f Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 14:25:10 -0800 Subject: [PATCH 46/78] lock provider for google_project_services removal error --- test/setup/versions.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/setup/versions.tf b/test/setup/versions.tf index 832ec1df..b8dffa42 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -17,3 +17,11 @@ terraform { required_version = ">= 0.12" } + +provider "google" { + version = "~> 2.18.0" +} + +provider "google-beta" { + version = "~> 2.18.0" +} From 3a3a408e39ce648e629c0dd63081a17f0f9c8c87 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 15:24:08 -0800 Subject: [PATCH 47/78] specify provider versions for each module for terraform init/validate --- modules/fabric-net-firewall/versions.tf | 4 ++++ modules/fabric-net-svpc-access/versions.tf | 4 ++++ modules/routes/versions.tf | 23 ++++++++++++++++++++++ modules/subnets/versions.tf | 23 ++++++++++++++++++++++ modules/vpc/versions.tf | 23 ++++++++++++++++++++++ test/setup/versions.tf | 4 ++-- versions.tf | 4 ++++ 7 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 modules/routes/versions.tf create mode 100644 modules/subnets/versions.tf create mode 100644 modules/vpc/versions.tf diff --git a/modules/fabric-net-firewall/versions.tf b/modules/fabric-net-firewall/versions.tf index 1fe4caaa..44e659f8 100644 --- a/modules/fabric-net-firewall/versions.tf +++ b/modules/fabric-net-firewall/versions.tf @@ -17,3 +17,7 @@ terraform { required_version = "~> 0.12.0" } + +provider "google" { + version = "~> 2.18.0" +} diff --git a/modules/fabric-net-svpc-access/versions.tf b/modules/fabric-net-svpc-access/versions.tf index 1fe4caaa..44e659f8 100644 --- a/modules/fabric-net-svpc-access/versions.tf +++ b/modules/fabric-net-svpc-access/versions.tf @@ -17,3 +17,7 @@ terraform { required_version = "~> 0.12.0" } + +provider "google" { + version = "~> 2.18.0" +} diff --git a/modules/routes/versions.tf b/modules/routes/versions.tf new file mode 100644 index 00000000..44e659f8 --- /dev/null +++ b/modules/routes/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" +} + +provider "google" { + version = "~> 2.18.0" +} diff --git a/modules/subnets/versions.tf b/modules/subnets/versions.tf new file mode 100644 index 00000000..44e659f8 --- /dev/null +++ b/modules/subnets/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" +} + +provider "google" { + version = "~> 2.18.0" +} diff --git a/modules/vpc/versions.tf b/modules/vpc/versions.tf new file mode 100644 index 00000000..44e659f8 --- /dev/null +++ b/modules/vpc/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" +} + +provider "google" { + version = "~> 2.18.0" +} diff --git a/test/setup/versions.tf b/test/setup/versions.tf index b8dffa42..03f562b4 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ terraform { - required_version = ">= 0.12" + required_version = "~> 0.12.0" } provider "google" { diff --git a/versions.tf b/versions.tf index 1fe4caaa..44e659f8 100644 --- a/versions.tf +++ b/versions.tf @@ -17,3 +17,7 @@ terraform { required_version = "~> 0.12.0" } + +provider "google" { + version = "~> 2.18.0" +} From fec8285517c21c5f84662b30dc9df834c70bc3ab Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 15:32:23 -0800 Subject: [PATCH 48/78] Revert "specify provider versions for each module for terraform init/validate" This reverts commit 3a3a408e39ce648e629c0dd63081a17f0f9c8c87. --- modules/fabric-net-firewall/versions.tf | 4 ---- modules/fabric-net-svpc-access/versions.tf | 4 ---- modules/routes/versions.tf | 23 ---------------------- modules/subnets/versions.tf | 23 ---------------------- modules/vpc/versions.tf | 23 ---------------------- test/setup/versions.tf | 4 ++-- versions.tf | 4 ---- 7 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 modules/routes/versions.tf delete mode 100644 modules/subnets/versions.tf delete mode 100644 modules/vpc/versions.tf diff --git a/modules/fabric-net-firewall/versions.tf b/modules/fabric-net-firewall/versions.tf index 44e659f8..1fe4caaa 100644 --- a/modules/fabric-net-firewall/versions.tf +++ b/modules/fabric-net-firewall/versions.tf @@ -17,7 +17,3 @@ terraform { required_version = "~> 0.12.0" } - -provider "google" { - version = "~> 2.18.0" -} diff --git a/modules/fabric-net-svpc-access/versions.tf b/modules/fabric-net-svpc-access/versions.tf index 44e659f8..1fe4caaa 100644 --- a/modules/fabric-net-svpc-access/versions.tf +++ b/modules/fabric-net-svpc-access/versions.tf @@ -17,7 +17,3 @@ terraform { required_version = "~> 0.12.0" } - -provider "google" { - version = "~> 2.18.0" -} diff --git a/modules/routes/versions.tf b/modules/routes/versions.tf deleted file mode 100644 index 44e659f8..00000000 --- a/modules/routes/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 0.12.0" -} - -provider "google" { - version = "~> 2.18.0" -} diff --git a/modules/subnets/versions.tf b/modules/subnets/versions.tf deleted file mode 100644 index 44e659f8..00000000 --- a/modules/subnets/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 0.12.0" -} - -provider "google" { - version = "~> 2.18.0" -} diff --git a/modules/vpc/versions.tf b/modules/vpc/versions.tf deleted file mode 100644 index 44e659f8..00000000 --- a/modules/vpc/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 0.12.0" -} - -provider "google" { - version = "~> 2.18.0" -} diff --git a/test/setup/versions.tf b/test/setup/versions.tf index 03f562b4..b8dffa42 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2018 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ terraform { - required_version = "~> 0.12.0" + required_version = ">= 0.12" } provider "google" { diff --git a/versions.tf b/versions.tf index 44e659f8..1fe4caaa 100644 --- a/versions.tf +++ b/versions.tf @@ -17,7 +17,3 @@ terraform { required_version = "~> 0.12.0" } - -provider "google" { - version = "~> 2.18.0" -} From 560bbb2a5795220c6236f697b19e94591a4fd04e Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 15:36:21 -0800 Subject: [PATCH 49/78] required provider versions on modules --- modules/fabric-net-firewall/versions.tf | 3 +++ modules/fabric-net-svpc-access/versions.tf | 3 +++ modules/routes/versions.tf | 22 ++++++++++++++++++++++ modules/subnets/versions.tf | 22 ++++++++++++++++++++++ modules/vpc/versions.tf | 22 ++++++++++++++++++++++ versions.tf | 3 +++ 6 files changed, 75 insertions(+) create mode 100644 modules/routes/versions.tf create mode 100644 modules/subnets/versions.tf create mode 100644 modules/vpc/versions.tf diff --git a/modules/fabric-net-firewall/versions.tf b/modules/fabric-net-firewall/versions.tf index 1fe4caaa..5b19897c 100644 --- a/modules/fabric-net-firewall/versions.tf +++ b/modules/fabric-net-firewall/versions.tf @@ -16,4 +16,7 @@ terraform { required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } } diff --git a/modules/fabric-net-svpc-access/versions.tf b/modules/fabric-net-svpc-access/versions.tf index 1fe4caaa..5b19897c 100644 --- a/modules/fabric-net-svpc-access/versions.tf +++ b/modules/fabric-net-svpc-access/versions.tf @@ -16,4 +16,7 @@ terraform { required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } } diff --git a/modules/routes/versions.tf b/modules/routes/versions.tf new file mode 100644 index 00000000..5b19897c --- /dev/null +++ b/modules/routes/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } +} diff --git a/modules/subnets/versions.tf b/modules/subnets/versions.tf new file mode 100644 index 00000000..5b19897c --- /dev/null +++ b/modules/subnets/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } +} diff --git a/modules/vpc/versions.tf b/modules/vpc/versions.tf new file mode 100644 index 00000000..5b19897c --- /dev/null +++ b/modules/vpc/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } +} diff --git a/versions.tf b/versions.tf index 1fe4caaa..5b19897c 100644 --- a/versions.tf +++ b/versions.tf @@ -16,4 +16,7 @@ terraform { required_version = "~> 0.12.0" + required_providers { + google = "~> 2.19.0" + } } From fc02b376cd1b59abfaef1834bc15864dbd4b5a26 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 15:37:24 -0800 Subject: [PATCH 50/78] Update CHANGELOG.md Co-Authored-By: Morgante Pell --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e41d76a..3ab85452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ v2.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./d ### Added -- Split main module up into vpc, subnets, and routes submodules. Changed `routes` input to be comptiable with `for_each` `maps` [#103] +- Split main module up into vpc, subnets, and routes submodules. Changed `routes` input to be compatible with `for_each` `maps` [#103] ### Fixed From 188bf1cda3fb240e0f00dadd734c72f9de89e700 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:08:45 -0800 Subject: [PATCH 51/78] remove compatability from submodules --- modules/routes/README.md | 4 ---- modules/subnets/README.md | 4 ---- modules/vpc/README.md | 4 ---- 3 files changed, 12 deletions(-) diff --git a/modules/routes/README.md b/modules/routes/README.md index b840c00b..a38871d6 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -7,10 +7,6 @@ It supports creating: - Routes within vpc network. - Optionally deletes the default internet gateway routes. -## Compatibility - -This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). - ## Usage Basic usage of this submodule is as follows: diff --git a/modules/subnets/README.md b/modules/subnets/README.md index ac47948b..8563f796 100644 --- a/modules/subnets/README.md +++ b/modules/subnets/README.md @@ -6,10 +6,6 @@ It supports creating: - Subnets within vpc network. -## Compatibility - -This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). - ## Usage Basic usage of this submodule is as follows: diff --git a/modules/vpc/README.md b/modules/vpc/README.md index 67ff2c8d..ded03e33 100644 --- a/modules/vpc/README.md +++ b/modules/vpc/README.md @@ -7,10 +7,6 @@ It supports creating: - A VPC Network - Optionally enabling the network as a Shared VPC host -## Compatibility - -This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) and need a Terraform 0.11.x-compatible version of this module, the last released version intended for Terraform 0.11.x is [0.8.0](https://registry.terraform.io/modules/terraform-google-modules/network/google/0.8.0). - ## Usage Basic usage of this submodule is as follows: From 28319411b32f25a1b5211b466f9f1d5f1c9f2359 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:09:05 -0800 Subject: [PATCH 52/78] update variable descriptions --- modules/routes/README.md | 4 ++-- modules/routes/variables.tf | 4 ++-- modules/subnets/README.md | 4 ++-- modules/subnets/variables.tf | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/routes/README.md b/modules/routes/README.md index a38871d6..0479e41e 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -46,9 +46,9 @@ module "vpc" { |------|-------------|:----:|:-----:|:-----:| | delete\_default\_internet\_gateway\_routes | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | string | `"false"` | no | | network | The network resource that we depend on being created first | string | `"null"` | no | -| network\_name | The name of the network being created | string | n/a | yes | -| project\_id | The ID of the project where this VPC will be created | string | n/a | yes | | routes | Map of routes being created in this VPC | map(any) | `` | no | +| network\_name | The name of the network where routes will be created | string | n/a | yes | +| project\_id | The ID of the project where the routes will be created | string | n/a | yes | | subnets | The subnet resources that we depend on being created first | list | `` | no | ## Outputs diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf index 020ceb9a..06530a25 100644 --- a/modules/routes/variables.tf +++ b/modules/routes/variables.tf @@ -15,11 +15,11 @@ */ variable "project_id" { - description = "The ID of the project where this VPC will be created" + description = "The ID of the project where the routes will be created" } variable "network_name" { - description = "The name of the network being created" + description = "The name of the network where routes will be created" } variable "network" { diff --git a/modules/subnets/README.md b/modules/subnets/README.md index 8563f796..6203ae25 100644 --- a/modules/subnets/README.md +++ b/modules/subnets/README.md @@ -62,8 +62,8 @@ module "vpc" { | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | -| network\_name | The name of the network being created | string | n/a | yes | -| project\_id | The ID of the project where this VPC will be created | string | n/a | yes | +| network\_name | The name of the network where subnets will be created | string | n/a | yes | +| project\_id | The ID of the project where subnets will be created | string | n/a | yes | | secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | | subnets | The list of subnets being created | list(map(string)) | n/a | yes | diff --git a/modules/subnets/variables.tf b/modules/subnets/variables.tf index 29f7b0d2..c2031d50 100644 --- a/modules/subnets/variables.tf +++ b/modules/subnets/variables.tf @@ -15,11 +15,11 @@ */ variable "project_id" { - description = "The ID of the project where this VPC will be created" + description = "The ID of the project where subnets will be created" } variable "network_name" { - description = "The name of the network being created" + description = "The name of the network where subnets will be created" } variable "subnets" { From 8649e43e6aa9db45bed5688cecc4842b5702e594 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:10:29 -0800 Subject: [PATCH 53/78] switch routes back to list type with map generated within module --- README.md | 16 +++++++++------- examples/multi_vpc/main.tf | 23 +++++++++++++---------- modules/routes/README.md | 16 +++++++++------- modules/routes/main.tf | 9 ++++++++- modules/routes/variables.tf | 6 +++--- variables.tf | 6 +++--- 6 files changed, 45 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 89ca4526..565ee995 100644 --- a/README.md +++ b/README.md @@ -62,21 +62,23 @@ module "vpc" { subnet-02 = [] } - routes = { - "egress-internet" = { + routes = [ + { + name = "egress-internet" description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - } - "app-proxy" = { + }, + { + name = "app-proxy" description = "route through proxy to reach app" destination_range = "10.50.10.0/24" tags = "app-proxy" next_hop_instance = "app-proxy-instance" next_hop_instance_zone = "us-west1-a" - } - } + }, + ] } ``` @@ -97,7 +99,7 @@ Then perform the following commands on the root folder: | description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | | network\_name | The name of the network being created | string | n/a | yes | | project\_id | The ID of the project where this VPC will be created | string | n/a | yes | -| routes | Map of routes being created in this VPC | map(any) | `` | no | +| routes | List of routes being created in this VPC | list(map(string)) | `` | no | | routing\_mode | The network routing mode (default 'GLOBAL') | string | `"GLOBAL"` | no | | secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | | shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | string | `"false"` | no | diff --git a/examples/multi_vpc/main.tf b/examples/multi_vpc/main.tf index 500eda11..1d0b7247 100644 --- a/examples/multi_vpc/main.tf +++ b/examples/multi_vpc/main.tf @@ -29,29 +29,32 @@ locals { network_02_subnet_01 = "${var.network_02_name}-subnet-01" network_02_subnet_02 = "${var.network_02_name}-subnet-02" - network_01_routes = { - "${var.network_01_name}-egress-inet" = { + network_01_routes = [ + { + name = "${var.network_01_name}-egress-inet" description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - } - } + }, + ] - network_02_routes = { - "${var.network_02_name}-egress-inet" = { + network_02_routes = [ + { + name = "${var.network_02_name}-egress-inet" description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - } - "${var.network_02_name}-testapp-proxy" = { + }, + { + name = "${var.network_02_name}-testapp-proxy" description = "route through proxy to reach app" destination_range = "10.50.10.0/24" tags = "app-proxy" next_hop_ip = "10.10.40.10" - } - } + }, + ] } module "test-vpc-module-01" { diff --git a/modules/routes/README.md b/modules/routes/README.md index 0479e41e..26c41506 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -21,21 +21,23 @@ module "vpc" { delete_default_internet_gateway_routes = false - routes = { - "egress-internet" = { + routes = [ + { + name = "egress-internet" description = "route through IGW to access internet" destination_range = "0.0.0.0/0" tags = "egress-inet" next_hop_internet = "true" - } - "app-proxy" = { + }, + { + name = "app-proxy" description = "route through proxy to reach app" destination_range = "10.50.10.0/24" tags = "app-proxy" next_hop_instance = "app-proxy-instance" next_hop_instance_zone = "us-west1-a" - } - } + }, + ] } ``` @@ -46,9 +48,9 @@ module "vpc" { |------|-------------|:----:|:-----:|:-----:| | delete\_default\_internet\_gateway\_routes | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | string | `"false"` | no | | network | The network resource that we depend on being created first | string | `"null"` | no | -| routes | Map of routes being created in this VPC | map(any) | `` | no | | network\_name | The name of the network where routes will be created | string | n/a | yes | | project\_id | The ID of the project where the routes will be created | string | n/a | yes | +| routes | List of routes being created in this VPC | list(map(string)) | `` | no | | subnets | The subnet resources that we depend on being created first | list | `` | no | ## Outputs diff --git a/modules/routes/main.tf b/modules/routes/main.tf index 9b00994e..cc263747 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -14,11 +14,18 @@ * limitations under the License. */ +locals { + routes = { + for i, route in var.routes : + lookup(route, "name", format("%s-%s-%d", lower(var.network_name), "route", i)) => route + } +} + /****************************************** Routes *****************************************/ resource "google_compute_route" "route" { - for_each = var.routes + for_each = local.routes project = var.project_id network = var.network_name diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf index 06530a25..ba68bf82 100644 --- a/modules/routes/variables.tf +++ b/modules/routes/variables.tf @@ -33,9 +33,9 @@ variable "subnets" { } variable "routes" { - type = map(any) - description = "Map of routes being created in this VPC" - default = {} + type = list(map(string)) + description = "List of routes being created in this VPC" + default = [] } variable "delete_default_internet_gateway_routes" { diff --git a/variables.tf b/variables.tf index a08c78fa..51d1d78b 100644 --- a/variables.tf +++ b/variables.tf @@ -46,9 +46,9 @@ variable "secondary_ranges" { } variable "routes" { - type = map(any) - description = "Map of routes being created in this VPC" - default = {} + type = list(map(string)) + description = "List of routes being created in this VPC" + default = [] } variable "delete_default_internet_gateway_routes" { From d567d892fd42007b0a267ac6668ef3630b9a8320 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:18:07 -0800 Subject: [PATCH 54/78] default rotue values to null so google_compute_route defaults are used --- modules/routes/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/routes/main.tf b/modules/routes/main.tf index cc263747..ba3802d2 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -31,15 +31,15 @@ resource "google_compute_route" "route" { network = var.network_name name = each.key - description = lookup(each.value, "description", "") + description = lookup(each.value, "description", null) tags = compact(split(",", lookup(each.value, "tags", ""))) - dest_range = lookup(each.value, "destination_range", "") + dest_range = lookup(each.value, "destination_range", null) next_hop_gateway = lookup(each.value, "next_hop_internet", "false") == "true" ? "default-internet-gateway" : "" - next_hop_ip = lookup(each.value, "next_hop_ip", "") - next_hop_instance = lookup(each.value, "next_hop_instance", "") - next_hop_instance_zone = lookup(each.value, "next_hop_instance_zone", "") - next_hop_vpn_tunnel = lookup(each.value, "next_hop_vpn_tunnel", "") - priority = lookup(each.value, "priority", "1000") + next_hop_ip = lookup(each.value, "next_hop_ip", null) + next_hop_instance = lookup(each.value, "next_hop_instance", null) + next_hop_instance_zone = lookup(each.value, "next_hop_instance_zone", null) + next_hop_vpn_tunnel = lookup(each.value, "next_hop_vpn_tunnel", null) + priority = lookup(each.value, "priority",null) depends_on = [ var.network, From b5b9ad81b93fce3ea6c616791b72c183db1fee2c Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:31:16 -0800 Subject: [PATCH 55/78] update output names per pr --- README.md | 2 +- .../delete_default_gateway_routes/README.md | 2 +- .../delete_default_gateway_routes/outputs.tf | 4 +- examples/multi_vpc/outputs.tf | 4 +- examples/secondary_ranges/README.md | 2 +- examples/secondary_ranges/outputs.tf | 4 +- examples/simple_project/README.md | 2 +- examples/simple_project/outputs.tf | 4 +- .../README.md | 2 +- .../outputs.tf | 4 +- examples/submodule_firewall/README.md | 2 +- examples/submodule_firewall/outputs.tf | 4 +- modules/routes/README.md | 2 +- modules/routes/outputs.tf | 4 +- modules/subnets/README.md | 9 +---- modules/subnets/outputs.tf | 37 +------------------ outputs.tf | 20 +++++----- 17 files changed, 33 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 565ee995..87a5161e 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Then perform the following commands on the root folder: |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The route names associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IPs and CIDRs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/delete_default_gateway_routes/README.md b/examples/delete_default_gateway_routes/README.md index 42052f0d..2735dfb5 100644 --- a/examples/delete_default_gateway_routes/README.md +++ b/examples/delete_default_gateway_routes/README.md @@ -18,7 +18,7 @@ This VPC has a single subnet with no secondary ranges, and ensures the default i |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/delete_default_gateway_routes/outputs.tf b/examples/delete_default_gateway_routes/outputs.tf index 429e0b97..d7a27ff4 100644 --- a/examples/delete_default_gateway_routes/outputs.tf +++ b/examples/delete_default_gateway_routes/outputs.tf @@ -54,7 +54,7 @@ output "subnets_secondary_ranges" { description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.test-vpc-module.routes +output "route_names" { + value = module.test-vpc-module.route_names description = "The routes associated with this VPC" } diff --git a/examples/multi_vpc/outputs.tf b/examples/multi_vpc/outputs.tf index d389a4b6..c2d6a828 100644 --- a/examples/multi_vpc/outputs.tf +++ b/examples/multi_vpc/outputs.tf @@ -56,7 +56,7 @@ output "network_01_subnets_secondary_ranges" { } output "network_01_routes" { - value = module.test-vpc-module-01.routes + value = module.test-vpc-module-01.route_names description = "The routes associated with network-01" } @@ -102,6 +102,6 @@ output "network_02_subnets_secondary_ranges" { } output "network_02_routes" { - value = module.test-vpc-module-02.routes + value = module.test-vpc-module-02.route_names description = "The routes associated with network-02" } diff --git a/examples/secondary_ranges/README.md b/examples/secondary_ranges/README.md index 3ffc4ec3..f44a18e6 100644 --- a/examples/secondary_ranges/README.md +++ b/examples/secondary_ranges/README.md @@ -19,7 +19,7 @@ ranges and the third being given a single secondary range. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/secondary_ranges/outputs.tf b/examples/secondary_ranges/outputs.tf index 95d5b63e..7bab5b12 100644 --- a/examples/secondary_ranges/outputs.tf +++ b/examples/secondary_ranges/outputs.tf @@ -59,7 +59,7 @@ output "subnets_secondary_ranges" { description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.vpc-secondary-ranges.routes +output "route_names" { + value = module.vpc-secondary-ranges.route_names description = "The routes associated with this VPC" } diff --git a/examples/simple_project/README.md b/examples/simple_project/README.md index 3f6cb802..a5fd3d97 100644 --- a/examples/simple_project/README.md +++ b/examples/simple_project/README.md @@ -18,7 +18,7 @@ This VPC has two subnets, with no secondary ranges. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/simple_project/outputs.tf b/examples/simple_project/outputs.tf index b5f631cc..9492e630 100644 --- a/examples/simple_project/outputs.tf +++ b/examples/simple_project/outputs.tf @@ -59,7 +59,7 @@ output "subnets_secondary_ranges" { description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.test-vpc-module.routes +output "route_names" { + value = module.test-vpc-module.route_names description = "The routes associated with this VPC" } diff --git a/examples/simple_project_with_regional_network/README.md b/examples/simple_project_with_regional_network/README.md index 445f3111..acd2b147 100644 --- a/examples/simple_project_with_regional_network/README.md +++ b/examples/simple_project_with_regional_network/README.md @@ -18,7 +18,7 @@ This VPC has two subnets, with no secondary ranges. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/simple_project_with_regional_network/outputs.tf b/examples/simple_project_with_regional_network/outputs.tf index b5f631cc..9492e630 100644 --- a/examples/simple_project_with_regional_network/outputs.tf +++ b/examples/simple_project_with_regional_network/outputs.tf @@ -59,7 +59,7 @@ output "subnets_secondary_ranges" { description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.test-vpc-module.routes +output "route_names" { + value = module.test-vpc-module.route_names description = "The routes associated with this VPC" } diff --git a/examples/submodule_firewall/README.md b/examples/submodule_firewall/README.md index faf52212..229a61da 100644 --- a/examples/submodule_firewall/README.md +++ b/examples/submodule_firewall/README.md @@ -20,7 +20,7 @@ This VPC has two subnets, with no secondary ranges. | internal\_ranges | Firewall attributes for internal ranges. | | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| routes | The routes associated with this VPC | +| route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | | subnets\_names | The names of the subnets being created | diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index 0586334f..662d9cac 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -69,7 +69,7 @@ output "subnets_secondary_ranges" { description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.test-vpc-module.routes +output "route_names" { + value = module.test-vpc-module.route_names description = "The routes associated with this VPC" } diff --git a/modules/routes/README.md b/modules/routes/README.md index 26c41506..2b0dc390 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -57,7 +57,7 @@ module "vpc" { | Name | Description | |------|-------------| -| routes | The routes associated with this VPC | +| routes | The created routes resources | diff --git a/modules/routes/outputs.tf b/modules/routes/outputs.tf index e45fbdf8..0f672ec6 100644 --- a/modules/routes/outputs.tf +++ b/modules/routes/outputs.tf @@ -15,6 +15,6 @@ */ output "routes" { - value = [for route in google_compute_route.route : route.name] - description = "The routes associated with this VPC" + value = google_compute_route.route + description = "The created routes resources" } diff --git a/modules/subnets/README.md b/modules/subnets/README.md index 6203ae25..df994d7a 100644 --- a/modules/subnets/README.md +++ b/modules/subnets/README.md @@ -71,14 +71,7 @@ module "vpc" { | Name | Description | |------|-------------| -| subnets | The subnet resources | -| subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | -| subnets\_ips | The IPs and CIDRs of the subnets being created | -| subnets\_names | The names of the subnets being created | -| subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | -| subnets\_regions | The region where the subnets will be created | -| subnets\_secondary\_ranges | The secondary ranges associated with these subnets | -| subnets\_self\_links | The self-links of subnets being created | +| subnets | The created subnet resources | diff --git a/modules/subnets/outputs.tf b/modules/subnets/outputs.tf index 950d814f..6ba07eb1 100644 --- a/modules/subnets/outputs.tf +++ b/modules/subnets/outputs.tf @@ -16,40 +16,5 @@ output "subnets" { value = google_compute_subnetwork.subnetwork - description = "The subnet resources" -} - -output "subnets_names" { - value = [for network in google_compute_subnetwork.subnetwork : network.name] - description = "The names of the subnets being created" -} - -output "subnets_ips" { - value = [for network in google_compute_subnetwork.subnetwork : network.ip_cidr_range] - description = "The IPs and CIDRs of the subnets being created" -} - -output "subnets_self_links" { - value = [for network in google_compute_subnetwork.subnetwork : network.self_link] - description = "The self-links of subnets being created" -} - -output "subnets_regions" { - value = [for network in google_compute_subnetwork.subnetwork : network.region] - description = "The region where the subnets will be created" -} - -output "subnets_private_access" { - value = [for network in google_compute_subnetwork.subnetwork : network.private_ip_google_access] - description = "Whether the subnets will have access to Google API's without a public IP" -} - -output "subnets_flow_logs" { - value = [for network in google_compute_subnetwork.subnetwork : network.enable_flow_logs] - description = "Whether the subnets will have VPC flow logs enabled" -} - -output "subnets_secondary_ranges" { - value = [for network in google_compute_subnetwork.subnetwork : network.secondary_ip_range] - description = "The secondary ranges associated with these subnets" + description = "The created subnet resources" } diff --git a/outputs.tf b/outputs.tf index fce8efaa..ae2591db 100644 --- a/outputs.tf +++ b/outputs.tf @@ -30,41 +30,41 @@ output "svpc_host_project_id" { } output "subnets_names" { - value = module.subnets.subnets_names + value = [for network in module.subnets.subnets : network.name] description = "The names of the subnets being created" } output "subnets_ips" { - value = module.subnets.subnets_ips + value = [for network in module.subnets.subnets : network.ip_cidr_range] description = "The IPs and CIDRs of the subnets being created" } output "subnets_self_links" { - value = module.subnets.subnets_self_links + value = [for network in module.subnets.subnets : network.self_link] description = "The self-links of subnets being created" } output "subnets_regions" { - value = module.subnets.subnets_regions + value = [for network in module.subnets.subnets : network.region] description = "The region where the subnets will be created" } output "subnets_private_access" { - value = module.subnets.subnets_private_access + value = [for network in module.subnets.subnets : network.private_ip_google_access] description = "Whether the subnets will have access to Google API's without a public IP" } output "subnets_flow_logs" { - value = module.subnets.subnets_flow_logs + value = [for network in module.subnets.subnets : network.enable_flow_logs] description = "Whether the subnets will have VPC flow logs enabled" } output "subnets_secondary_ranges" { - value = module.subnets.subnets_secondary_ranges + value = [for network in module.subnets.subnets : network.secondary_ip_range] description = "The secondary ranges associated with these subnets" } -output "routes" { - value = module.routes.routes - description = "The routes associated with this VPC" +output "route_names" { + value = [for route in module.routes.routes : route.name] + description = "The route names associated with this VPC" } From e95a6b7f8ce75dbb098b51e00ec7f05715439bc5 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:35:05 -0800 Subject: [PATCH 56/78] switch shared_vpc_host to bool --- README.md | 2 +- examples/submodule_svpc_access/main.tf | 2 +- modules/vpc/README.md | 2 +- modules/vpc/main.tf | 2 +- modules/vpc/variables.tf | 4 ++-- variables.tf | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 87a5161e..00501b50 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Then perform the following commands on the root folder: | routes | List of routes being created in this VPC | list(map(string)) | `` | no | | routing\_mode | The network routing mode (default 'GLOBAL') | string | `"GLOBAL"` | no | | secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | -| shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | string | `"false"` | no | +| shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | bool | `"false"` | no | | subnets | The list of subnets being created | list(map(string)) | n/a | yes | ## Outputs diff --git a/examples/submodule_svpc_access/main.tf b/examples/submodule_svpc_access/main.tf index dc0777bb..6ff832d1 100644 --- a/examples/submodule_svpc_access/main.tf +++ b/examples/submodule_svpc_access/main.tf @@ -25,7 +25,7 @@ module "net-vpc-shared" { source = "../.." project_id = var.host_project_id network_name = var.network_name - shared_vpc_host = "true" + shared_vpc_host = true subnets = [ { diff --git a/modules/vpc/README.md b/modules/vpc/README.md index ded03e33..43fbb32b 100644 --- a/modules/vpc/README.md +++ b/modules/vpc/README.md @@ -32,7 +32,7 @@ module "vpc" { | network\_name | The name of the network being created | string | n/a | yes | | project\_id | The ID of the project where this VPC will be created | string | n/a | yes | | routing\_mode | The network routing mode (default 'GLOBAL') | string | `"GLOBAL"` | no | -| shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | string | `"false"` | no | +| shared\_vpc\_host | Makes this project a Shared VPC host if 'true' (default 'false') | bool | `"false"` | no | ## Outputs diff --git a/modules/vpc/main.tf b/modules/vpc/main.tf index a1b249ae..55703793 100644 --- a/modules/vpc/main.tf +++ b/modules/vpc/main.tf @@ -29,7 +29,7 @@ resource "google_compute_network" "network" { Shared VPC *****************************************/ resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { - count = var.shared_vpc_host == "true" ? 1 : 0 + count = var.shared_vpc_host ? 1 : 0 project = var.project_id depends_on = [google_compute_network.network] } diff --git a/modules/vpc/variables.tf b/modules/vpc/variables.tf index 80089284..a96751c4 100644 --- a/modules/vpc/variables.tf +++ b/modules/vpc/variables.tf @@ -29,9 +29,9 @@ variable "routing_mode" { } variable "shared_vpc_host" { - type = string + type = bool description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = "false" + default = false } variable "description" { diff --git a/variables.tf b/variables.tf index 51d1d78b..1770d50f 100644 --- a/variables.tf +++ b/variables.tf @@ -29,9 +29,9 @@ variable "routing_mode" { } variable "shared_vpc_host" { - type = string + type = bool description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = "false" + default = false } variable "subnets" { From a351412b1447dbd54743b4f1ada8d27cc6204d79 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:41:39 -0800 Subject: [PATCH 57/78] simpler way to return project id for shared vpc --- modules/vpc/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vpc/outputs.tf b/modules/vpc/outputs.tf index e5238214..f2d46d8f 100644 --- a/modules/vpc/outputs.tf +++ b/modules/vpc/outputs.tf @@ -30,6 +30,6 @@ output "network_self_link" { } output "svpc_host_project_id" { - value = element(concat(google_compute_shared_vpc_host_project.shared_vpc_host.*.project, list("")), 0) + value = var.shared_vpc_host ? var.project_id : null description = "Shared VPC host project id." } From 41462836d3d216896d8e58b79974ccd66036ce21 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:52:29 -0800 Subject: [PATCH 58/78] update changelog and readme for list structure --- CHANGELOG.md | 2 +- README.md | 4 ++-- modules/routes/README.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab85452..24c47e68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ v2.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./d ### Added -- Split main module up into vpc, subnets, and routes submodules. Changed `routes` input to be compatible with `for_each` `maps` [#103] +- Split main module up into vpc, subnets, and routes submodules. [#103] ### Fixed diff --git a/README.md b/README.md index 00501b50..07087ee5 100644 --- a/README.md +++ b/README.md @@ -137,11 +137,11 @@ The subnets list contains maps, where each object represents a subnet. Each map ### Route Inputs -The `routes` map `key` is the unique route `name` and the `value` object represents the route input options. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): +The routes list contains maps, where each object represents a route. For the next_hop_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each map has the following inputs (please see examples folder for additional references): | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| key | The name of the route being created | string | - | yes | +| name | The name of the route being created | string | - | yes | | description | The description of the route being created | string | - | no | | tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | | destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes diff --git a/modules/routes/README.md b/modules/routes/README.md index 2b0dc390..0255887c 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -64,11 +64,11 @@ module "vpc" { ### Routes Input -The `routes` map `key` is the unique route `name` and the `value` object represents the route input options. For the next\_hop\_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each route has the following inputs (please see examples folder for additional references): +The routes list contains maps, where each object represents a route. For the next_hop_* inputs, only one is possible to be used in each route. Having two next_hop_* inputs will produce an error. Each map has the following inputs (please see examples folder for additional references): | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| key | The name of the route being created | string | - | yes | +| name | The name of the route being created | string | - | yes | | description | The description of the route being created | string | - | no | | tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | | destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes From c9a6090c387522bab1e6559e818e74350e859ed3 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 16:52:44 -0800 Subject: [PATCH 59/78] fix terraform fmt issues --- modules/routes/main.tf | 2 +- test/fixtures/shared/outputs.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/routes/main.tf b/modules/routes/main.tf index ba3802d2..78eaaa32 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -39,7 +39,7 @@ resource "google_compute_route" "route" { next_hop_instance = lookup(each.value, "next_hop_instance", null) next_hop_instance_zone = lookup(each.value, "next_hop_instance_zone", null) next_hop_vpn_tunnel = lookup(each.value, "next_hop_vpn_tunnel", null) - priority = lookup(each.value, "priority",null) + priority = lookup(each.value, "priority", null) depends_on = [ var.network, diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index fba7438b..d0d0713b 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -71,6 +71,6 @@ output "output_subnets_secondary_ranges" { } output "output_routes" { - value = module.example.routes - description = "The routes associated with this VPC" + value = module.example.route_names + description = "The route names associated with this VPC" } From db0448ea6c4661aab292efd7f404f008fdad5adc Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 17:16:31 -0800 Subject: [PATCH 60/78] name isn't required anymore --- README.md | 2 +- modules/routes/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 07087ee5..6ffb33d3 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ The routes list contains maps, where each object represents a route. For the nex | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| name | The name of the route being created | string | - | yes | +| name | The name of the route being created | string | - | no | | description | The description of the route being created | string | - | no | | tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | | destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes diff --git a/modules/routes/README.md b/modules/routes/README.md index 0255887c..3cc707bd 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -68,7 +68,7 @@ The routes list contains maps, where each object represents a route. For the nex | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| name | The name of the route being created | string | - | yes | +| name | The name of the route being created | string | - | no | | description | The description of the route being created | string | - | no | | tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | | destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes From 84119a83f3f8a43af41fb36be8d5d4276e4e51d2 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 25 Nov 2019 17:17:03 -0800 Subject: [PATCH 61/78] default output needs to be to match original value so tests and backwards compatability don't break --- modules/vpc/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vpc/outputs.tf b/modules/vpc/outputs.tf index f2d46d8f..64b1fd31 100644 --- a/modules/vpc/outputs.tf +++ b/modules/vpc/outputs.tf @@ -30,6 +30,6 @@ output "network_self_link" { } output "svpc_host_project_id" { - value = var.shared_vpc_host ? var.project_id : null + value = var.shared_vpc_host ? var.project_id : "" description = "Shared VPC host project id." } From babfafa764e9bf1523b67c66ce1cfc458e391329 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 10:51:21 -0800 Subject: [PATCH 62/78] fix readme examples --- modules/subnets/README.md | 2 +- modules/vpc/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/subnets/README.md b/modules/subnets/README.md index df994d7a..b68f79de 100644 --- a/modules/subnets/README.md +++ b/modules/subnets/README.md @@ -12,7 +12,7 @@ Basic usage of this submodule is as follows: ```hcl module "vpc" { - source = "terraform-google-modules/network/google//modules/routes" + source = "terraform-google-modules/network/google//modules/subnets" version = "~> 2.0.0" project_id = "" diff --git a/modules/vpc/README.md b/modules/vpc/README.md index 43fbb32b..be502915 100644 --- a/modules/vpc/README.md +++ b/modules/vpc/README.md @@ -13,7 +13,7 @@ Basic usage of this submodule is as follows: ```hcl module "vpc" { - source = "terraform-google-modules/network/google//modules/routes" + source = "terraform-google-modules/network/google//modules/vpc" version = "~> 2.0.0" project_id = "" From 6b1055594cd3c06e14dbd8d8befe3c891cb30351 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 10:52:26 -0800 Subject: [PATCH 63/78] remove unused leftover var --- modules/subnets/README.md | 1 - modules/subnets/variables.tf | 6 ------ 2 files changed, 7 deletions(-) diff --git a/modules/subnets/README.md b/modules/subnets/README.md index b68f79de..d3d764fc 100644 --- a/modules/subnets/README.md +++ b/modules/subnets/README.md @@ -61,7 +61,6 @@ module "vpc" { | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| description | An optional description of this resource. The resource must be recreated to modify this field. | string | `""` | no | | network\_name | The name of the network where subnets will be created | string | n/a | yes | | project\_id | The ID of the project where subnets will be created | string | n/a | yes | | secondary\_ranges | Secondary ranges that will be used in some of the subnets | object | `` | no | diff --git a/modules/subnets/variables.tf b/modules/subnets/variables.tf index c2031d50..84d7b099 100644 --- a/modules/subnets/variables.tf +++ b/modules/subnets/variables.tf @@ -32,9 +32,3 @@ variable "secondary_ranges" { description = "Secondary ranges that will be used in some of the subnets" default = {} } - -variable "description" { - type = string - description = "An optional description of this resource. The resource must be recreated to modify this field." - default = "" -} From f0443f7ed0195759cd140d04e28bfa5dc0041b59 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 10:54:04 -0800 Subject: [PATCH 64/78] remove network var for route depedency. Its inherit by passing in network name --- main.tf | 1 - modules/routes/README.md | 1 - modules/routes/main.tf | 2 -- modules/routes/variables.tf | 5 ----- 4 files changed, 9 deletions(-) diff --git a/main.tf b/main.tf index ca5c20b3..ad12e396 100644 --- a/main.tf +++ b/main.tf @@ -48,6 +48,5 @@ module "routes" { network_name = module.vpc.network_name routes = var.routes delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - network = module.vpc.network subnets = module.subnets.subnets } diff --git a/modules/routes/README.md b/modules/routes/README.md index 3cc707bd..9d0595de 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -47,7 +47,6 @@ module "vpc" { | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | delete\_default\_internet\_gateway\_routes | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | string | `"false"` | no | -| network | The network resource that we depend on being created first | string | `"null"` | no | | network\_name | The name of the network where routes will be created | string | n/a | yes | | project\_id | The ID of the project where the routes will be created | string | n/a | yes | | routes | List of routes being created in this VPC | list(map(string)) | `` | no | diff --git a/modules/routes/main.tf b/modules/routes/main.tf index 78eaaa32..17b74054 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -42,7 +42,6 @@ resource "google_compute_route" "route" { priority = lookup(each.value, "priority", null) depends_on = [ - var.network, var.subnets, ] } @@ -59,7 +58,6 @@ resource "null_resource" "delete_default_internet_gateway_routes" { } depends_on = [ - var.network, var.subnets, google_compute_route.route, ] diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf index ba68bf82..349c0c96 100644 --- a/modules/routes/variables.tf +++ b/modules/routes/variables.tf @@ -22,11 +22,6 @@ variable "network_name" { description = "The name of the network where routes will be created" } -variable "network" { - description = "The network resource that we depend on being created first" - default = null -} - variable "subnets" { description = "The subnet resources that we depend on being created first" default = [] From f86ad432865d981afd8ca0a81f57f058abfb10d7 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 11:05:26 -0800 Subject: [PATCH 65/78] remove description --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index ad12e396..f7255c63 100644 --- a/main.tf +++ b/main.tf @@ -34,7 +34,6 @@ module "subnets" { source = "./modules/subnets" project_id = var.project_id network_name = module.vpc.network_name - description = var.description subnets = var.subnets secondary_ranges = var.secondary_ranges } From 2f6aee72cb379161a4fd7bd645695484b37b6c81 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 11:32:48 -0800 Subject: [PATCH 66/78] rename svpc_project_id to project_id --- .kitchen.yml | 4 ++-- README.md | 2 +- examples/secondary_ranges/README.md | 2 +- examples/secondary_ranges/outputs.tf | 6 +++--- examples/simple_project/README.md | 2 +- examples/simple_project/outputs.tf | 6 +++--- examples/simple_project_with_regional_network/README.md | 2 +- examples/simple_project_with_regional_network/outputs.tf | 6 +++--- examples/submodule_firewall/README.md | 2 +- examples/submodule_firewall/outputs.tf | 6 +++--- examples/submodule_svpc_access/main.tf | 2 +- modules/vpc/README.md | 2 +- modules/vpc/outputs.tf | 6 +++--- outputs.tf | 6 +++--- test/fixtures/shared/outputs.tf | 6 +++--- .../secondary_ranges/controls/inspec_attributes.rb | 4 ++-- test/integration/secondary_ranges/inspec.yml | 2 +- .../submodule_firewall/controls/inspec_attributes.rb | 4 ++-- test/integration/submodule_firewall/inspec.yml | 2 +- 19 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index ed06b86b..23ea6d2f 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -74,7 +74,7 @@ suites: customized_inspec_attribute: output_subnets_private_access customized_inspec_attribute: output_subnets_regions customized_inspec_attribute: output_subnets_secondary_ranges - customized_inspec_attribute: output_svpc_host_project_id + customized_inspec_attribute: output_project_id backend: local @@ -131,7 +131,7 @@ suites: customized_inspec_attribute: output_subnets_private_access customized_inspec_attribute: output_subnets_regions customized_inspec_attribute: output_subnets_secondary_ranges - customized_inspec_attribute: output_svpc_host_project_id + customized_inspec_attribute: output_project_id backend: local controls: - gcloud diff --git a/README.md b/README.md index 6ffb33d3..859eb10f 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ Then perform the following commands on the root folder: |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | +| project\_id | VPC project id | | route\_names | The route names associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IPs and CIDRs of the subnets being created | @@ -119,7 +120,6 @@ Then perform the following commands on the root folder: | subnets\_regions | The region where the subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | | subnets\_self\_links | The self-links of subnets being created | -| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/secondary_ranges/README.md b/examples/secondary_ranges/README.md index f44a18e6..acca7c73 100644 --- a/examples/secondary_ranges/README.md +++ b/examples/secondary_ranges/README.md @@ -19,6 +19,7 @@ ranges and the third being given a single secondary range. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | +| project\_id | VPC project id | | route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | @@ -26,6 +27,5 @@ ranges and the third being given a single secondary range. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | -| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/secondary_ranges/outputs.tf b/examples/secondary_ranges/outputs.tf index 7bab5b12..6c3f49cb 100644 --- a/examples/secondary_ranges/outputs.tf +++ b/examples/secondary_ranges/outputs.tf @@ -24,9 +24,9 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = module.vpc-secondary-ranges.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc-secondary-ranges.project_id + description = "VPC project id" } output "subnets_names" { diff --git a/examples/simple_project/README.md b/examples/simple_project/README.md index a5fd3d97..a4325668 100644 --- a/examples/simple_project/README.md +++ b/examples/simple_project/README.md @@ -18,6 +18,7 @@ This VPC has two subnets, with no secondary ranges. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | +| project\_id | VPC project id | | route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | @@ -25,6 +26,5 @@ This VPC has two subnets, with no secondary ranges. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | -| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/simple_project/outputs.tf b/examples/simple_project/outputs.tf index 9492e630..ca38a0bf 100644 --- a/examples/simple_project/outputs.tf +++ b/examples/simple_project/outputs.tf @@ -24,9 +24,9 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = module.test-vpc-module.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc-secondary-ranges.project_id + description = "VPC project id" } output "subnets_names" { diff --git a/examples/simple_project_with_regional_network/README.md b/examples/simple_project_with_regional_network/README.md index acd2b147..354711e2 100644 --- a/examples/simple_project_with_regional_network/README.md +++ b/examples/simple_project_with_regional_network/README.md @@ -18,6 +18,7 @@ This VPC has two subnets, with no secondary ranges. |------|-------------| | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | +| project\_id | VPC project id | | route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | @@ -25,6 +26,5 @@ This VPC has two subnets, with no secondary ranges. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | -| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/simple_project_with_regional_network/outputs.tf b/examples/simple_project_with_regional_network/outputs.tf index 9492e630..ca38a0bf 100644 --- a/examples/simple_project_with_regional_network/outputs.tf +++ b/examples/simple_project_with_regional_network/outputs.tf @@ -24,9 +24,9 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = module.test-vpc-module.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc-secondary-ranges.project_id + description = "VPC project id" } output "subnets_names" { diff --git a/examples/submodule_firewall/README.md b/examples/submodule_firewall/README.md index 229a61da..48f2bd1c 100644 --- a/examples/submodule_firewall/README.md +++ b/examples/submodule_firewall/README.md @@ -20,6 +20,7 @@ This VPC has two subnets, with no secondary ranges. | internal\_ranges | Firewall attributes for internal ranges. | | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | +| project\_id | VPC project id | | route\_names | The routes associated with this VPC | | subnets\_flow\_logs | Whether the subnets will have VPC flow logs enabled | | subnets\_ips | The IP and cidrs of the subnets being created | @@ -27,6 +28,5 @@ This VPC has two subnets, with no secondary ranges. | subnets\_private\_access | Whether the subnets will have access to Google API's without a public IP | | subnets\_regions | The region where subnets will be created | | subnets\_secondary\_ranges | The secondary ranges associated with these subnets | -| svpc\_host\_project\_id | Shared VPC host project id. | diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index 662d9cac..63a5b0df 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -34,9 +34,9 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = module.test-vpc-module.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc-secondary-ranges.project_id + description = "VPC project id" } output "subnets_names" { diff --git a/examples/submodule_svpc_access/main.tf b/examples/submodule_svpc_access/main.tf index 6ff832d1..edb65bb1 100644 --- a/examples/submodule_svpc_access/main.tf +++ b/examples/submodule_svpc_access/main.tf @@ -43,7 +43,7 @@ module "net-vpc-shared" { module "net-svpc-access" { source = "../../modules/fabric-net-svpc-access" - host_project_id = module.net-vpc-shared.svpc_host_project_id + host_project_id = module.net-vpc-shared.project_id service_project_num = 1 service_project_ids = [var.service_project_id] host_subnets = ["data"] diff --git a/modules/vpc/README.md b/modules/vpc/README.md index be502915..cae59d02 100644 --- a/modules/vpc/README.md +++ b/modules/vpc/README.md @@ -41,6 +41,6 @@ module "vpc" { | network | The VPC resource being created | | network\_name | The name of the VPC being created | | network\_self\_link | The URI of the VPC being created | -| svpc\_host\_project\_id | Shared VPC host project id. | +| project\_id | VPC project id | diff --git a/modules/vpc/outputs.tf b/modules/vpc/outputs.tf index 64b1fd31..8133169d 100644 --- a/modules/vpc/outputs.tf +++ b/modules/vpc/outputs.tf @@ -29,7 +29,7 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = var.shared_vpc_host ? var.project_id : "" - description = "Shared VPC host project id." +output "project_id" { + value = var.project_id + description = "VPC project id" } diff --git a/outputs.tf b/outputs.tf index ae2591db..44aebcad 100644 --- a/outputs.tf +++ b/outputs.tf @@ -24,9 +24,9 @@ output "network_self_link" { description = "The URI of the VPC being created" } -output "svpc_host_project_id" { - value = module.vpc.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc.project_id + description = "VPC project id" } output "subnets_names" { diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index d0d0713b..9f9de444 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -34,9 +34,9 @@ output "output_network_self_link" { description = "The URI of the VPC being created" } -output "output_svpc_host_project_id" { - value = module.example.svpc_host_project_id - description = "Shared VPC host project id." +output "project_id" { + value = module.vpc-secondary-ranges.project_id + description = "VPC project id" } output "output_subnets_names" { diff --git a/test/integration/secondary_ranges/controls/inspec_attributes.rb b/test/integration/secondary_ranges/controls/inspec_attributes.rb index 9e11b6cd..b878c13d 100644 --- a/test/integration/secondary_ranges/controls/inspec_attributes.rb +++ b/test/integration/secondary_ranges/controls/inspec_attributes.rb @@ -55,7 +55,7 @@ it { should eq [{"ip_cidr_range"=>"192.168.64.0/24", "range_name"=>"#{network_name}-subnet-01-01"}, {"ip_cidr_range"=>"192.168.65.0/24", "range_name"=>"#{network_name}-subnet-01-02"}, {"ip_cidr_range"=>"192.168.66.0/24", "range_name"=>"#{network_name}-subnet-03-01"}] } end - describe attribute("output_svpc_host_project_id") do - it { should eq "" } + describe attribute("project_id") do + it { should eq project_id } end end diff --git a/test/integration/secondary_ranges/inspec.yml b/test/integration/secondary_ranges/inspec.yml index ea9c3b05..c11e6612 100644 --- a/test/integration/secondary_ranges/inspec.yml +++ b/test/integration/secondary_ranges/inspec.yml @@ -26,5 +26,5 @@ attributes: required: true - name: output_subnets_secondary_ranges required: true - - name: output_svpc_host_project_id + - name: output_project_id required: true diff --git a/test/integration/submodule_firewall/controls/inspec_attributes.rb b/test/integration/submodule_firewall/controls/inspec_attributes.rb index 05ca71cb..25320c41 100644 --- a/test/integration/submodule_firewall/controls/inspec_attributes.rb +++ b/test/integration/submodule_firewall/controls/inspec_attributes.rb @@ -55,7 +55,7 @@ it { should eq [[],[]] } end - describe attribute("output_svpc_host_project_id") do - it { should eq "" } + describe attribute("output_project_id") do + it { should eq project_id } end end diff --git a/test/integration/submodule_firewall/inspec.yml b/test/integration/submodule_firewall/inspec.yml index 67efdd29..43e9d744 100644 --- a/test/integration/submodule_firewall/inspec.yml +++ b/test/integration/submodule_firewall/inspec.yml @@ -30,5 +30,5 @@ attributes: required: true - name: output_subnets_secondary_ranges required: true - - name: output_svpc_host_project_id + - name: output_project_id required: true From 6e7ffd18fd638bf4caa15fc74ce10d46e2b64f0e Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 11:43:19 -0800 Subject: [PATCH 67/78] use a module_depends_on list instead of using explicit subnet --- main.tf | 2 +- modules/routes/README.md | 2 +- modules/routes/main.tf | 5 +---- modules/routes/variables.tf | 11 ++++++----- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/main.tf b/main.tf index f7255c63..93794145 100644 --- a/main.tf +++ b/main.tf @@ -47,5 +47,5 @@ module "routes" { network_name = module.vpc.network_name routes = var.routes delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - subnets = module.subnets.subnets + module_depends_on = [module.subnets.subnets] } diff --git a/modules/routes/README.md b/modules/routes/README.md index 9d0595de..8051ac5d 100644 --- a/modules/routes/README.md +++ b/modules/routes/README.md @@ -47,10 +47,10 @@ module "vpc" { | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | delete\_default\_internet\_gateway\_routes | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | string | `"false"` | no | +| module\_depends\_on | List of modules or resources this module depends on. | list | `` | no | | network\_name | The name of the network where routes will be created | string | n/a | yes | | project\_id | The ID of the project where the routes will be created | string | n/a | yes | | routes | List of routes being created in this VPC | list(map(string)) | `` | no | -| subnets | The subnet resources that we depend on being created first | list | `` | no | ## Outputs diff --git a/modules/routes/main.tf b/modules/routes/main.tf index 17b74054..cb1edf31 100644 --- a/modules/routes/main.tf +++ b/modules/routes/main.tf @@ -41,9 +41,7 @@ resource "google_compute_route" "route" { next_hop_vpn_tunnel = lookup(each.value, "next_hop_vpn_tunnel", null) priority = lookup(each.value, "priority", null) - depends_on = [ - var.subnets, - ] + depends_on = [var.module_depends_on] } resource "null_resource" "delete_default_internet_gateway_routes" { @@ -58,7 +56,6 @@ resource "null_resource" "delete_default_internet_gateway_routes" { } depends_on = [ - var.subnets, google_compute_route.route, ] } diff --git a/modules/routes/variables.tf b/modules/routes/variables.tf index 349c0c96..8eed495f 100644 --- a/modules/routes/variables.tf +++ b/modules/routes/variables.tf @@ -22,11 +22,6 @@ variable "network_name" { description = "The name of the network where routes will be created" } -variable "subnets" { - description = "The subnet resources that we depend on being created first" - default = [] -} - variable "routes" { type = list(map(string)) description = "List of routes being created in this VPC" @@ -37,3 +32,9 @@ variable "delete_default_internet_gateway_routes" { description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" default = "false" } + +variable "module_depends_on" { + description = "List of modules or resources this module depends on." + type = list + default = [] +} From da5372d51b36b5c7bc9c018353ab227104aa01b3 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 26 Nov 2019 11:49:32 -0800 Subject: [PATCH 68/78] lint/validate fixes --- examples/simple_project/outputs.tf | 2 +- examples/simple_project_with_regional_network/outputs.tf | 2 +- examples/submodule_firewall/outputs.tf | 2 +- test/fixtures/shared/outputs.tf | 6 ------ 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/examples/simple_project/outputs.tf b/examples/simple_project/outputs.tf index ca38a0bf..f69ae043 100644 --- a/examples/simple_project/outputs.tf +++ b/examples/simple_project/outputs.tf @@ -25,7 +25,7 @@ output "network_self_link" { } output "project_id" { - value = module.vpc-secondary-ranges.project_id + value = module.test-vpc-module.project_id description = "VPC project id" } diff --git a/examples/simple_project_with_regional_network/outputs.tf b/examples/simple_project_with_regional_network/outputs.tf index ca38a0bf..f69ae043 100644 --- a/examples/simple_project_with_regional_network/outputs.tf +++ b/examples/simple_project_with_regional_network/outputs.tf @@ -25,7 +25,7 @@ output "network_self_link" { } output "project_id" { - value = module.vpc-secondary-ranges.project_id + value = module.test-vpc-module.project_id description = "VPC project id" } diff --git a/examples/submodule_firewall/outputs.tf b/examples/submodule_firewall/outputs.tf index 63a5b0df..182dc845 100644 --- a/examples/submodule_firewall/outputs.tf +++ b/examples/submodule_firewall/outputs.tf @@ -35,7 +35,7 @@ output "network_self_link" { } output "project_id" { - value = module.vpc-secondary-ranges.project_id + value = module.test-vpc-module.project_id description = "VPC project id" } diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 9f9de444..651f0e00 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -34,11 +34,6 @@ output "output_network_self_link" { description = "The URI of the VPC being created" } -output "project_id" { - value = module.vpc-secondary-ranges.project_id - description = "VPC project id" -} - output "output_subnets_names" { value = module.example.subnets_names description = "The names of the subnets being created" @@ -49,7 +44,6 @@ output "output_subnets_ips" { description = "The IP and cidrs of the subnets being created" } - output "output_subnets_regions" { value = module.example.subnets_regions description = "The region where subnets will be created" From 7ec22d87ed57cb38a0b69d85507777bb129f8e16 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Wed, 27 Nov 2019 10:19:25 -0800 Subject: [PATCH 69/78] initial migrate.py from gke module --- helpers/migrate.py | 318 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 helpers/migrate.py diff --git a/helpers/migrate.py b/helpers/migrate.py new file mode 100644 index 00000000..8f2d71cf --- /dev/null +++ b/helpers/migrate.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import subprocess +import sys +import re + +MIGRATIONS = [ + { + "resource_type": "google_container_cluster", + "name": "zonal_primary", + "rename": "primary", + "module": "" + }, + { + "resource_type": "google_container_node_pool", + "name": "zonal_pools", + "rename": "pools", + "module": "" + }, + { + "resource_type": "null_resource", + "name": "wait_for_zonal_cluster", + "rename": "wait_for_cluster", + "module": "", + "plural": False + }, +] + + +class ModuleMigration: + """ + Migrate the resources from a flat project factory to match the new + module structure created by the G Suite refactor. + """ + + def __init__(self, source_module): + self.source_module = source_module + + def moves(self): + """ + Generate the set of old/new resource pairs that will be migrated + to the `destination` module. + """ + resources = self.targets() + moves = [] + for (old, migration) in resources: + new = copy.deepcopy(old) + new.module += migration["module"] + + # Update the copied resource with the "rename" value if it is set + if "rename" in migration: + new.name = migration["rename"] + + old.plural = migration.get("plural", True) + new.plural = migration.get("plural", True) + + pair = (old.path(), new.path()) + moves.append(pair) + return moves + + def targets(self): + """ + A list of resources that will be moved to the new module """ + to_move = [] + + for migration in MIGRATIONS: + resource_type = migration["resource_type"] + resource_name = migration["name"] + matching_resources = self.source_module.get_resources( + resource_type, + resource_name) + to_move += [(r, migration) for r in matching_resources] + + return to_move + + +class TerraformModule: + """ + A Terraform module with associated resources. + """ + + def __init__(self, name, resources): + """ + Create a new module and associate it with a list of resources. + """ + self.name = name + self.resources = resources + + def get_resources(self, resource_type=None, resource_name=None): + """ + Return a list of resources matching the given resource type and name. + """ + + ret = [] + for resource in self.resources: + matches_type = (resource_type is None or + resource_type == resource.resource_type) + + name_pattern = re.compile(r'%s(\[\d+\])?' % resource_name) + matches_name = (resource_name is None or + name_pattern.match(resource.name)) + + if matches_type and matches_name: + ret.append(resource) + + return ret + + def has_resource(self, resource_type=None, resource_name=None): + """ + Does this module contain a resource with the matching type and name? + """ + for resource in self.resources: + matches_type = (resource_type is None or + resource_type == resource.resource_type) + + matches_name = (resource_name is None or + resource_name in resource.name) + + if matches_type and matches_name: + return True + + return False + + def __repr__(self): + return "{}({!r}, {!r})".format( + self.__class__.__name__, + self.name, + [repr(resource) for resource in self.resources]) + + +class TerraformResource: + """ + A Terraform resource, defined by the the identifier of that resource. + """ + + @classmethod + def from_path(cls, path): + """ + Generate a new Terraform resource, based on the fully qualified + Terraform resource path. + """ + if re.match(r'\A[\w.\[\]-]+\Z', path) is None: + raise ValueError( + "Invalid Terraform resource path {!r}".format(path)) + + parts = path.split(".") + name = parts.pop() + resource_type = parts.pop() + module = ".".join(parts) + return cls(module, resource_type, name) + + def __init__(self, module, resource_type, name): + """ + Create a new TerraformResource from a pre-parsed path. + """ + self.module = module + self.resource_type = resource_type + + find_suffix = re.match(r'(^.+)\[(\d+)\]', name) + if find_suffix: + self.name = find_suffix.group(1) + self.index = find_suffix.group(2) + else: + self.name = name + self.index = -1 + + def path(self): + """ + Return the fully qualified resource path. + """ + parts = [self.module, self.resource_type, self.name] + if parts[0] == '': + del parts[0] + path = ".".join(parts) + if self.index != -1 and self.plural: + path = "{0}[{1}]".format(path, self.index) + return path + + def __repr__(self): + return "{}({!r}, {!r}, {!r})".format( + self.__class__.__name__, + self.module, + self.resource_type, + self.name) + + +def group_by_module(resources): + """ + Group a set of resources according to their containing module. + """ + + groups = {} + for resource in resources: + if resource.module in groups: + groups[resource.module].append(resource) + else: + groups[resource.module] = [resource] + + return [ + TerraformModule(name, contained) + for name, contained in groups.items() + ] + + +def read_state(statefile=None): + """ + Read the terraform state at the given path. + """ + argv = ["terraform", "state", "list"] + result = subprocess.run(argv, + capture_output=True, + check=True, + encoding='utf-8') + elements = result.stdout.split("\n") + elements.pop() + return elements + + +def state_changes_for_module(module, statefile=None): + """ + Compute the Terraform state changes (deletions and moves) for a single + module. + """ + commands = [] + + migration = ModuleMigration(module) + + for (old, new) in migration.moves(): + wrapper = '"{0}"' + argv = ["terraform", + "state", + "mv", + wrapper.format(old), + wrapper.format(new)] + commands.append(argv) + + return commands + + +def migrate(statefile=None, dryrun=False): + """ + Migrate the terraform state in `statefile` to match the post-refactor + resource structure. + """ + + # Generate a list of Terraform resource states from the output of + # `terraform state list` + resources = [ + TerraformResource.from_path(path) + for path in read_state(statefile) + ] + + # Group resources based on the module where they're defined. + modules = group_by_module(resources) + + # Filter our list of Terraform modules down to anything that looks like a + # zonal GKE module. We key this off the presence off of + # `google_container_cluster.zonal_primary` since that should almost always + # be unique to a GKE module. + modules_to_migrate = [ + module for module in modules + if module.has_resource("google_container_cluster", "zonal_primary") + ] + + print("---- Migrating the following modules:") + for module in modules_to_migrate: + print("-- " + module.name) + + # Collect a list of resources for each module + commands = [] + for module in modules_to_migrate: + commands += state_changes_for_module(module, statefile) + + print("---- Commands to run:") + for argv in commands: + if dryrun: + print(" ".join(argv)) + else: + argv = [arg.strip('"') for arg in argv] + subprocess.run(argv, check=True, encoding='utf-8') + + +def main(argv): + parser = argparser() + args = parser.parse_args(argv[1:]) + + # print("cp {} {}".format(args.oldstate, args.newstate)) + # shutil.copy(args.oldstate, args.newstate) + + migrate(dryrun=args.dryrun) + + +def argparser(): + parser = argparse.ArgumentParser(description='Migrate Terraform state') + parser.add_argument('--dryrun', action='store_true', + help='Print the `terraform state mv` commands instead ' + 'of running the commands.') + return parser + + +if __name__ == "__main__": + main(sys.argv) From 99a3819feb12cae6f81298adb5dc943b99ba3336 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Wed, 27 Nov 2019 13:12:31 -0800 Subject: [PATCH 70/78] update migrate.py to work with google-network resources and for_each migration --- helpers/migrate.py | 169 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 132 insertions(+), 37 deletions(-) mode change 100644 => 100755 helpers/migrate.py diff --git a/helpers/migrate.py b/helpers/migrate.py old mode 100644 new mode 100755 index 8f2d71cf..62c7ae11 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -19,27 +19,39 @@ import subprocess import sys import re +import json MIGRATIONS = [ { - "resource_type": "google_container_cluster", - "name": "zonal_primary", - "rename": "primary", - "module": "" + "resource_type": "google_compute_network", + "name": "network", + "module": ".module.vpc", + "new_plural": False }, { - "resource_type": "google_container_node_pool", - "name": "zonal_pools", - "rename": "pools", - "module": "" + "resource_type": "google_compute_shared_vpc_host_project", + "name": "shared_vpc_host", + "module": ".module.vpc" }, { - "resource_type": "null_resource", - "name": "wait_for_zonal_cluster", - "rename": "wait_for_cluster", - "module": "", - "plural": False + "resource_type": "google_compute_subnetwork", + "name": "subnetwork", + "module": ".module.subnets", + "for_each_migration": True, + "for_each_migration_key": "id" + }, + { + "resource_type": "google_compute_route", + "name": "route", + "module": ".module.routes", + "for_each_migration": True, + "for_each_migration_key": "id" }, + { + "resource_type": "null_resource", + "name": "delete_default_internet_gateway_routes", + "module": ".module.routes" + } ] @@ -49,8 +61,9 @@ class ModuleMigration: module structure created by the G Suite refactor. """ - def __init__(self, source_module): + def __init__(self, source_module, state): self.source_module = source_module + self.state = state def moves(self): """ @@ -58,6 +71,8 @@ def moves(self): to the `destination` module. """ resources = self.targets() + for_each_migrations = [] + moves = [] for (old, migration) in resources: new = copy.deepcopy(old) @@ -67,13 +82,48 @@ def moves(self): if "rename" in migration: new.name = migration["rename"] - old.plural = migration.get("plural", True) - new.plural = migration.get("plural", True) + old.plural = migration.get("old_plural", True) + new.plural = migration.get("new_plural", True) + + if migration.get("for_each_migration", False) and migration.get("old_plural", True): + for_each_migrations.append((old, new, migration)) + else: + pair = (old.path(), new.path()) + moves.append(pair) + + + for_each_moves = self.for_each_moves(for_each_migrations) + return moves + for_each_moves + + def for_each_moves(self, for_each_migrations): + """ + When migrating from count to for_each we need to move the whole collection first + https://github.com/hashicorp/terraform/issues/22301 + """ + for_each_initial_migration = {} + moves = [] - pair = (old.path(), new.path()) + for (old, new, migration) in for_each_migrations: + # Do the initial migration of the whole collection only once if it hasn't been done yet + key = old.resource_type + "." + old.name + if key not in for_each_initial_migration: + for_each_initial_migration[key] = True + old.plural = False + new.plural = False + + pair = (old.path(), new.path()) + moves.append(pair) + + # Whole collection is moved to new location. Now needs right index + new.plural = True + new_indexed = copy.deepcopy(new) + new_indexed.key = self.state.resource_value(old, migration["for_each_migration_key"]) + pair = (new.path(), new_indexed.path()) moves.append(pair) + return moves + def targets(self): """ A list of resources that will be moved to the new module """ @@ -155,7 +205,7 @@ def from_path(cls, path): Generate a new Terraform resource, based on the fully qualified Terraform resource path. """ - if re.match(r'\A[\w.\[\]-]+\Z', path) is None: + if re.match(r'\A[\w.\["/\]-]+\Z', path) is None: raise ValueError( "Invalid Terraform resource path {!r}".format(path)) @@ -171,6 +221,8 @@ def __init__(self, module, resource_type, name): """ self.module = module self.resource_type = resource_type + self.key = None + self.plural = True find_suffix = re.match(r'(^.+)\[(\d+)\]', name) if find_suffix: @@ -188,7 +240,9 @@ def path(self): if parts[0] == '': del parts[0] path = ".".join(parts) - if self.index != -1 and self.plural: + if self.key is not None: + path = "{0}[\"{1}\"]".format(path, self.key) + elif self.index != -1 and self.plural: path = "{0}[{1}]".format(path, self.index) return path @@ -199,6 +253,50 @@ def __repr__(self): self.resource_type, self.name) +class TerraformState: + """ + A Terraform state representation, pulled from terraform state pull + Used for getting values out of individual resources + """ + def __init__(self): + self.read_state() + + + def read_state(self): + """ + Read the terraform state + """ + argv = ["terraform", "state", "pull"] + result = subprocess.run(argv, + capture_output=True, + check=True, + encoding='utf-8') + + self.state = json.loads(result.stdout) + + def resource_value(self, resource, key): + # Find the resource in the state + state_resource_list = [r for r in self.state["resources"] if + r["module"] == resource.module and + r["type"] == resource.resource_type and + r["name"] == resource.name ] + + if (len(state_resource_list) != 1): + raise ValueError("Could not find resource list in state for {}".format(resource)) + + index = int(resource.index) + # If this a collection use the index to find the right resource, otherwise use the first + if (index >= 0): + state_resource = [r for r in state_resource_list[0]["instances"] if + r["index_key"] == index] + + if (len(state_resource) != 1): + raise ValueError("Could not find resource in state for {} key {}".format(resource, resource.index)) + else: + state_resource = state_resource_list[0]["instances"] + + return state_resource[0]["attributes"][key] + def group_by_module(resources): """ @@ -218,7 +316,7 @@ def group_by_module(resources): ] -def read_state(statefile=None): +def read_resources(): """ Read the terraform state at the given path. """ @@ -232,17 +330,17 @@ def read_state(statefile=None): return elements -def state_changes_for_module(module, statefile=None): +def state_changes_for_module(module, state): """ Compute the Terraform state changes (deletions and moves) for a single module. """ commands = [] - migration = ModuleMigration(module) + migration = ModuleMigration(module, state) for (old, new) in migration.moves(): - wrapper = '"{0}"' + wrapper = "'{0}'" argv = ["terraform", "state", "mv", @@ -253,29 +351,28 @@ def state_changes_for_module(module, statefile=None): return commands -def migrate(statefile=None, dryrun=False): +def migrate(state=None, dryrun=False): """ - Migrate the terraform state in `statefile` to match the post-refactor - resource structure. + Generate and run terraform state mv commands to migrate resources from one + state structure to another """ # Generate a list of Terraform resource states from the output of # `terraform state list` resources = [ TerraformResource.from_path(path) - for path in read_state(statefile) + for path in read_resources() ] # Group resources based on the module where they're defined. modules = group_by_module(resources) # Filter our list of Terraform modules down to anything that looks like a - # zonal GKE module. We key this off the presence off of - # `google_container_cluster.zonal_primary` since that should almost always - # be unique to a GKE module. + # google network original module. We key this off the presence off of + # `terraform-google-network` resource type and names modules_to_migrate = [ module for module in modules - if module.has_resource("google_container_cluster", "zonal_primary") + if module.has_resource("google_compute_network", "network") ] print("---- Migrating the following modules:") @@ -285,14 +382,14 @@ def migrate(statefile=None, dryrun=False): # Collect a list of resources for each module commands = [] for module in modules_to_migrate: - commands += state_changes_for_module(module, statefile) + commands += state_changes_for_module(module, state) print("---- Commands to run:") for argv in commands: if dryrun: print(" ".join(argv)) else: - argv = [arg.strip('"') for arg in argv] + argv = [arg.strip("'") for arg in argv] subprocess.run(argv, check=True, encoding='utf-8') @@ -300,11 +397,9 @@ def main(argv): parser = argparser() args = parser.parse_args(argv[1:]) - # print("cp {} {}".format(args.oldstate, args.newstate)) - # shutil.copy(args.oldstate, args.newstate) - - migrate(dryrun=args.dryrun) + state = TerraformState() + migrate(state=state, dryrun=args.dryrun) def argparser(): parser = argparse.ArgumentParser(description='Migrate Terraform state') From 69615daebac411d199b79c50d39c97c00ab9153f Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Wed, 27 Nov 2019 13:13:17 -0800 Subject: [PATCH 71/78] remove old sh migrate script --- helpers/migrate.sh | 61 ---------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100755 helpers/migrate.sh diff --git a/helpers/migrate.sh b/helpers/migrate.sh deleted file mode 100755 index 752fc06a..00000000 --- a/helpers/migrate.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# shellcheck shell=bash -# Output Terraform Commands to migrate to new subnet config -set -e -set -o pipefail -CMD="terraform state" - -while (( "$#" )); do - # shellcheck disable=SC2221,SC2222 - case "$1" in - -d|--dry-run) - DRY_RUN=true - shift 1 - ;; - --) # end argument parsing - shift - break - ;; - -*|--*=) # unsupported flags - echo "Error: Unsupported flag $1" >&2 - exit 1 - ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift - ;; - esac -done - -eval set -- "$PARAMS" - -if [ ! -e "$(command -v terraform)" ]; then - echo "can not find terraform" - exit 1 -fi - -MODULES=$(${CMD} list | grep google_compute_network.network) -for module in $MODULES; do - NAME=$(sed 's/.google_compute_network.network//' <<<"${module}") - for x in $($CMD list | grep "${NAME}".google_compute_subnetwork.subnetwork); do - ID=$(${CMD} show "$x" | grep id | grep -v ip_cidr_range | awk '{ print $3 }'| tr -d '"') - if [[ $DRY_RUN ]]; then - echo "${CMD} mv $x ${NAME}.google_compute_subnetwork.subnetwork[\\\"${ID}\\\"]" - else - ${CMD} mv "$x" "${NAME}".google_compute_subnetwork.subnetwork[\""${ID}"\"] - fi - done -done From 2d4027a89663cc6b1350ca749c5ef8e24f003af2 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Wed, 27 Nov 2019 13:18:57 -0800 Subject: [PATCH 72/78] update readme for new py migrate script --- docs/upgrading_to_v2.0.md | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index cb397a0c..0eafbf98 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -125,26 +125,46 @@ actions need to be performed. 1. Download the script ```sh - curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/helpers/migrate.sh - chmod +x migrate.sh + curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/helpers/migrate.py + chmod +x migrate.py ``` 2. Run the script to output the migration commands: ```sh - $ ./migrate.sh --dry-run - terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"] - terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"] + $ ./migrate.py --dry-run + terraform state mv 'module.example.module.test-vpc-module-02.google_compute_network.network[0]' 'module.example.module.test-vpc-module-02.module.vpc.google_compute_network.network' + terraform state mv 'module.example.module.test-vpc-module-02.google_compute_subnetwork.subnetwork' 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork' + terraform state mv 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[0]' 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork["us-west1/multi-vpc-a1-02-subnet-01"]' + terraform state mv 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[1]' 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork["us-west1/multi-vpc-a1-02-subnet-02"]' + terraform state mv 'module.example.module.test-vpc-module-02.google_compute_route.route' 'module.example.module.test-vpc-module-02.module.routes.google_compute_route.route' + terraform state mv 'module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[0]' 'module.example.module.test-vpc-module-02.module.routes.google_compute_route.route["multi-vpc-a1-02-egress-inet"]' + terraform state mv 'module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[1]' 'module.example.module.test-vpc-module-02.module.routes.google_compute_route.route["multi-vpc-a1-02-testapp-proxy"]' + ``` 3. Execute the migration command ```sh - $ ./migrate.sh - Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]" + $ ./migrate.py + ---- Migrating the following modules: + -- module.example.module.test-vpc-module-02 + ---- Commands to run: + Move "module.example.module.test-vpc-module-02.google_compute_network.network[0]" to "module.example.module.test-vpc-module-02.module.vpc.google_compute_network.network" + Successfully moved 1 object(s). + Move "module.example.module.test-vpc-module-02.google_compute_subnetwork.subnetwork" to "module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork" + Successfully moved 1 object(s). + Move "module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[0]" to "module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/multi-vpc-a1-02-subnet-01\"]" + Successfully moved 1 object(s). + Move "module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[1]" to "module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/multi-vpc-a1-02-subnet-02\"]" Successfully moved 1 object(s). - Move "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1]" to "module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]" + Move "module.example.module.test-vpc-module-02.google_compute_route.route" to "module.example.module.test-vpc-module-02.module.routes.google_compute_route.route" Successfully moved 1 object(s). + Move "module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[0]" to "module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[\"multi-vpc-a1-02-egress-inet\"]" + Successfully moved 1 object(s). + Move "module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[1]" to "module.example.module.test-vpc-module-02.module.routes.google_compute_route.route[\"multi-vpc-a1-02-testapp-proxy\"]" + Successfully moved 1 object(s). + ``` 4. Run `terraform plan` to confirm no changes are expected. From 3afde82d6f6519e2cea0ea3dbe614b0f65a2866e Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Wed, 27 Nov 2019 13:33:31 -0800 Subject: [PATCH 73/78] python format --- helpers/migrate.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 62c7ae11..ca3ae89b 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -85,26 +85,28 @@ def moves(self): old.plural = migration.get("old_plural", True) new.plural = migration.get("new_plural", True) - if migration.get("for_each_migration", False) and migration.get("old_plural", True): + if (migration.get("for_each_migration", False) and + migration.get("old_plural", True)): for_each_migrations.append((old, new, migration)) else: pair = (old.path(), new.path()) moves.append(pair) - for_each_moves = self.for_each_moves(for_each_migrations) return moves + for_each_moves def for_each_moves(self, for_each_migrations): """ - When migrating from count to for_each we need to move the whole collection first + When migrating from count to for_each we need to move the + whole collection first https://github.com/hashicorp/terraform/issues/22301 """ for_each_initial_migration = {} moves = [] for (old, new, migration) in for_each_migrations: - # Do the initial migration of the whole collection only once if it hasn't been done yet + # Do the initial migration of the whole collection + # only once if it hasn't been done yet key = old.resource_type + "." + old.name if key not in for_each_initial_migration: for_each_initial_migration[key] = True @@ -117,13 +119,13 @@ def for_each_moves(self, for_each_migrations): # Whole collection is moved to new location. Now needs right index new.plural = True new_indexed = copy.deepcopy(new) - new_indexed.key = self.state.resource_value(old, migration["for_each_migration_key"]) + new_indexed.key = self.state.resource_value( + old, migration["for_each_migration_key"]) pair = (new.path(), new_indexed.path()) moves.append(pair) return moves - def targets(self): """ A list of resources that will be moved to the new module """ @@ -253,15 +255,16 @@ def __repr__(self): self.resource_type, self.name) + class TerraformState: """ A Terraform state representation, pulled from terraform state pull Used for getting values out of individual resources """ + def __init__(self): self.read_state() - def read_state(self): """ Read the terraform state @@ -277,21 +280,26 @@ def read_state(self): def resource_value(self, resource, key): # Find the resource in the state state_resource_list = [r for r in self.state["resources"] if - r["module"] == resource.module and - r["type"] == resource.resource_type and - r["name"] == resource.name ] + r["module"] == resource.module and + r["type"] == resource.resource_type and + r["name"] == resource.name] if (len(state_resource_list) != 1): - raise ValueError("Could not find resource list in state for {}".format(resource)) + raise ValueError( + "Could not find resource list in state for {}" + .format(resource)) index = int(resource.index) - # If this a collection use the index to find the right resource, otherwise use the first + # If this a collection use the index to find the right resource, + # otherwise use the first if (index >= 0): state_resource = [r for r in state_resource_list[0]["instances"] if - r["index_key"] == index] + r["index_key"] == index] if (len(state_resource) != 1): - raise ValueError("Could not find resource in state for {} key {}".format(resource, resource.index)) + raise ValueError( + "Could not find resource in state for {} key {}" + .format(resource, resource.index)) else: state_resource = state_resource_list[0]["instances"] @@ -372,7 +380,7 @@ def migrate(state=None, dryrun=False): # `terraform-google-network` resource type and names modules_to_migrate = [ module for module in modules - if module.has_resource("google_compute_network", "network") + if module.has_resource("google_compute_network", "network") ] print("---- Migrating the following modules:") @@ -401,6 +409,7 @@ def main(argv): migrate(state=state, dryrun=args.dryrun) + def argparser(): parser = argparse.ArgumentParser(description='Migrate Terraform state') parser.add_argument('--dryrun', action='store_true', From 9342a26ab3db411363ecc6c49412f80fc61ccc4d Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 2 Dec 2019 10:47:45 -0800 Subject: [PATCH 74/78] update migration doc to mention issue with single count mvs --- docs/upgrading_to_v2.0.md | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index 0eafbf98..cd3514fb 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -48,7 +48,7 @@ Terraform will perform the following actions: - self_link = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/regions/us-west1/subnetworks/simple-project-timh-subnet-02" -> null } - # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-01"] will be created + # module.example.module.test-vpc-module.google_compute_subnetwork.module.subnets.subnetwork["us-west1/simple-project-timh-subnet-01"] will be created + resource "google_compute_subnetwork" "subnetwork" { + creation_timestamp = (known after apply) + enable_flow_logs = false @@ -65,7 +65,7 @@ Terraform will perform the following actions: + self_link = (known after apply) } - # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"] will be created + # module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"] will be created + resource "google_compute_subnetwork" "subnetwork" { + creation_timestamp = (known after apply) + enable_flow_logs = true @@ -93,11 +93,17 @@ can't guarantee that exactly these actions will be performed if ### Manual Migration Steps -In this example here are the two commands used migrate the subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. You may also use the migration script. +In this example here are the commands used migrate the vpc and subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. You may also use the migration script. -- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` +- `terraform state mv module.example.module.test-vpc-module.google_compute_network.network module.example.module.test-vpc-module.module.vpc.google_compute_subnetwork.network` -- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]` +- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork` + +- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` + +- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]` + +*You'll notice that because of a terraform [issue](https://github.com/hashicorp/terraform/issues/22301), we need to move the whole resource collection first before renaming to the `for_each` keys* `terraform plan` should now return a no-op and show no new changes. @@ -168,3 +174,23 @@ actions need to be performed. ``` 4. Run `terraform plan` to confirm no changes are expected. + +### Known Issues + +If your previous state only contains a **single** subnet or route then `terraform mv` will throw an error similar to the following during migration: + +``` +Error: Invalid target address + +Cannot move to +module.example.module.test-vpc-module-01.module.routes.google_compute_route.route["multi-vpc-a1-01-egress-inet"]: +module.example.module.test-vpc-module-01.module.routes.google_compute_route.route +does not exist in the current state. +``` + +This is due to a terraform mv [issue](https://github.com/hashicorp/terraform/issues/22301) + +The workaround is to either + +1. Create a temporary subnet or route prior to migration +2. Manually updating the state file. Update the `index_key` of the appropriate user and push the to the remote state if necessary. From ecdfe25430d0ba286615a80dd4efd30b7fd2a1f2 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 6 Dec 2019 18:46:54 -0500 Subject: [PATCH 75/78] Remove unused locals from main.tf --- main.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/main.tf b/main.tf index 7544fefa..93794145 100644 --- a/main.tf +++ b/main.tf @@ -14,11 +14,6 @@ * limitations under the License. */ -locals { - network_self_link = var.create_network ? google_compute_network.network[0].self_link : data.google_compute_network.network[0].self_link - network_name = var.create_network ? google_compute_network.network[0].name : data.google_compute_network.network[0].name -} - /****************************************** VPC configuration *****************************************/ From 1a0602db53964ee73f2f5ee15389c658b53e6997 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 6 Dec 2019 19:06:31 -0500 Subject: [PATCH 76/78] Slightly rework upgrade guide --- docs/upgrading_to_v2.0.md | 168 ++++++++++++-------------------------- 1 file changed, 53 insertions(+), 115 deletions(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index cd3514fb..90965fa0 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -7,134 +7,37 @@ Because v2.x changed how the subnet resource is iterated on, resources in Terraf ## Migration Instructions -- Upgrade to the new version of this module. +First, upgrade to the new version of this module. -if you run `terraform plan` at this point. Terraform will inform you that it will attempt to delete and recreate your existing subnets. This is almost certainly not the behavior you want. For example: +```diff + module "kubernetes_engine_private_cluster" { + source = "terraform-google-modules/network/google" +- version = "~> 1.5" ++ version = "~> 2.0" -```Shell -Terraform will perform the following actions: - - # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork will be destroyed - - resource "google_compute_subnetwork" "subnetwork" { - - creation_timestamp = "2019-10-02T08:40:26.282-07:00" -> null - - enable_flow_logs = false -> null - - fingerprint = "f8LZx006zY4=" -> null - - gateway_address = "10.10.10.1" -> null - - id = "us-west1/simple-project-timh-subnet-01" -> null - - ip_cidr_range = "10.10.10.0/24" -> null - - name = "simple-project-timh-subnet-01" -> null - - network = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/global/networks/simple-project-timh" -> null - - private_ip_google_access = false -> null - - project = "dev-xpn-networking" -> null - - region = "us-west1" -> null - - secondary_ip_range = [] -> null - - self_link = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/regions/us-west1/subnetworks/simple-project-timh-subnet-01" -> null - } - - # module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork[1] will be destroyed - - resource "google_compute_subnetwork" "subnetwork" { - - creation_timestamp = "2019-10-02T08:40:26.292-07:00" -> null - - enable_flow_logs = true -> null - - fingerprint = "wOwStN9lK-Q=" -> null - - gateway_address = "10.10.20.1" -> null - - id = "us-west1/simple-project-timh-subnet-02" -> null - - ip_cidr_range = "10.10.20.0/24" -> null - - name = "simple-project-timh-subnet-02" -> null - - network = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/global/networks/simple-project-timh" -> null - - private_ip_google_access = true -> null - - project = "dev-xpn-networking" -> null - - region = "us-west1" -> null - - secondary_ip_range = [] -> null - - self_link = "https://www.googleapis.com/compute/v1/projects/dev-xpn-networking/regions/us-west1/subnetworks/simple-project-timh-subnet-02" -> null - } - - # module.example.module.test-vpc-module.google_compute_subnetwork.module.subnets.subnetwork["us-west1/simple-project-timh-subnet-01"] will be created - + resource "google_compute_subnetwork" "subnetwork" { - + creation_timestamp = (known after apply) - + enable_flow_logs = false - + fingerprint = (known after apply) - + gateway_address = (known after apply) - + id = (known after apply) - + ip_cidr_range = "10.10.10.0/24" - + name = "simple-project-timh-subnet-01" - + network = "simple-project-timh" - + private_ip_google_access = false - + project = "dev-xpn-networking" - + region = "us-west1" - + secondary_ip_range = [] - + self_link = (known after apply) - } - - # module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"] will be created - + resource "google_compute_subnetwork" "subnetwork" { - + creation_timestamp = (known after apply) - + enable_flow_logs = true - + fingerprint = (known after apply) - + gateway_address = (known after apply) - + id = (known after apply) - + ip_cidr_range = "10.10.20.0/24" - + name = "simple-project-timh-subnet-02" - + network = "simple-project-timh" - + private_ip_google_access = true - + project = "dev-xpn-networking" - + region = "us-west1" - + secondary_ip_range = [] - + self_link = (known after apply) - } - -Plan: 2 to add, 0 to change, 2 to destroy. - ------------------------------------------------------------------------- - -Note: You didn't specify an "-out" parameter to save this plan, so Terraform -can't guarantee that exactly these actions will be performed if -"terraform apply" is subsequently run. + # ... + } ``` -### Manual Migration Steps - -In this example here are the commands used migrate the vpc and subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. You may also use the migration script. - -- `terraform state mv module.example.module.test-vpc-module.google_compute_network.network module.example.module.test-vpc-module.module.vpc.google_compute_subnetwork.network` - -- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork` - -- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` - -- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]` - -*You'll notice that because of a terraform [issue](https://github.com/hashicorp/terraform/issues/22301), we need to move the whole resource collection first before renaming to the `for_each` keys* - -`terraform plan` should now return a no-op and show no new changes. - -```Shell -$ terraform plan -Refreshing Terraform state in-memory prior to plan... -The refreshed state will be used to calculate this plan, but will not be -persisted to local or remote state storage. +If you run `terraform plan` at this point, Terraform will inform you that it will attempt to delete and recreate your existing subnets. This is almost certainly not the behavior you want. -module.example.module.test-vpc-module.google_compute_network.network: Refreshing state... [id=simple-project-timh] -module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-02] -module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-01"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-01] - ------------------------------------------------------------------------- - -No changes. Infrastructure is up-to-date. - -This means that Terraform did not detect any differences between your -configuration and real physical resources that exist. As a result, no -actions need to be performed. -``` +You will need to migrate your state, either [manually](#manual-migration-steps) or [automatically](#migration-script). ### Migration Script -1. Download the script +1. Download the script: ```sh curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-network/master/helpers/migrate.py chmod +x migrate.py ``` +2. Back up your Terraform state: + + ```sh + terraform state pull >> state.bak + ``` + 2. Run the script to output the migration commands: ```sh @@ -149,7 +52,7 @@ actions need to be performed. ``` -3. Execute the migration command +3. Execute the migration script: ```sh $ ./migrate.py @@ -175,6 +78,41 @@ actions need to be performed. 4. Run `terraform plan` to confirm no changes are expected. +### Manual Migration Steps + +In this example here are the commands used migrate the vpc and subnets created by the `simple_project` in the examples directory. _please note the need to escape the quotes on the new resource_. You may also use the migration script. + +- `terraform state mv module.example.module.test-vpc-module.google_compute_network.network module.example.module.test-vpc-module.module.vpc.google_compute_subnetwork.network` + +- `terraform state mv module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork` + +- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[0] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-01\"]` + +- `terraform state mv module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[1] module.example.module.test-vpc-module.module.subnets.google_compute_subnetwork.subnetwork[\"us-west1/simple-project-timh-subnet-02\"]` + +*You'll notice that because of a terraform [issue](https://github.com/hashicorp/terraform/issues/22301), we need to move the whole resource collection first before renaming to the `for_each` keys* + +`terraform plan` should now return a no-op and show no new changes. + +```Shell +$ terraform plan +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +module.example.module.test-vpc-module.google_compute_network.network: Refreshing state... [id=simple-project-timh] +module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-02"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-02] +module.example.module.test-vpc-module.google_compute_subnetwork.subnetwork["us-west1/simple-project-timh-subnet-01"]: Refreshing state... [id=us-west1/simple-project-timh-subnet-01] + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +``` + ### Known Issues If your previous state only contains a **single** subnet or route then `terraform mv` will throw an error similar to the following during migration: From b51a1e578b710af1220ba2a81da168d2d22b32c9 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 6 Dec 2019 19:06:50 -0500 Subject: [PATCH 77/78] Small bugfixes to upgrade script --- helpers/migrate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index ca3ae89b..0e158f73 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -31,7 +31,8 @@ { "resource_type": "google_compute_shared_vpc_host_project", "name": "shared_vpc_host", - "module": ".module.vpc" + "module": ".module.vpc", + "new_plural": False }, { "resource_type": "google_compute_subnetwork", @@ -280,7 +281,7 @@ def read_state(self): def resource_value(self, resource, key): # Find the resource in the state state_resource_list = [r for r in self.state["resources"] if - r["module"] == resource.module and + r.get("module", "none") == resource.module and r["type"] == resource.resource_type and r["name"] == resource.name] From f4a98d12be1033338f58f8acf47de737f115bb57 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 6 Dec 2019 19:09:05 -0500 Subject: [PATCH 78/78] Fix dryrun command in docs --- docs/upgrading_to_v2.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrading_to_v2.0.md b/docs/upgrading_to_v2.0.md index 90965fa0..14ae83c4 100644 --- a/docs/upgrading_to_v2.0.md +++ b/docs/upgrading_to_v2.0.md @@ -41,7 +41,7 @@ You will need to migrate your state, either [manually](#manual-migration-steps) 2. Run the script to output the migration commands: ```sh - $ ./migrate.py --dry-run + $ ./migrate.py --dryrun terraform state mv 'module.example.module.test-vpc-module-02.google_compute_network.network[0]' 'module.example.module.test-vpc-module-02.module.vpc.google_compute_network.network' terraform state mv 'module.example.module.test-vpc-module-02.google_compute_subnetwork.subnetwork' 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork' terraform state mv 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork[0]' 'module.example.module.test-vpc-module-02.module.subnets.google_compute_subnetwork.subnetwork["us-west1/multi-vpc-a1-02-subnet-01"]'