diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 8a22a3dbeb..981d12e23c 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -369,17 +369,17 @@ steps: waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage apply --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestAll/examples/simple_zonal_with_asm --stage apply --verbose --test-dir test/integration'] - id: verify simple-zonal-with-asm-local waitFor: - apply simple-zonal-with-asm-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage verify --verbose'] -- id: destroy simple-zonal-with-asm-local + args: ['/bin/bash', '-c', 'cft test run TestAll/examples/simple_zonal_with_asm --stage verify --verbose --test-dir test/integration'] +- id: teardown simple-zonal-with-asm-local waitFor: - verify simple-zonal-with-asm-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage teardown --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestAll/examples/simple_zonal_with_asm --stage teardown --verbose --test-dir test/integration'] - id: apply simple-autopilot-private-local waitFor: - create-all diff --git a/docs/upgrading_to_v35.0.md b/docs/upgrading_to_v35.0.md index bd55288bb4..9a2ed8b69a 100644 --- a/docs/upgrading_to_v35.0.md +++ b/docs/upgrading_to_v35.0.md @@ -50,3 +50,41 @@ To avoid this, it is possible to edit the remote state of the `random_id` resour 1. Bump the serial number at the top 2. Push the modified state to the remote `terraform state push default.tfstate` 3. Confirm the `random_id` resource(s) no longer changes (or the corresponding `nodepool`) in a `terraform plan` + +### ASM Sub-Module Removal +The ASM Sub-Module has been removed in v35.0. Please use the [google_gke_hub_feature](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature#example-usage---enable-fleet-default-member-config-service-mesh) and [google_gke_hub_feature_membership](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#example-usage---service-mesh) resources. + + +```diff +-module "asm" { +- source = "terraform-google-modules/kubernetes-engine/google//modules/asm" +- version = "~> 34.0" + +- project_id = var.project_id +- cluster_name = module.gke.name +- cluster_location = module.gke.location +- multicluster_mode = "connected" +- enable_cni = true +- enable_fleet_registration = true +- enable_mesh_feature = true +-} + ++resource "google_gke_hub_feature" "mesh_feature" { ++ project = var.project_id ++ location = "global" ++ name = "servicemesh" ++} + ++resource "google_gke_hub_feature_membership" "mesh_feature_membership" { ++ project = var.project_id ++ location = "global" + ++ feature = google_gke_hub_feature.mesh_feature.name ++ membership = module.gke.fleet_membership ++ membership_location = module.gke.region + ++ mesh { ++ management = "MANAGEMENT_AUTOMATIC" ++ } ++} +``` diff --git a/examples/simple_zonal_with_asm/README.md b/examples/simple_zonal_with_asm/README.md index 1019aa2404..33d4832be1 100644 --- a/examples/simple_zonal_with_asm/README.md +++ b/examples/simple_zonal_with_asm/README.md @@ -7,32 +7,25 @@ This example illustrates how to create a simple zonal cluster with ASM. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | -| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | -| network | The VPC network to host the cluster in | `any` | n/a | yes | -| project\_id | The project ID to host the cluster in | `any` | n/a | yes | -| region | The region to host the cluster in | `any` | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | -| zones | The zone to host the cluster in (required if is a zonal cluster) | `list(string)` | n/a | yes | +| enable\_fleet\_feature | Whether to enable the Mesh feature on the fleet. | `bool` | `true` | no | +| mesh\_management | ASM Management mode. For more information, see the [gke\_hub\_feature\_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `"MANAGEMENT_AUTOMATIC"` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | `"us-central1"` | no | +| zone | The zone to host the cluster in (required if is a zonal cluster) | `string` | `"us-central1-a"` | no | ## Outputs | Name | Description | |------|-------------| -| ca\_certificate | n/a | -| client\_token | n/a | | cluster\_name | Cluster name | -| identity\_namespace | n/a | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | n/a | -| location | n/a | -| master\_kubernetes\_version | The master Kubernetes version | -| network | n/a | -| project\_id | n/a | -| region | n/a | +| location | Cluster Location | +| network | Network name | +| project\_id | Project ID | +| region | Cluster Region | | service\_account | The default service account used for running nodes. | -| subnetwork | n/a | +| subnetwork | Subnetwork name | | zones | List of zones in which the cluster resides | diff --git a/test/fixtures/simple_zonal_with_asm/variables.tf b/examples/simple_zonal_with_asm/asm.tf similarity index 50% rename from test/fixtures/simple_zonal_with_asm/variables.tf rename to examples/simple_zonal_with_asm/asm.tf index a1961c0abb..f2a455b86d 100644 --- a/test/fixtures/simple_zonal_with_asm/variables.tf +++ b/examples/simple_zonal_with_asm/asm.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,19 +14,27 @@ * limitations under the License. */ -variable "project_ids" { - type = list(string) - description = "The GCP projects to use for integration tests" -} +resource "google_gke_hub_feature" "mesh_feature" { + name = "servicemesh" + project = var.project_id + location = "global" -variable "region" { - description = "The GCP region to create and test resources in" - default = "us-central1" + count = var.enable_fleet_feature ? 1 : 0 } -variable "zones" { - type = list(string) - description = "The GCP zones to create and test resources in, for applicable tests" - default = ["us-central1-a", "us-central1-b", "us-central1-c"] -} +resource "google_gke_hub_feature_membership" "mesh_feature_membership" { + project = var.project_id + location = "global" + + feature = "servicemesh" + membership = module.gke.fleet_membership + membership_location = module.gke.region + mesh { + management = var.mesh_management + } + + depends_on = [ + google_gke_hub_feature.mesh_feature + ] +} diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf index 039125ef6e..f1d38fc9d9 100644 --- a/examples/simple_zonal_with_asm/main.tf +++ b/examples/simple_zonal_with_asm/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,36 +14,24 @@ * limitations under the License. */ -data "google_client_config" "default" {} - -provider "kubernetes" { - host = "https://${module.gke.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(module.gke.ca_certificate) -} - -data "google_project" "project" { - project_id = var.project_id -} - module "gke" { source = "terraform-google-modules/kubernetes-engine/google" version = "~> 34.0" - project_id = var.project_id - name = "test-prefix-cluster-test-suffix" - regional = false - region = var.region - zones = var.zones - release_channel = "REGULAR" - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - network_policy = false - cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" } - identity_namespace = "${var.project_id}.svc.id.goog" - deletion_protection = false + project_id = var.project_id + fleet_project = var.project_id + name = "test-prefix-cluster-test-suffix" + regional = false + region = var.region + zones = [var.zone] + release_channel = "REGULAR" + + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + + deletion_protection = false node_pools = [ { name = "asm-node-pool" @@ -54,17 +42,3 @@ module "gke" { }, ] } - -module "asm" { - source = "terraform-google-modules/kubernetes-engine/google//modules/asm" - version = "~> 34.0" - - project_id = var.project_id - cluster_name = module.gke.name - cluster_location = module.gke.location - multicluster_mode = "connected" - enable_cni = true - enable_fleet_registration = true - enable_mesh_feature = true - -} diff --git a/test/fixtures/simple_zonal_with_asm/network.tf b/examples/simple_zonal_with_asm/network.tf similarity index 92% rename from test/fixtures/simple_zonal_with_asm/network.tf rename to examples/simple_zonal_with_asm/network.tf index 0b538b4b58..2f0c8ab53d 100644 --- a/test/fixtures/simple_zonal_with_asm/network.tf +++ b/examples/simple_zonal_with_asm/network.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2021-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,16 +20,14 @@ resource "random_string" "suffix" { upper = false } -provider "google" { - project = var.project_ids[2] -} - resource "google_compute_network" "main" { + project = var.project_id name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { + project = var.project_id name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/examples/simple_zonal_with_asm/outputs.tf b/examples/simple_zonal_with_asm/outputs.tf index 1e9b7913da..a47c8b8ce2 100644 --- a/examples/simple_zonal_with_asm/outputs.tf +++ b/examples/simple_zonal_with_asm/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,53 @@ * limitations under the License. */ -output "kubernetes_endpoint" { - sensitive = true - value = module.gke.endpoint +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account } -output "client_token" { - sensitive = true - value = base64encode(data.google_client_config.default.access_token) +# Standard test outputs +output "project_id" { + description = "Project ID" + value = var.project_id } -output "ca_certificate" { - sensitive = true - value = module.gke.ca_certificate +output "region" { + description = " Cluster Region" + value = module.gke.region } -output "service_account" { - description = "The default service account used for running nodes." - value = module.gke.service_account +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network" { + description = "Network name" + value = google_compute_network.main.name +} + +output "subnetwork" { + description = "Subnetwork name" + value = google_compute_subnetwork.main.name +} + +output "location" { + description = "Cluster Location" + value = module.gke.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones } diff --git a/examples/simple_zonal_with_asm/test_outputs.tf b/examples/simple_zonal_with_asm/test_outputs.tf deleted file mode 100644 index 71e5965e05..0000000000 --- a/examples/simple_zonal_with_asm/test_outputs.tf +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// These outputs are used to test the module with kitchen-terraform -// They do not need to be included in real-world uses of this module - -output "project_id" { - value = var.project_id -} - -output "region" { - value = module.gke.region -} - -output "cluster_name" { - description = "Cluster name" - value = module.gke.name -} - -output "network" { - value = var.network -} - -output "subnetwork" { - value = var.subnetwork -} - -output "location" { - value = module.gke.location -} - -output "ip_range_pods" { - description = "The secondary IP range used for pods" - value = var.ip_range_pods -} - -output "ip_range_services" { - description = "The secondary IP range used for services" - value = var.ip_range_services -} - -output "zones" { - description = "List of zones in which the cluster resides" - value = module.gke.zones -} - -output "master_kubernetes_version" { - description = "The master Kubernetes version" - value = module.gke.master_version -} - -output "identity_namespace" { - value = module.gke.identity_namespace -} diff --git a/examples/simple_zonal_with_asm/variables.tf b/examples/simple_zonal_with_asm/variables.tf index 82b0cd184b..c362507718 100644 --- a/examples/simple_zonal_with_asm/variables.tf +++ b/examples/simple_zonal_with_asm/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,29 +16,38 @@ variable "project_id" { description = "The project ID to host the cluster in" + type = string } variable "region" { description = "The region to host the cluster in" + type = string + default = "us-central1" } -variable "zones" { - type = list(string) +variable "zone" { description = "The zone to host the cluster in (required if is a zonal cluster)" + type = string + default = "us-central1-a" } -variable "network" { - description = "The VPC network to host the cluster in" +variable "enable_fleet_feature" { + description = "Whether to enable the Mesh feature on the fleet." + type = bool + default = true } -variable "subnetwork" { - description = "The subnetwork to host the cluster in" -} - -variable "ip_range_pods" { - description = "The secondary ip range to use for pods" -} - -variable "ip_range_services" { - description = "The secondary ip range to use for services" +variable "mesh_management" { + default = "MANAGEMENT_AUTOMATIC" + description = "ASM Management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh)" + type = string + validation { + condition = anytrue([ + var.mesh_management == null, + var.mesh_management == "", + var.mesh_management == "MANAGEMENT_AUTOMATIC", + var.mesh_management == "MANAGEMENT_MANUAL", + ]) + error_message = "Must be null, empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL." + } } diff --git a/examples/simple_zonal_with_asm/versions.tf b/examples/simple_zonal_with_asm/versions.tf index 6dfcbcb74a..7f287495c1 100644 --- a/examples/simple_zonal_with_asm/versions.tf +++ b/examples/simple_zonal_with_asm/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ terraform { - required_version = ">= 0.13" + required_version = ">= 1.3" required_providers { google = { source = "hashicorp/google" diff --git a/modules/asm/README.md b/modules/asm/README.md deleted file mode 100644 index 5b3e8f5ead..0000000000 --- a/modules/asm/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Terraform Kubernetes Engine ASM Submodule - -This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes Engine (GKE) cluster. - -> [!WARNING] -> This Terraform Kubernetes Engine ASM Submodule has been deprecated in release v34.0.0 and will be removed entirely in a future release. It is recommended to migrate to the [google_gke_hub_feature](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature#example-usage---enable-fleet-default-member-config-service-mesh) and [google_gke_hub_feature_membership](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#example-usage---service-mesh) resources. - -## Usage - -The ASM module requires Terraform 0.14+. There are a few prerequisites to using this module that can be done either through Terraform and/or manually: - -1. Enable the `mesh.googleapis.com` service -1. [Enable](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature#example-usage---gkehub-feature-anthos-service-mesh) the GKEhub `servicemesh` feature -1. [Register](https://cloud.google.com/anthos/fleet-management/docs/register/gke#terraform) target cluster to the servicemesh-enabled hub -1. Configure the [Kubernetes Provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs) for the target cluster, for example: - -```tf -provider "kubernetes" { - host = "https://${module.gke.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(module.gke.ca_certificate) -} - -data "google_client_config" "default" {} -``` - -There is a full example provided [here](../../examples/simple_zonal_with_asm). Detailed usage example is as follows: - -```tf -module "asm" { - source = "terraform-google-modules/kubernetes-engine/google//modules/asm" - project_id = var.project_id - cluster_name = module.gke.name - cluster_location = module.gke.location - enable_cni = true -} -``` - -Note that the [`mesh_id` label on the cluster](https://cloud.google.com/service-mesh/docs/managed/auto-control-plane-with-fleet#apply_the_mesh_id_label) is required for metrics to get displayed on the Anthos Service Mesh pages in the Cloud console (Topology, etc.). Illustrated with the full example mentioned above, here is an example of what your cluster should have: - -```tf -module "gke" { -... - cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" } -... -} -``` - -To deploy this config: - -1. Run `terraform apply` - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| channel | The channel to use for this ASM installation. | `string` | `""` | no | -| cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes | -| cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes | -| create\_system\_namespace | Determines whether the module creates the istio-system namespace. | `bool` | `true` | no | -| enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no | -| enable\_fleet\_registration | Determines whether the module registers the cluster to the fleet. | `bool` | `false` | no | -| enable\_mesh\_feature | Determines whether the module enables the mesh feature on the fleet. | `bool` | `false` | no | -| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc) | `bool` | `false` | no | -| fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no | -| internal\_ip | Use internal ip for the cluster endpoint when running kubectl commands. | `bool` | `false` | no | -| mesh\_management | ASM Management mode. For more information, see the [gke\_hub\_feature\_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `""` | no | -| module\_depends\_on | List of modules or resources this module depends on. If multiple, all items must be the same type. | `list(any)` | `[]` | no | -| multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no | -| project\_id | The project in which the resource belongs. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| revision\_name | The name of the installed managed ASM revision. | -| wait | An output to use when depending on the ASM installation finishing. | - - diff --git a/modules/asm/hub.tf b/modules/asm/hub.tf deleted file mode 100644 index 15e44cc82f..0000000000 --- a/modules/asm/hub.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_gke_hub_membership" "membership" { - count = var.enable_fleet_registration ? 1 : 0 - provider = google-beta - project = local.fleet_id - membership_id = "${data.google_container_cluster.asm.name}-membership" - endpoint { - gke_cluster { - resource_link = "//container.googleapis.com/${data.google_container_cluster.asm.id}" - } - } -} - -resource "google_gke_hub_feature" "mesh" { - count = var.enable_mesh_feature ? 1 : 0 - name = "servicemesh" - project = local.fleet_id - location = "global" - provider = google-beta -} - -resource "google_gke_hub_feature_membership" "mesh_feature_membership" { - count = var.enable_fleet_registration && var.enable_mesh_feature && var.mesh_management != "" ? 1 : 0 - - project = local.fleet_id - location = "global" - feature = google_gke_hub_feature.mesh[0].name - membership = google_gke_hub_membership.membership[0].membership_id - mesh { - management = var.mesh_management - } - provider = google-beta -} diff --git a/modules/asm/main.tf b/modules/asm/main.tf deleted file mode 100644 index 666621c816..0000000000 --- a/modules/asm/main.tf +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954 - gke_release_channel = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : "" - gke_release_channel_filtered = lower(local.gke_release_channel) == "unspecified" ? "" : local.gke_release_channel - // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel - channel = lower(coalesce(var.channel, local.gke_release_channel_filtered, "regular")) - revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}" - // Fleet ID should default to project ID if unset - fleet_id = coalesce(var.fleet_id, var.project_id) -} - -data "google_container_cluster" "asm" { - project = var.project_id - name = var.cluster_name - location = var.cluster_location - - depends_on = [var.module_depends_on] -} - -resource "kubernetes_namespace" "system" { - count = var.create_system_namespace && var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 - - metadata { - name = "istio-system" - } -} - -resource "kubernetes_config_map" "asm_options" { - count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 - - metadata { - name = "asm-options" - namespace = try(kubernetes_namespace.system[0].metadata[0].name, "istio-system") - } - - data = { - multicluster_mode = var.multicluster_mode - ASM_OPTS = var.enable_cni ? "CNI=on" : null - } - - depends_on = [google_gke_hub_membership.membership, google_gke_hub_feature.mesh, var.module_depends_on] -} - -module "cpr" { - count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 - - source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" - version = "~> 3.1" - - project_id = var.project_id - cluster_name = var.cluster_name - cluster_location = var.cluster_location - internal_ip = var.internal_ip - - kubectl_create_command = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${var.enable_cni} ${var.enable_vpc_sc}" - kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}" - - module_depends_on = [kubernetes_config_map.asm_options] -} - -# Wait for the ControlPlaneRevision custom resource to be ready. -# Add an explicit "retry until the resource is created" until -module "kubectl_asm_wait_for_controlplanerevision_custom_resource_definition" { - count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0 - - source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" - version = "~> 3.1" - - project_id = var.project_id - cluster_name = var.cluster_name - cluster_location = var.cluster_location - kubectl_create_command = "/bin/sh -c 'while ! kubectl wait crd/controlplanerevisions.mesh.cloud.google.com --for condition=established --timeout=60m --all-namespaces; do echo \"crd/controlplanerevisions.mesh.cloud.google.com not yet available, waiting...\"; sleep 5; done'" - kubectl_destroy_command = "" - - module_depends_on = [ - google_gke_hub_feature_membership.mesh_feature_membership - ] -} - -# Wait for the ASM control plane revision to be ready so we can safely deploy resources that depend -# on ASM mutating webhooks. -# Add an explicit "retry until the resource is created" until -module "kubectl_asm_wait_for_controlplanerevision" { - count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0 - - source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" - version = "~> 3.1" - - project_id = var.project_id - cluster_name = var.cluster_name - cluster_location = var.cluster_location - kubectl_create_command = "/bin/sh -c 'while ! kubectl -n istio-system wait ControlPlaneRevision --all --timeout=60m --for condition=Reconciled; do echo \"ControlPlaneRevision not yet available, waiting...\"; sleep 5; done'" - kubectl_destroy_command = "" - - module_depends_on = [ - module.kubectl_asm_wait_for_controlplanerevision_custom_resource_definition[0].wait - ] -} diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf deleted file mode 100644 index 936f148ff2..0000000000 --- a/modules/asm/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "revision_name" { - value = local.revision_name - description = "The name of the installed managed ASM revision." -} - -output "wait" { - value = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? module.kubectl_asm_wait_for_controlplanerevision[0].wait : module.cpr[0].wait - description = "An output to use when depending on the ASM installation finishing." -} diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh deleted file mode 100755 index 9ceb332312..0000000000 --- a/modules/asm/scripts/create_cpr.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ "$#" -lt 4 ]; then - >&2 echo "Not all expected arguments set." - exit 1 -fi - -REVISION_NAME=$1; shift -CHANNEL=$1; shift -ENABLE_CNI=$1; shift -ENABLE_VPC_SC=$1; shift - -# Wait for the CRD to get created before creating the CPR. Not possible to use `kubectl --wait ...` here since -# the CRD won't exist at the time of checking (https://stackoverflow.com/questions/57115602/how-to-kubectl-wait-for-crd-creation) -readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com -for _i in {1..18}; do - echo "Ensuring ControlPlaneRevision exists in cluster... attempt ${_i}" - if kubectl get crd "${CPR_RESOURCE}" - then - break - else - sleep 10 - fi -done - -kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}" - -cat <&2 echo "Not all expected arguments set." - exit 1 -fi - -REVISION_NAME=$1; shift - -if ! kubectl delete controlplanerevision -n istio-system "${REVISION_NAME}" ; then - echo "ControlPlaneRevision ${REVISION_NAME} not found" -fi diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf deleted file mode 100644 index f5a0cb6941..0000000000 --- a/modules/asm/variables.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project in which the resource belongs." - type = string -} - -variable "cluster_name" { - description = "The unique name to identify the cluster in ASM." - type = string -} - -variable "cluster_location" { - description = "The cluster location for this ASM installation." - type = string -} - -variable "fleet_id" { - description = "The fleet to use for this ASM installation." - type = string - default = "" -} - -variable "channel" { - description = "The channel to use for this ASM installation." - type = string - validation { - condition = anytrue([ - var.channel == "rapid", - var.channel == "regular", - var.channel == "stable", - var.channel == "", // if unset, use GKE data source and use release cluster channel - ]) - error_message = "Must be one of rapid, regular, or stable." - } - default = "" -} - -variable "mesh_management" { - default = "" - description = "ASM Management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh)" - type = string - validation { - condition = anytrue([ - var.mesh_management == null, - var.mesh_management == "", - var.mesh_management == "MANAGEMENT_AUTOMATIC", - var.mesh_management == "MANAGEMENT_MANUAL", - ]) - error_message = "Must be null, empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL." - } -} - -variable "multicluster_mode" { - description = "[Preview] Determines whether remote secrets should be autogenerated across fleet cluster." - type = string - validation { - condition = anytrue([ - var.multicluster_mode == "manual", - var.multicluster_mode == "connected", - ]) - error_message = "Must be one of manual or connected." - } - default = "manual" -} - -variable "enable_cni" { - description = "Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP)." - type = bool - default = false -} - -variable "enable_vpc_sc" { - description = "Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc)" - type = bool - default = false -} - -variable "enable_fleet_registration" { - description = "Determines whether the module registers the cluster to the fleet." - type = bool - default = false -} - -variable "enable_mesh_feature" { - description = "Determines whether the module enables the mesh feature on the fleet." - type = bool - default = false -} - -variable "internal_ip" { - description = "Use internal ip for the cluster endpoint when running kubectl commands." - type = bool - default = false -} - -variable "module_depends_on" { - description = "List of modules or resources this module depends on. If multiple, all items must be the same type." - type = list(any) - default = [] -} - -variable "create_system_namespace" { - description = "Determines whether the module creates the istio-system namespace." - type = bool - default = true -} diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf deleted file mode 100644 index 1227961368..0000000000 --- a/modules/asm/versions.tf +++ /dev/null @@ -1,45 +0,0 @@ - -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.1" - - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.0" - } - google = { - source = "hashicorp/google" - # Avoid v4.49 and v4.50 for https://github.com/hashicorp/terraform-provider-google/issues/13507 - version = ">= 4.47.0, != 4.49.0, != 4.50.0, < 7" - } - google-beta = { - source = "hashicorp/google-beta" - # Avoid v4.49 and v4.50 for https://github.com/hashicorp/terraform-provider-google/issues/13507 - version = ">= 4.47.0, != 4.49.0, != 4.50.0, < 7" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v34.0.0" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v34.0.0" - } -} diff --git a/test/fixtures/simple_zonal_with_asm/.gitignore b/test/fixtures/simple_zonal_with_asm/.gitignore deleted file mode 100644 index f78a088f66..0000000000 --- a/test/fixtures/simple_zonal_with_asm/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# asm outdir -simple-zonal-* -# install asm script -install_asm* diff --git a/test/fixtures/simple_zonal_with_asm/example.tf b/test/fixtures/simple_zonal_with_asm/example.tf deleted file mode 100644 index 76b573ada5..0000000000 --- a/test/fixtures/simple_zonal_with_asm/example.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_project" "project" { - project_id = var.project_ids[2] -} - -module "example" { - source = "../../../examples/simple_zonal_with_asm" - - project_id = var.project_ids[2] - region = var.region - zones = slice(var.zones, 0, 1) - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name -} diff --git a/test/fixtures/simple_zonal_with_asm/outputs.tf b/test/fixtures/simple_zonal_with_asm/outputs.tf deleted file mode 100644 index 2a5990df27..0000000000 --- a/test/fixtures/simple_zonal_with_asm/outputs.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "project_id" { - value = module.example.project_id -} - -output "region" { - value = module.example.region -} - -output "cluster_name" { - description = "Cluster name" - value = module.example.cluster_name -} - -output "network" { - value = google_compute_network.main.name -} - -output "subnetwork" { - value = google_compute_subnetwork.main.name -} - -output "location" { - value = module.example.location -} - -output "ip_range_pods" { - description = "The secondary IP range used for pods" - value = google_compute_subnetwork.main.secondary_ip_range[0].range_name -} - -output "ip_range_services" { - description = "The secondary IP range used for services" - value = google_compute_subnetwork.main.secondary_ip_range[1].range_name -} - -output "zones" { - description = "List of zones in which the cluster resides" - value = module.example.zones -} - -output "master_kubernetes_version" { - description = "The master Kubernetes version" - value = module.example.master_kubernetes_version -} - -output "kubernetes_endpoint" { - sensitive = true - value = module.example.kubernetes_endpoint -} - -output "client_token" { - sensitive = true - value = module.example.client_token -} - -output "ca_certificate" { - description = "The cluster CA certificate" - value = module.example.ca_certificate - sensitive = true -} - -output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." - value = module.example.service_account -} - -output "project_number" { - description = "The project number of the ASM project" - value = data.google_project.project.number -} diff --git a/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go b/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go deleted file mode 100644 index 33791e9fa7..0000000000 --- a/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2022-2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package simple_zonal_with_asm - -import ( - "fmt" - "testing" - "time" - - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" - "github.com/gruntwork-io/terratest/modules/k8s" - "github.com/stretchr/testify/assert" - "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" -) - -func TestSimpleZonalWithASM(t *testing.T) { - bpt := tft.NewTFBlueprintTest(t, - tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), - ) - - bpt.DefineVerify(func(assert *assert.Assertions) { - //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token - // bpt.DefaultVerify(assert) - - projectId := bpt.GetStringOutput("project_id") - projectNumber := bpt.GetStringOutput("project_number") - location := bpt.GetStringOutput("location") - clusterName := bpt.GetStringOutput("cluster_name") - - op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) - assert.Contains(op.Get("resourceLabels.mesh_id").String(), fmt.Sprintf("proj-%s", projectNumber), "Mesh ID's exists") - - op1 := gcloud.Runf(t, "container hub memberships describe %s-membership --project %s ", clusterName, projectId) - assert.Contains(op1.Get("endpoint.gkeCluster.resourceLink").String(), fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName), "Custom Terraform Created") - - gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) - k8sOpts := k8s.KubectlOptions{} - listNameSpace, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "ns", "istio-system", "-o", "json") - assert.NoError(err) - kubeNS := utils.ParseKubectlJSONResult(t, listNameSpace) - assert.Contains(kubeNS.Get("metadata.name").String(), "istio-system", "Namespace is Functional") - listConfigMap, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "configmap", "asm-options", "-n", "istio-system", "-o", "json") - assert.NoError(err) - kubeCM := utils.ParseKubectlJSONResult(t, listConfigMap) - assert.Contains(kubeCM.Get("metadata.name").String(), "asm-options", "Configmap is Present") - - }) - - bpt.Test() -} diff --git a/test/setup/iam.tf b/test/setup/iam.tf index fe97685dd8..d98f6f6bba 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -112,6 +112,14 @@ resource "google_project_iam_member" "int_test_fleet" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "int_test_default" { + for_each = toset(local.int_required_roles) + + project = module.gke-project-default.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_service_account_key" "int_test" { service_account_id = google_service_account.int_test.id } diff --git a/test/setup/main.tf b/test/setup/main.tf index b94c404385..6f1626d19f 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2019-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -119,3 +119,17 @@ module "gke-project-fleet" { activate_apis = local.apis } +module "gke-project-default" { + source = "terraform-google-modules/project-factory/google" + version = "~> 17.0" + + name = "ci-gke-default-${random_id.random_project_id_suffix.hex}" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + # due to https://github.com/hashicorp/terraform-provider-google/issues/9505 for AP + default_service_account = "keep" + + activate_apis = local.apis +} diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 54c01df2e7..c9c989fe89 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2019-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,10 @@ output "project_ids" { value = [module.gke-project-1.project_id, module.gke-project-2.project_id, module.gke-project-asm.project_id, module.gke-project-fleet.project_id] } +output "project_id" { + value = module.gke-project-default.project_id +} + output "sa_key" { value = google_service_account_key.int_test.private_key sensitive = true