Merge branch 'GoogleCloudPlatform:main' into patch-1

.ci/magician/github/membership_data.go

@@ -45,11 +45,6 @@ var (
 			startDate: newDate(2024, 4, 30, pdtLoc),
 			endDate:   newDate(2024, 7, 31, pdtLoc),
 		},
-		{
-			id:        "SarahFrench",
-			startDate: newDate(2024, 8, 2, bstLoc),
-			endDate:   newDate(2024, 8, 6, bstLoc),
-		},
 		{
 			id:        "shuyama1",
 			startDate: newDate(2024, 5, 22, pdtLoc),
@@ -95,5 +90,10 @@ var (
 			startDate: newDate(2024, 9, 13, pdtLoc),
 			endDate:   newDate(2024, 9, 20, pdtLoc),
 		},
+		{
+			id:        "SarahFrench",
+			startDate: newDate(2024, 9, 20, bstLoc),
+			endDate:   newDate(2024, 9, 23, bstLoc),
+		},
 	}
 )

mmv1/api/resource.go

@@ -1192,8 +1192,7 @@ func (r Resource) ExtractIdentifiers(url string) []string {
 	return result
 }
 
-// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
-func (r Resource) RawImportIdFormatsFromIam() []string {
+func (r Resource) IamImportFormats() []string {
 	var importFormat []string
 
 	if r.IamPolicy != nil {
@@ -1202,8 +1201,12 @@ func (r Resource) RawImportIdFormatsFromIam() []string {
 	if len(importFormat) == 0 {
 		importFormat = r.ImportFormat
 	}
+	return importFormat
+}
 
-	return ImportIdFormats(importFormat, r.Identity, r.BaseUrl)
+// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
+func (r Resource) RawImportIdFormatsFromIam() []string {
+	return ImportIdFormats(r.IamImportFormats(), r.Identity, r.BaseUrl)
 }
 
 // For example, projects/(?P<project>[^/]+)/schemas/(?P<schema>[^/]+)", "(?P<project>[^/]+)/(?P<schema>[^/]+)", "(?P<schema>[^/]+)
@@ -1668,3 +1671,34 @@ func (r Resource) CaiApiVersion(productBackendName, caiProductBaseUrl string) st
 	}
 	return ""
 }
+
+// For example: the uri "projects/{{project}}/schemas/{{name}}"
+// The paramerter is "schema" as "project" is not returned.
+func (r Resource) CaiIamResourceParams() []string {
+	resourceUri := strings.ReplaceAll(r.IamResourceUri(), "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+
+	return google.Reject(r.ExtractIdentifiers(resourceUri), func(param string) bool {
+		return param == "project"
+	})
+}
+
+// Gets the Cai IAM asset name template
+// For example: //monitoring.googleapis.com/v3/projects/{{project}}/services/{{service_id}}
+func (r Resource) CaiIamAssetNameTemplate(productBackendName string) string {
+	iamImportFormat := r.IamImportFormats()
+	if len(iamImportFormat) > 0 {
+		name := strings.ReplaceAll(iamImportFormat[0], "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+		name = strings.ReplaceAll(name, "%", "")
+		return fmt.Sprintf("//%s.googleapis.com/%s", productBackendName, name)
+	}
+
+	caiBaseUrl := r.CaiBaseUrl
+
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.SelfLink
+	}
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.BaseUrl
+	}
+	return fmt.Sprintf("//%s.googleapis.com/%s/{{%s}}", productBackendName, caiBaseUrl, r.IamParentResourceName())
+}

mmv1/go.mod

@@ -8,3 +8,7 @@ require (
 )
 
 require github.com/golang/glog v1.2.0
+
+require github.com/otiai10/copy v1.9.0
+
+require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect

mmv1/go.sum

@@ -2,8 +2,16 @@ github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
 github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4=
+github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

mmv1/google/slice_utils.go

@@ -39,3 +39,19 @@ func Reject[T any](S []T, test func(T) bool) (ret []T) {
 func Concat[T any](S1 []T, S2 []T) (ret []T) {
 	return append(S1, S2...)
 }
+
+// difference returns the elements in `S1` that aren't in `S2`.
+func Diff(S1, S2 []string) []string {
+	var ret []string
+	mb := make(map[string]bool, len(S2))
+	for _, x := range S2 {
+		mb[x] = true
+	}
+
+	for _, x := range S1 {
+		if _, found := mb[x]; !found {
+			ret = append(ret, x)
+		}
+	}
+	return ret
+}

mmv1/google/template_utils.go

@@ -65,6 +65,7 @@ var TemplateFunctions = template.FuncMap{
 	"join":          strings.Join,
 	"lower":         strings.ToLower,
 	"upper":         strings.ToUpper,
+	"hasSuffix":     strings.HasSuffix,
 	"dict":          wrapMultipleParams,
 	"format2regex":  Format2Regex,
 	"hasPrefix":     strings.HasPrefix,

mmv1/main.go

@@ -149,7 +149,6 @@ func main() {
 	providerToGenerate = setProvider(*forceProvider, *version, productsForVersion[0], startTime)
 	providerToGenerate.CopyCommonFiles(*outputPath, generateCode, generateDocs)
 
-	log.Printf("Compiling common files for terraform")
 	if generateCode {
 		providerToGenerate.CompileCommonFiles(*outputPath, productsForVersion, "")
 
@@ -236,6 +235,8 @@ func setProvider(forceProvider, version string, productApi *api.Product, startTi
 	switch forceProvider {
 	case "tgc":
 		return provider.NewTerraformGoogleConversion(productApi, version, startTime)
+	case "tgc_cai2hcl":
+		return provider.NewCaiToTerraformConversion(productApi, version, startTime)
 	default:
 		return provider.NewTerraform(productApi, version, startTime)
 	}

mmv1/products/bigquerydatapolicy/DataPolicy.yaml

@@ -82,6 +82,7 @@ properties:
       Policy tag resource name, in the format of projects/{project_number}/locations/{locationId}/taxonomies/{taxonomyId}/policyTags/{policyTag_id}.
     required: true
     diff_suppress_func: 'tpgresource.ProjectNumberDiffSuppress'
+    custom_flatten: 'templates/terraform/custom_flatten/bigquery_datapolicy_policy_tag_location_to_lower_case.go.erb'
   - !ruby/object:Api::Type::Enum
     name: dataPolicyType
     description: |

mmv1/products/cloudrun/Service.yaml

@@ -53,6 +53,17 @@ examples:
       cloud_run_service_name: 'cloudrun-srv'
     test_env_vars:
       project: :PROJECT_NAME
+  - !ruby/object:Provider::Terraform::Examples
+    name: 'cloud_run_service_gpu'
+    min_version: 'beta'
+    primary_resource_id: 'default'
+    primary_resource_name: "fmt.Sprintf(\"tf-test-cloudrun-srv%s\",
+      context[\"random_suffix\"\
+      ])"
+    vars:
+      cloud_run_service_name: 'cloudrun-srv'
+    test_env_vars:
+      project: :PROJECT_NAME
   - !ruby/object:Provider::Terraform::Examples
     name: 'cloud_run_service_sql'
     primary_resource_id: 'default'
@@ -743,7 +754,13 @@ properties:
                                 The name of the service to place in the gRPC HealthCheckRequest
                                 (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
                                 If this is not specified, the default behavior is defined by gRPC.
-
+              - !ruby/object:Api::Type::KeyValuePairs
+                name: nodeSelector
+                min_version: beta
+                description: |-
+                  Node Selector describes the hardware requirements of the resources.
+                  Use the following node selector keys to configure features on a Revision:
+                    - `run.googleapis.com/accelerator` sets the [type of GPU](https://cloud.google.com/run/docs/configuring/services/gpu) required by the Revision to run.
               - !ruby/object:Api::Type::Integer
                 name: containerConcurrency
                 description: |-

mmv1/products/cloudrun/go_Service.yaml

@@ -69,6 +69,14 @@ examples:
       cloud_run_service_name: 'cloudrun-srv'
     test_env_vars:
       project: 'PROJECT_NAME'
+  - name: 'cloud_run_service_gpu'
+    primary_resource_id: 'default'
+    primary_resource_name: 'fmt.Sprintf("tf-test-cloudrun-srv%s", context["random_suffix"])'
+    min_version: 'beta'
+    vars:
+      cloud_run_service_name: 'cloudrun-srv'
+    test_env_vars:
+      project: 'PROJECT_NAME'
   - name: 'cloud_run_service_sql'
     primary_resource_id: 'default'
     primary_resource_name: 'fmt.Sprintf("tf-test-cloudrun-srv%s", context["random_suffix"])'
@@ -737,7 +745,13 @@ properties:
                                 The name of the service to place in the gRPC HealthCheckRequest
                                 (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
                                 If this is not specified, the default behavior is defined by gRPC.
-
+              - name: 'nodeSelector'
+                type: KeyValuePairs
+                description: |-
+                  Node Selector describes the hardware requirements of the resources.
+                  Use the following node selector keys to configure features on a Revision:
+                    - `run.googleapis.com/accelerator` sets the [type of GPU](https://cloud.google.com/run/docs/configuring/services/gpu) required by the Revision to run.
+                min_version: 'beta'
               - name: 'containerConcurrency'
                 type: Integer
                 description: |-

mmv1/products/cloudrunv2/Service.yaml

@@ -112,6 +112,15 @@ examples:
       cloud_run_service_name: 'cloudrun-service'
     ignore_read_extra:
       - 'deletion_protection'
+  - !ruby/object:Provider::Terraform::Examples
+    name: 'cloudrunv2_service_gpu'
+    min_version: 'beta'
+    primary_resource_id: 'default'
+    primary_resource_name: "fmt.Sprintf(\"tf-test-cloudrun-srv%s\", context[\"random_suffix\"])"
+    vars:
+      cloud_run_service_name: 'cloudrun-service'
+    ignore_read_extra:
+      - 'deletion_protection'
   - !ruby/object:Provider::Terraform::Examples
     name: 'cloudrunv2_service_probes'
     primary_resource_id: 'default'
@@ -498,7 +507,7 @@ properties:
                 - !ruby/object:Api::Type::KeyValuePairs
                   name: 'limits'
                   description: |-
-                    Only memory and CPU are supported. Use key `cpu` for CPU limit and `memory` for memory limit. Note: The only supported values for CPU are '1', '2', '4', and '8'. Setting 4 CPU requires at least 2Gi of memory. The values of the map is string form of the 'quantity' k8s type: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/api/resource/quantity.go
+                    Only memory, CPU, and nvidia.com/gpu are supported. Use key `cpu` for CPU limit, `memory` for memory limit, `nvidia.com/gpu` for gpu limit. Note: The only supported values for CPU are '1', '2', '4', and '8'. Setting 4 CPU requires at least 2Gi of memory. The values of the map is string form of the 'quantity' k8s type: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/api/resource/quantity.go
                   default_from_api: true
                 - !ruby/object:Api::Type::Boolean
                   name: 'cpuIdle'
@@ -899,6 +908,16 @@ properties:
             name: 'mesh'
             description: |-
               The Mesh resource name. For more information see https://cloud.google.com/service-mesh/docs/reference/network-services/rest/v1/projects.locations.meshes#resource:-mesh.
+      - !ruby/object:Api::Type::NestedObject
+        name: 'nodeSelector'
+        min_version: beta
+        description: Node Selector describes the hardware requirements of the resources.
+        properties:
+          - !ruby/object:Api::Type::String
+            name: 'accelerator'
+            required: true
+            description:
+              The GPU to attach to an instance. See https://cloud.google.com/run/docs/configuring/services/gpu for configuring GPU.
   - !ruby/object:Api::Type::Array
     name: 'traffic'
     description: |-

mmv1/products/cloudrunv2/go_Service.yaml

@@ -100,6 +100,14 @@ examples:
       cloud_run_service_name: 'cloudrun-service'
     ignore_read_extra:
       - 'deletion_protection'
+  - name: 'cloudrunv2_service_gpu'
+    primary_resource_id: 'default'
+    primary_resource_name: 'fmt.Sprintf("tf-test-cloudrun-srv%s", context["random_suffix"])'
+    min_version: 'beta'
+    vars:
+      cloud_run_service_name: 'cloudrun-service'
+    ignore_read_extra:
+      - 'deletion_protection'
   - name: 'cloudrunv2_service_probes'
     primary_resource_id: 'default'
     primary_resource_name: 'fmt.Sprintf("tf-test-cloudrun-srv%s", context["random_suffix"])'
@@ -495,7 +503,7 @@ properties:
                 - name: 'limits'
                   type: KeyValuePairs
                   description: |-
-                    Only memory and CPU are supported. Use key `cpu` for CPU limit and `memory` for memory limit. Note: The only supported values for CPU are '1', '2', '4', and '8'. Setting 4 CPU requires at least 2Gi of memory. The values of the map is string form of the 'quantity' k8s type: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/api/resource/quantity.go
+                    Only memory, CPU, and nvidia.com/gpu are supported. Use key `cpu` for CPU limit, `memory` for memory limit, `nvidia.com/gpu` for gpu limit. Note: The only supported values for CPU are '1', '2', '4', and '8'. Setting 4 CPU requires at least 2Gi of memory. The values of the map is string form of the 'quantity' k8s type: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/api/resource/quantity.go
                   default_from_api: true
                 - name: 'cpuIdle'
                   type: Boolean
@@ -904,6 +912,16 @@ properties:
             type: String
             description: |-
               The Mesh resource name. For more information see https://cloud.google.com/service-mesh/docs/reference/network-services/rest/v1/projects.locations.meshes#resource:-mesh.
+      - name: 'nodeSelector'
+        type: NestedObject
+        description: Node Selector describes the hardware requirements of the resources.
+        min_version: 'beta'
+        properties:
+          - name: 'accelerator'
+            type: String
+            description:
+              The GPU to attach to an instance. See https://cloud.google.com/run/docs/configuring/services/gpu for configuring GPU.
+            required: true
   - name: 'traffic'
     type: Array
     description: |-