diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 3ee402c2..1a881854 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -69,6 +69,21 @@ steps: - go-verify-logbucket-org name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestLogBucketOrgModule --stage teardown --verbose'] +- id: go-apply-logbucket-project + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage apply --verbose'] +- id: go-verify-logbucket-project + waitFor: + - go-apply-logbucket-org + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage verify --verbose'] +- id: go-teardown-logbucket-project + waitFor: + - go-verify-logbucket-project + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md new file mode 100644 index 00000000..e6b569df --- /dev/null +++ b/examples/logbucket/project/README.md @@ -0,0 +1,30 @@ +# Log Export: Log Bucket destination at Project level + +These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project or in separated projects. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| parent\_resource\_project | The ID of the project in which the log export will be created. | `string` | n/a | yes | +| project\_destination\_logbkt\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| log\_bkt\_name\_same\_proj | The name for the log bucket for sink and logbucket in same project example. | +| log\_bkt\_same\_proj | The project where the log bucket is created for sink and logbucket in same project example. | +| log\_bucket\_name | The name for the log bucket. | +| log\_bucket\_project | The project where the log bucket is created. | +| log\_sink\_dest\_uri\_same\_proj | A fully qualified URI for the log sink for sink and logbucket in same project example. | +| log\_sink\_destination\_uri | A fully qualified URI for the log sink. | +| log\_sink\_id\_same\_proj | The project id where the log sink is created for sink and logbucket in same project example. | +| log\_sink\_project\_id | The project id where the log sink is created. | +| log\_sink\_resource\_name | The resource name of the log sink that was created. | +| log\_sink\_resource\_name\_same\_proj | The resource name of the log sink that was created in same project example. | +| log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. | +| log\_sink\_writer\_identity\_same\_proj | The service account in same project example that logging uses to write log entries to the destination. | + + diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf new file mode 100644 index 00000000..e936df31 --- /dev/null +++ b/examples/logbucket/project/main.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + upper = false + special = false +} + +module "log_export" { + source = "../../../" + destination_uri = module.destination.destination_uri + filter = "resource.type = gce_instance" + log_sink_name = "logbucket_other_project" + parent_resource_id = var.parent_resource_project + parent_resource_type = "project" + unique_writer_identity = true +} + +module "destination" { + source = "../../..//modules/logbucket" + project_id = var.project_destination_logbkt_id + name = "logbucket_from_other_project_${random_string.suffix.result}" + location = "global" + log_sink_writer_identity = module.log_export.writer_identity +} + +#-------------------------------------# +# Log Bucket and Sink in same project # +#-------------------------------------# +module "log_export_same_proj" { + source = "../../../" + destination_uri = module.dest_same_proj.destination_uri + filter = "resource.type = gce_instance" + log_sink_name = "logbucket_same_project" + parent_resource_id = var.project_destination_logbkt_id + parent_resource_type = "project" + unique_writer_identity = true +} + +module "dest_same_proj" { + source = "../../..//modules/logbucket" + project_id = var.project_destination_logbkt_id + name = "logbucket_from_same_project_${random_string.suffix.result}" + location = "global" + log_sink_writer_identity = module.log_export_same_proj.writer_identity + grant_write_permission_on_bkt = false +} diff --git a/examples/logbucket/project/outputs.tf b/examples/logbucket/project/outputs.tf new file mode 100644 index 00000000..50b292be --- /dev/null +++ b/examples/logbucket/project/outputs.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "log_bucket_project" { + description = "The project where the log bucket is created." + value = module.destination.project +} + +output "log_bucket_name" { + description = "The name for the log bucket." + value = module.destination.resource_name +} + +output "log_sink_project_id" { + description = "The project id where the log sink is created." + value = module.log_export.parent_resource_id +} + +output "log_sink_destination_uri" { + description = "A fully qualified URI for the log sink." + value = module.destination.destination_uri +} + +output "log_sink_resource_name" { + description = "The resource name of the log sink that was created." + value = module.log_export.log_sink_resource_name +} + +output "log_sink_writer_identity" { + description = "The service account that logging uses to write log entries to the destination." + value = module.log_export.writer_identity +} + + +#-------------------------------------# +# Log Bucket and Sink in same project # +#-------------------------------------# +output "log_bkt_same_proj" { + description = "The project where the log bucket is created for sink and logbucket in same project example." + value = module.dest_same_proj.project +} + +output "log_bkt_name_same_proj" { + description = "The name for the log bucket for sink and logbucket in same project example." + value = module.dest_same_proj.resource_name +} + +output "log_sink_id_same_proj" { + description = "The project id where the log sink is created for sink and logbucket in same project example." + value = module.log_export_same_proj.parent_resource_id +} + +output "log_sink_dest_uri_same_proj" { + description = "A fully qualified URI for the log sink for sink and logbucket in same project example." + value = module.dest_same_proj.destination_uri +} + +output "log_sink_resource_name_same_proj" { + description = "The resource name of the log sink that was created in same project example." + value = module.log_export_same_proj.log_sink_resource_name +} + +output "log_sink_writer_identity_same_proj" { + description = "The service account in same project example that logging uses to write log entries to the destination." + value = module.log_export_same_proj.writer_identity +} diff --git a/examples/logbucket/project/variables.tf b/examples/logbucket/project/variables.tf new file mode 100644 index 00000000..16bb70e5 --- /dev/null +++ b/examples/logbucket/project/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_destination_logbkt_id" { + description = "The ID of the project in which log bucket destination will be created." + type = string +} + +variable "parent_resource_project" { + description = "The ID of the project in which the log export will be created." + type = string +} diff --git a/examples/logbucket/project/versions.tf b/examples/logbucket/project/versions.tf new file mode 100644 index 00000000..9bd2103e --- /dev/null +++ b/examples/logbucket/project/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +terraform { + required_version = ">= 0.13" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + } + } +} diff --git a/modules/logbucket/README.md b/modules/logbucket/README.md index f96dc91f..81799f85 100644 --- a/modules/logbucket/README.md +++ b/modules/logbucket/README.md @@ -37,6 +37,7 @@ module "destination" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| grant\_write\_permission\_on\_bkt | (Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions. | `bool` | `true` | no | | location | The location of the log bucket. | `string` | `"global"` | no | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | name | The name of the log bucket to be created and used for log entries matching the filter. | `string` | n/a | yes | diff --git a/modules/logbucket/main.tf b/modules/logbucket/main.tf index 2b758824..356e1a1e 100644 --- a/modules/logbucket/main.tf +++ b/modules/logbucket/main.tf @@ -43,6 +43,8 @@ resource "google_logging_project_bucket_config" "bucket" { # Service account IAM membership # #--------------------------------# resource "google_project_iam_member" "logbucket_sink_member" { + count = var.grant_write_permission_on_bkt ? 1 : 0 + project = google_logging_project_bucket_config.bucket.project role = "roles/logging.bucketWriter" member = var.log_sink_writer_identity diff --git a/modules/logbucket/variables.tf b/modules/logbucket/variables.tf index f39133a4..5b3e78a4 100644 --- a/modules/logbucket/variables.tf +++ b/modules/logbucket/variables.tf @@ -40,3 +40,9 @@ variable "retention_days" { type = number default = 30 } + +variable "grant_write_permission_on_bkt" { + description = "(Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions." + type = bool + default = true +} diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go new file mode 100644 index 00000000..9247c717 --- /dev/null +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logbucket_project + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/stretchr/testify/assert" +) + +const ( + defaultRetentionDays int64 = 30 +) + +func TestLogBucketProjectModule(t *testing.T) { + + bpt := tft.NewTFBlueprintTest(t, + tft.WithTFDir("../../../examples/logbucket/project"), + ) + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + for _, tc := range []struct { + projId string + bktName string + sinkDest string + sinkProjId string + sinkName string + writerIdentity string + }{ + { + projId: bpt.GetStringOutput("log_bucket_project"), + bktName: bpt.GetStringOutput("log_bucket_name"), + sinkDest: bpt.GetStringOutput("log_sink_destination_uri"), + sinkProjId: bpt.GetStringOutput("log_sink_project_id"), + sinkName: bpt.GetStringOutput("log_sink_resource_name"), + writerIdentity: bpt.GetStringOutput("log_sink_writer_identity"), + }, + { + projId: bpt.GetStringOutput("log_bkt_same_proj"), + bktName: bpt.GetStringOutput("log_bkt_name_same_proj"), + sinkDest: bpt.GetStringOutput("log_sink_dest_uri_same_proj"), + sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"), + sinkName: bpt.GetStringOutput("log_sink_resource_name_same_proj"), + // writerIdentity: As sink and bucket are in same project no service account is needed and writerIdentity is empty + }, + } { + //************************ + // Assert bucket details * + //************************ + bktFullName := fmt.Sprintf("projects/%s/locations/%s/buckets/%s", tc.projId, "global", tc.bktName) + logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", tc.bktName, "global", tc.projId)) + + // assert log bucket name, retention days & location + assert.Equal(bktFullName, logBucketDetails.Get("name").String(), "log bucket name should match") + assert.Equal(defaultRetentionDays, logBucketDetails.Get("retentionDays").Int(), "retention days should match") + + logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", tc.sinkName, tc.sinkProjId)) + + // assert log sink name, destination & filter + assert.Equal(tc.sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") + assert.Equal(tc.writerIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should match") + } + + //***************************** + // Assert SAs and Permissions * + //***************************** + bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj") + sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") + + projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId)) + listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"roles/logging.bucketWriter\").members").Array()) + + // assert sink writer identity service account permission + assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match") + assert.Len(listMembers, 1, "only one writer identity should have logbucket write permission") + }) + bpt.Test() +} diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 49109b3b..ffe3749c 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -87,6 +87,12 @@ resource "google_service_account" "int_test" { display_name = "ci-account" } +resource "google_service_account" "int_test_logbkt" { + project = module.project_destination_logbkt.project_id + account_id = "ci-account" + display_name = "ci-account" +} + resource "google_project_iam_member" "int_test" { for_each = toset(local.log_export_required_roles) @@ -95,6 +101,14 @@ resource "google_project_iam_member" "int_test" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "int_test_logbkt" { + for_each = toset(local.log_export_required_roles) + + project = module.project_destination_logbkt.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_billing_account_iam_member" "int_test" { for_each = toset(local.log_export_billing_account_roles) @@ -139,6 +153,7 @@ resource "null_resource" "wait_permissions" { google_billing_account_iam_member.int_test, google_folder_iam_member.int_test, google_organization_iam_member.int_test, - google_project_iam_member.int_test + google_project_iam_member.int_test, + google_project_iam_member.int_test_logbkt ] } diff --git a/test/setup/main.tf b/test/setup/main.tf index 63d87ba0..31c15eb7 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -41,12 +41,41 @@ module "project" { ] } +module "project_destination_logbkt" { + source = "terraform-google-modules/project-factory/google" + version = "~> 10.0" + + name = "ci-destination-logbkt" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + activate_apis = [ + "cloudapis.googleapis.com", + "cloudbuild.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", + "securitycenter.googleapis.com", + "cloudresourcemanager.googleapis.com", + "oslogin.googleapis.com", + "compute.googleapis.com", + "pubsub.googleapis.com", + "storage-component.googleapis.com", + "storage-api.googleapis.com", + "iam.googleapis.com", + "cloudbilling.googleapis.com" + ] +} + resource "null_resource" "wait_apis" { # Adding a pause as a workaround for of the provider issue # https://github.com/terraform-providers/terraform-provider-google/issues/1131 provisioner "local-exec" { command = "echo sleep 120s for APIs to get enabled; sleep 120" } - depends_on = [module.project.project_id] + depends_on = [ + module.project.project_id, + module.project_destination_logbkt.project_id + ] } - diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 6b45b70b..5a4dd9bc 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -38,3 +38,7 @@ output "parent_resource_billing_account" { output "parent_resource_organization" { value = var.org_id } + +output "project_destination_logbkt_id" { + value = module.project_destination_logbkt.project_id +}