fix: support logbucket sink in same project (#118)

* Fix Copyright year * New variable to deal with sink and logbucket in same project * When sink and logbucket in same project no new service account is created * New examples for logbucket at project-level * Changed name of new variable to a meaningful name * Created example of logbucket with project usage * Changed name of bucket due to colisions during integrated tests * Setup a new project for logbucket project testing in same and other projects * Integrated tests to Logbucket Project example * Revert mischanged copyright year * Shorten variable names * Fix logbucket/project example README.md Co-authored-by: Daniel Andrade <[email protected]> * Fix logbucket/project example README.md Co-authored-by: Daniel Andrade <[email protected]> * Revert mischanged copyright year * Inline constants * Changed variables creation place to right before usage * Remove retention_days to keep it default user better understanding example * -Created constant defaultRetentionDays and kept both examples with this default for user better understanding code and example -Changed assert bucket name to remove substring and compare to a built string fully qualified name -Combined similar asserts to tabular * Removed because empty is string default value Co-authored-by: Daniel Andrade <[email protected]> * Removed sink prefix because there is only one writerIdentity Co-authored-by: Daniel Andrade <[email protected]> * Fix assert writerIdentity message for examples Co-authored-by: Daniel Andrade <[email protected]> Co-authored-by: Bharath KKB <[email protected]>
terraform-google-modules · Aug 9, 2022 · 44758c2 · 44758c2
1 parent ef43513
commit 44758c2
Show file tree

Hide file tree

Showing 13 changed files with 395 additions and 3 deletions.
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
@@ -69,6 +69,21 @@ steps:
     - go-verify-logbucket-org
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestLogBucketOrgModule --stage teardown --verbose']
+- id: go-apply-logbucket-project
+  waitFor:
+    - init-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage apply --verbose']
+- id: go-verify-logbucket-project
+  waitFor:
+    - go-apply-logbucket-org
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage verify --verbose']
+- id: go-teardown-logbucket-project
+  waitFor:
+    - go-verify-logbucket-project
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md
@@ -0,0 +1,30 @@
+# Log Export: Log Bucket destination at Project level
+
+These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project or in separated projects.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| parent\_resource\_project | The ID of the project in which the log export will be created. | `string` | n/a | yes |
+| project\_destination\_logbkt\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| log\_bkt\_name\_same\_proj | The name for the log bucket for sink and logbucket in same project example. |
+| log\_bkt\_same\_proj | The project where the log bucket is created for sink and logbucket in same project example. |
+| log\_bucket\_name | The name for the log bucket. |
+| log\_bucket\_project | The project where the log bucket is created. |
+| log\_sink\_dest\_uri\_same\_proj | A fully qualified URI for the log sink for sink and logbucket in same project example. |
+| log\_sink\_destination\_uri | A fully qualified URI for the log sink. |
+| log\_sink\_id\_same\_proj | The project id where the log sink is created for sink and logbucket in same project example. |
+| log\_sink\_project\_id | The project id where the log sink is created. |
+| log\_sink\_resource\_name | The resource name of the log sink that was created. |
+| log\_sink\_resource\_name\_same\_proj | The resource name of the log sink that was created in same project example. |
+| log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. |
+| log\_sink\_writer\_identity\_same\_proj | The service account in same project example that logging uses to write log entries to the destination. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf
@@ -0,0 +1,61 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "suffix" {
+  length  = 4
+  upper   = false
+  special = false
+}
+
+module "log_export" {
+  source                 = "../../../"
+  destination_uri        = module.destination.destination_uri
+  filter                 = "resource.type = gce_instance"
+  log_sink_name          = "logbucket_other_project"
+  parent_resource_id     = var.parent_resource_project
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "destination" {
+  source                   = "../../..//modules/logbucket"
+  project_id               = var.project_destination_logbkt_id
+  name                     = "logbucket_from_other_project_${random_string.suffix.result}"
+  location                 = "global"
+  log_sink_writer_identity = module.log_export.writer_identity
+}
+
+#-------------------------------------#
+# Log Bucket and Sink in same project #
+#-------------------------------------#
+module "log_export_same_proj" {
+  source                 = "../../../"
+  destination_uri        = module.dest_same_proj.destination_uri
+  filter                 = "resource.type = gce_instance"
+  log_sink_name          = "logbucket_same_project"
+  parent_resource_id     = var.project_destination_logbkt_id
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "dest_same_proj" {
+  source                        = "../../..//modules/logbucket"
+  project_id                    = var.project_destination_logbkt_id
+  name                          = "logbucket_from_same_project_${random_string.suffix.result}"
+  location                      = "global"
+  log_sink_writer_identity      = module.log_export_same_proj.writer_identity
+  grant_write_permission_on_bkt = false
+}
diff --git a/examples/logbucket/project/outputs.tf b/examples/logbucket/project/outputs.tf
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "log_bucket_project" {
+  description = "The project where the log bucket is created."
+  value       = module.destination.project
+}
+
+output "log_bucket_name" {
+  description = "The name for the log bucket."
+  value       = module.destination.resource_name
+}
+
+output "log_sink_project_id" {
+  description = "The project id where the log sink is created."
+  value       = module.log_export.parent_resource_id
+}
+
+output "log_sink_destination_uri" {
+  description = "A fully qualified URI for the log sink."
+  value       = module.destination.destination_uri
+}
+
+output "log_sink_resource_name" {
+  description = "The resource name of the log sink that was created."
+  value       = module.log_export.log_sink_resource_name
+}
+
+output "log_sink_writer_identity" {
+  description = "The service account that logging uses to write log entries to the destination."
+  value       = module.log_export.writer_identity
+}
+
+
+#-------------------------------------#
+# Log Bucket and Sink in same project #
+#-------------------------------------#
+output "log_bkt_same_proj" {
+  description = "The project where the log bucket is created for sink and logbucket in same project example."
+  value       = module.dest_same_proj.project
+}
+
+output "log_bkt_name_same_proj" {
+  description = "The name for the log bucket for sink and logbucket in same project example."
+  value       = module.dest_same_proj.resource_name
+}
+
+output "log_sink_id_same_proj" {
+  description = "The project id where the log sink is created for sink and logbucket in same project example."
+  value       = module.log_export_same_proj.parent_resource_id
+}
+
+output "log_sink_dest_uri_same_proj" {
+  description = "A fully qualified URI for the log sink for sink and logbucket in same project example."
+  value       = module.dest_same_proj.destination_uri
+}
+
+output "log_sink_resource_name_same_proj" {
+  description = "The resource name of the log sink that was created in same project example."
+  value       = module.log_export_same_proj.log_sink_resource_name
+}
+
+output "log_sink_writer_identity_same_proj" {
+  description = "The service account in same project example that logging uses to write log entries to the destination."
+  value       = module.log_export_same_proj.writer_identity
+}
diff --git a/examples/logbucket/project/variables.tf b/examples/logbucket/project/variables.tf
@@ -0,0 +1,25 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_destination_logbkt_id" {
+  description = "The ID of the project in which log bucket destination will be created."
+  type        = string
+}
+
+variable "parent_resource_project" {
+  description = "The ID of the project in which the log export will be created."
+  type        = string
+}
diff --git a/examples/logbucket/project/versions.tf b/examples/logbucket/project/versions.tf
@@ -0,0 +1,29 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.0"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+  }
+}
diff --git a/modules/logbucket/README.md b/modules/logbucket/README.md
@@ -37,6 +37,7 @@ module "destination" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| grant\_write\_permission\_on\_bkt | (Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions. | `bool` | `true` | no |
 | location | The location of the log bucket. | `string` | `"global"` | no |
 | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes |
 | name | The name of the log bucket to be created and used for log entries matching the filter. | `string` | n/a | yes |

diff --git a/modules/logbucket/main.tf b/modules/logbucket/main.tf
@@ -43,6 +43,8 @@ resource "google_logging_project_bucket_config" "bucket" {
 # Service account IAM membership #
 #--------------------------------#
 resource "google_project_iam_member" "logbucket_sink_member" {
+  count = var.grant_write_permission_on_bkt ? 1 : 0
+
   project = google_logging_project_bucket_config.bucket.project
   role    = "roles/logging.bucketWriter"
   member  = var.log_sink_writer_identity

diff --git a/modules/logbucket/variables.tf b/modules/logbucket/variables.tf
@@ -40,3 +40,9 @@ variable "retention_days" {
   type        = number
   default     = 30
 }
+
+variable "grant_write_permission_on_bkt" {
+  description = "(Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions."
+  type        = bool
+  default     = true
+}
diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go
@@ -0,0 +1,96 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logbucket_project
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	defaultRetentionDays int64 = 30
+)
+
+func TestLogBucketProjectModule(t *testing.T) {
+
+	bpt := tft.NewTFBlueprintTest(t,
+		tft.WithTFDir("../../../examples/logbucket/project"),
+	)
+	bpt.DefineVerify(func(assert *assert.Assertions) {
+		bpt.DefaultVerify(assert)
+
+		for _, tc := range []struct {
+			projId         string
+			bktName        string
+			sinkDest       string
+			sinkProjId     string
+			sinkName       string
+			writerIdentity string
+		}{
+			{
+				projId:         bpt.GetStringOutput("log_bucket_project"),
+				bktName:        bpt.GetStringOutput("log_bucket_name"),
+				sinkDest:       bpt.GetStringOutput("log_sink_destination_uri"),
+				sinkProjId:     bpt.GetStringOutput("log_sink_project_id"),
+				sinkName:       bpt.GetStringOutput("log_sink_resource_name"),
+				writerIdentity: bpt.GetStringOutput("log_sink_writer_identity"),
+			},
+			{
+				projId:     bpt.GetStringOutput("log_bkt_same_proj"),
+				bktName:    bpt.GetStringOutput("log_bkt_name_same_proj"),
+				sinkDest:   bpt.GetStringOutput("log_sink_dest_uri_same_proj"),
+				sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"),
+				sinkName:   bpt.GetStringOutput("log_sink_resource_name_same_proj"),
+				// writerIdentity: As sink and bucket are in same project no service account is needed and writerIdentity is empty
+			},
+		} {
+			//************************
+			// Assert bucket details *
+			//************************
+			bktFullName := fmt.Sprintf("projects/%s/locations/%s/buckets/%s", tc.projId, "global", tc.bktName)
+			logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", tc.bktName, "global", tc.projId))
+
+			// assert log bucket name, retention days & location
+			assert.Equal(bktFullName, logBucketDetails.Get("name").String(), "log bucket name should match")
+			assert.Equal(defaultRetentionDays, logBucketDetails.Get("retentionDays").Int(), "retention days should match")
+
+			logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", tc.sinkName, tc.sinkProjId))
+
+			// assert log sink name, destination & filter
+			assert.Equal(tc.sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match")
+			assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match")
+			assert.Equal(tc.writerIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should match")
+		}
+
+		//*****************************
+		// Assert SAs and Permissions *
+		//*****************************
+		bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj")
+		sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity")
+
+		projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId))
+		listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"roles/logging.bucketWriter\").members").Array())
+
+		// assert sink writer identity service account permission
+		assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match")
+		assert.Len(listMembers, 1, "only one writer identity should have logbucket write permission")
+	})
+	bpt.Test()
+}