Skip to content

Commit

Permalink
Support for cloud logging buckets
Browse files Browse the repository at this point in the history
  • Loading branch information
juliocc committed Mar 3, 2021
1 parent 0f469a2 commit ad68fc4
Show file tree
Hide file tree
Showing 23 changed files with 847 additions and 234 deletions.
24 changes: 22 additions & 2 deletions modules/folder/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ module "pubsub" {
name = "pubsub_sink"
}
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = "my-project"
id = "bucket"
}
module "folder-sink" {
source = "./modules/folder"
parent = "folders/657104291943"
Expand All @@ -74,27 +81,40 @@ module "folder-sink" {
filter = "severity=WARNING"
iam = false
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
iam = false
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
iam = true
include_children = true
exclusions = {}
}
debug = {
type = "logging"
destination = module.bucket.id
filter = "severity=DEBUG"
iam = true
include_children = true
exclusions = {
no-compute = "logName:compute"
}
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=9
# tftest:modules=5:resources=11
```

### Hierarchical firewall policies
Expand Down Expand Up @@ -151,7 +171,7 @@ module "folder2" {
| *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | <code title="map&#40;set&#40;string&#41;&#41;">map(set(string))</code> | | <code title="">{}</code> |
| *id* | Folder ID in case you use folder_create=false | <code title="">string</code> | | <code title="">null</code> |
| *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *name* | Folder name. | <code title="">string</code> | | <code title="">null</code> |
| *parent* | Parent in folders/folder_id or organizations/org_id format. | <code title="">string</code> | | <code title="null&#10;validation &#123;&#10;condition &#61; var.parent &#61;&#61; null &#124;&#124; can&#40;regex&#40;&#34;&#40;organizations&#124;folders&#41;&#47;&#91;0-9&#93;&#43;&#34;, var.parent&#41;&#41;&#10;error_message &#61; &#34;Parent must be of the form folders&#47;folder_id or organizations&#47;organization_id.&#34;&#10;&#125;">...</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |
Expand Down
12 changes: 10 additions & 2 deletions modules/folder/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ locals {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
Expand Down Expand Up @@ -192,6 +191,15 @@ resource "google_logging_folder_sink" "sink" {
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
include_children = each.value.include_children

dynamic "exclusions" {
for_each = each.value.exclusions
iterator = exclusion
content {
name = exclusion.key
filter = exclusion.value
}
}
}

resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
Expand Down
2 changes: 2 additions & 0 deletions modules/folder/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
# TODO exclusions also support description and disabled
exclusions = map(string)
}))
default = {}
}
Expand Down
61 changes: 61 additions & 0 deletions modules/logging-bucket/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Google Cloud Logging Buckets Module

This module manages [logging buckets](https://cloud.google.com/logging/docs/storage#logs-buckets) for a project, folder, organization or billing account.

Note that some logging buckets are automatically created for a given folder, project, organization, and billing account cannot be deleted. Creating a resource of this type will acquire and update the resource that already exists at the desired location. These buckets cannot be removed so deleting this resource will remove the bucket config from your terraform state but will leave the logging bucket unchanged. The buckets that are currently automatically created are "_Default" and "_Required".

See also the `logging_sinks` argument within the [project](../project/), [folder](../folder/) and [organization](../organization) modules.

## Examples

### Create custom logging bucket in a project

```hcl
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = var.project_id
id = "mybucket"
}
# tftest:modules=1:resources=1
```


### Change retention period of a folder's _Default bucket

```hcl
module "folder" {
source = "./modules/folder"
parent = "folders/657104291943"
name = "my folder"
}
module "bucket-default" {
source = "./modules/logging-bucket"
parent_type = "folder"
parent = module.folder.id
id = "_Default"
retention = 10
}
# tftest:modules=2:resources=2
```


<!-- BEGIN TFDOC -->
## Variables

| name | description | type | required | default |
|---|---|:---: |:---:|:---:|
| id | Name of the logging bucket. | <code title="">string</code> || |
| parent | ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'. | <code title="">string</code> || |
| parent_type | Parent object type for the bucket (project, folder, organization, billing_account). | <code title="">string</code> || |
| *description* | Human-readable description for the logging bucket. | <code title="">string</code> | | <code title="">null</code> |
| *location* | Location of the bucket. | <code title="">string</code> | | <code title="">global</code> |
| *retention* | Retention time in days for the logging bucket. | <code title="">number</code> | | <code title="">30</code> |

## Outputs

| name | description | sensitive |
|---|---|:---:|
| id | None | |
<!-- END TFDOC -->
51 changes: 51 additions & 0 deletions modules/logging-bucket/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

resource "google_logging_project_bucket_config" "bucket" {
count = var.parent_type == "project" ? 1 : 0
project = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}

resource "google_logging_folder_bucket_config" "bucket" {
count = var.parent_type == "folder" ? 1 : 0
folder = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}

resource "google_logging_organization_bucket_config" "bucket" {
count = var.parent_type == "organization" ? 1 : 0
organization = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}

resource "google_logging_billing_account_bucket_config" "bucket" {
count = var.parent_type == "billing_account" ? 1 : 0
billing_account = var.parent
location = var.location
retention_days = var.retention
bucket_id = var.id
description = var.description
}
24 changes: 24 additions & 0 deletions modules/logging-bucket/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

output "id" {
value = try(
google_logging_project_bucket_config.bucket.0.id,
google_logging_folder_bucket_config.bucket.0.id,
google_logging_organization_bucket_config.bucket.0.id,
google_logging_billing_account_bucket_config.bucket.0.id,
)
}
48 changes: 48 additions & 0 deletions modules/logging-bucket/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

variable "parent_type" {
description = "Parent object type for the bucket (project, folder, organization, billing_account)."
type = string
}

variable "parent" {
description = "ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'."
type = string
}

variable "location" {
description = "Location of the bucket."
type = string
default = "global"
}

variable "id" {
description = "Name of the logging bucket."
type = string
}

variable "description" {
description = "Human-readable description for the logging bucket."
type = string
default = null
}

variable "retention" {
description = "Retention time in days for the logging bucket."
type = number
default = 30
}
24 changes: 22 additions & 2 deletions modules/organization/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,13 @@ module "pubsub" {
name = "pubsub_sink"
}
module "bucket" {
source = "./modules/logging-bucket"
parent_type = "project"
parent = "my-project"
id = "bucket"
}
module "org" {
source = "./modules/organization"
organization_id = var.organization_id
Expand All @@ -91,27 +98,40 @@ module "org" {
filter = "severity=WARNING"
iam = false
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
destination = module.dataset.id
filter = "severity=INFO"
iam = false
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
destination = module.pubsub.id
filter = "severity=NOTICE"
iam = true
include_children = true
exclusions = {}
}
debug = {
type = "logging"
destination = module.bucket.id
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
}
}
}
logging_exclusions = {
no-gce-instances = "resource.type=gce_instance"
}
}
# tftest:modules=4:resources=8
# tftest:modules=5:resources=10
```


Expand All @@ -132,7 +152,7 @@ module "org" {
| *iam_audit_config_authoritative* | IAM Authoritative service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. Audit config should also be authoritative when using authoritative bindings. Use with caution. | <code title="map&#40;map&#40;list&#40;string&#41;&#41;&#41;">map(map(list(string)))</code> | | <code title="">null</code> |
| *iam_bindings_authoritative* | IAM authoritative bindings, in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared. Bindings should also be authoritative when using authoritative audit config. Use with caution. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">null</code> |
| *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this organization. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *logging_sinks* | Logging sinks to create for this organization. | <code title="map&#40;object&#40;&#123;&#10;destination &#61; string&#10;type &#61; string&#10;filter &#61; string&#10;iam &#61; bool&#10;include_children &#61; bool&#10;exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | <code title="map&#40;bool&#41;">map(bool)</code> | | <code title="">{}</code> |
| *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | <code title="map&#40;object&#40;&#123;&#10;inherit_from_parent &#61; bool&#10;suggested_value &#61; string&#10;status &#61; bool&#10;values &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |

Expand Down
12 changes: 10 additions & 2 deletions modules/organization/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ locals {
gcs = "storage.googleapis.com"
bigquery = "bigquery.googleapis.com"
pubsub = "pubsub.googleapis.com"
# TODO: add logging buckets support
# logging = "logging.googleapis.com"
logging = "logging.googleapis.com"
}
sink_bindings = {
for type in ["gcs", "bigquery", "pubsub", "logging"] :
Expand Down Expand Up @@ -256,6 +255,15 @@ resource "google_logging_organization_sink" "sink" {
destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}"
filter = each.value.filter
include_children = each.value.include_children

dynamic "exclusions" {
for_each = each.value.exclusions
iterator = exclusion
content {
name = exclusion.key
filter = exclusion.value
}
}
}

resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" {
Expand Down
2 changes: 2 additions & 0 deletions modules/organization/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
# TODO exclusions also support description and disabled
exclusions = map(string)
}))
default = {}
}
Expand Down
Loading

0 comments on commit ad68fc4

Please sign in to comment.