From a5843b86de6384b39bbcdef9ddc18d0f33f49ef9 Mon Sep 17 00:00:00 2001 From: umairidris Date: Fri, 16 Oct 2020 20:16:31 -0400 Subject: [PATCH 1/5] migrate logging to cft modules --- .../generated/folder_foundation/audit/main.tf | 115 ++++++++---------- .../generated/org_foundation/audit/main.tf | 115 ++++++++---------- templates/tfengine/components/audit/main.tf | 115 ++++++++---------- 3 files changed, 144 insertions(+), 201 deletions(-) diff --git a/examples/tfengine/generated/folder_foundation/audit/main.tf b/examples/tfengine/generated/folder_foundation/audit/main.tf index 6936a035c..b73661802 100644 --- a/examples/tfengine/generated/folder_foundation/audit/main.tf +++ b/examples/tfengine/generated/folder_foundation/audit/main.tf @@ -61,84 +61,65 @@ resource "google_folder_iam_audit_config" "config" { } } -# BigQuery log sink. -resource "google_logging_folder_sink" "bigquery_audit_logs_sink" { - name = "bigquery-audit-logs-sink" - folder = var.folder - include_children = true - filter = "logName:\"logs/cloudaudit.googleapis.com\" OR logName=\"logs/forseti\" OR logName=\"logs/application\"" - destination = "bigquery.googleapis.com/projects/${module.project.project_id}/datasets/${module.bigquery_destination.bigquery_dataset.dataset_id}" +module "bigquery_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + destination_uri = module.bigquery_destination.destination_uri + filter = "logName:\"logs/cloudaudit.googleapis.com\" OR logName=\"logs/forseti\" OR logName=\"logs/application\"" + log_sink_name = "bigquery-audit-logs-sink" + parent_resource_type = "folder" + parent_resource_id = var.folder + unique_writer_identity = true + include_children = true } module "bigquery_destination" { - source = "terraform-google-modules/bigquery/google" - version = "~> 4.3.0" - - dataset_id = "1yr_folder_audit_logs" - project_id = module.project.project_id - location = "us-east1" - default_table_expiration_ms = 365 * 8.64 * pow(10, 7) # 365 days - access = [ - { - role = "roles/bigquery.dataOwner", - special_group = "projectOwners" - }, - { - role = "roles/bigquery.dataViewer", - group_by_email = var.auditors_group - }, - ] -} + source = "terraform-google-modules/log-export/google//modules/bigquery" + version = "~> 5.0.0" -resource "google_project_iam_member" "bigquery_sink_member" { - project = module.bigquery_destination.bigquery_dataset.project - role = "roles/bigquery.dataEditor" - member = google_logging_folder_sink.bigquery_audit_logs_sink.writer_identity + dataset_name = "1yr_folder_audit_logs" + project_id = module.project.project_id + log_sink_writer_identity = module.bigquery_export.writer_identity + expiration_days = 365 } -# Cloud Storage log sink. -resource "google_logging_folder_sink" "storage_audit_logs_sink" { - name = "storage-audit-logs-sink" - folder = var.folder - include_children = true - filter = "logName:\"logs/cloudaudit.googleapis.com\" OR logName=\"logs/forseti\" OR logName=\"logs/application\"" - destination = "storage.googleapis.com/${module.storage_destination.bucket.name}" +module "storage_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + destination_uri = module.storage_destination.destination_uri + filter = "logName:\"logs/cloudaudit.googleapis.com\" OR logName=\"logs/forseti\" OR logName=\"logs/application\"" + log_sink_name = "storage-audit-logs-sink" + parent_resource_type = "folder" + parent_resource_id = var.folder + unique_writer_identity = true + include_children = true } module "storage_destination" { - source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" - version = "~> 1.7.0" - - name = "7yr-folder-audit-logs" - project_id = module.project.project_id - location = "us-central1" - storage_class = "COLDLINE" - - lifecycle_rules = [{ - action = { - type = "Delete" - } - condition = { - age = 7 * 365 # 7 years - with_state = "ANY" - } - }] - - iam_members = [ - { - role = "roles/storage.objectViewer" - member = "group:${var.auditors_group}" - }, - ] + source = "terraform-google-modules/log-export/google//modules/storage" + version = "~> 5.0.0" + + storage_bucket_name = "7yr-folder-audit-logs" + project_id = module.project.project_id + log_sink_writer_identity = module.storage_export.writer_identity + storage_class = "COLDLINE" + expiration_days = 7 * 365 + retention_policy = { + is_locked = true + retention_period_days = 6 * 365 + } } -# Deploy sink member as separate resource otherwise Terraform will return error: -# `The "for_each" value depends on resource attributes that cannot be determined -# until apply, so Terraform cannot predict how many instances will be created.` -resource "google_storage_bucket_iam_member" "storage_sink_member" { - bucket = module.storage_destination.bucket.name - role = "roles/storage.objectCreator" - member = google_logging_folder_sink.storage_audit_logs_sink.writer_identity +resource "google_project_iam_member" "logs_viewers_auditors" { + for_each = toset([ + "roles/bigquery.user", + "roles/storage.objectViewer", + ]) + project = module.project.project_id + role = each.key + member = "group:${var.auditors_group}" } # IAM permissions to grant log Auditors iam.securityReviewer role to view the logs. diff --git a/examples/tfengine/generated/org_foundation/audit/main.tf b/examples/tfengine/generated/org_foundation/audit/main.tf index 8b2e509f7..186c4da6d 100644 --- a/examples/tfengine/generated/org_foundation/audit/main.tf +++ b/examples/tfengine/generated/org_foundation/audit/main.tf @@ -60,84 +60,65 @@ resource "google_organization_iam_audit_config" "config" { } } -# BigQuery log sink. -resource "google_logging_organization_sink" "bigquery_audit_logs_sink" { - name = "bigquery-audit-logs-sink" - org_id = var.org_id - include_children = true - filter = "logName:\"logs/cloudaudit.googleapis.com\"" - destination = "bigquery.googleapis.com/projects/${module.project.project_id}/datasets/${module.bigquery_destination.bigquery_dataset.dataset_id}" +module "bigquery_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + destination_uri = module.bigquery_destination.destination_uri + filter = "logName:\"logs/cloudaudit.googleapis.com\"" + log_sink_name = "bigquery-audit-logs-sink" + parent_resource_type = "organization" + parent_resource_id = var.org_id + unique_writer_identity = true + include_children = true } module "bigquery_destination" { - source = "terraform-google-modules/bigquery/google" - version = "~> 4.3.0" - - dataset_id = "1yr_org_audit_logs" - project_id = module.project.project_id - location = "us-east1" - default_table_expiration_ms = 365 * 8.64 * pow(10, 7) # 365 days - access = [ - { - role = "roles/bigquery.dataOwner", - special_group = "projectOwners" - }, - { - role = "roles/bigquery.dataViewer", - group_by_email = var.auditors_group - }, - ] -} + source = "terraform-google-modules/log-export/google//modules/bigquery" + version = "~> 5.0.0" -resource "google_project_iam_member" "bigquery_sink_member" { - project = module.bigquery_destination.bigquery_dataset.project - role = "roles/bigquery.dataEditor" - member = google_logging_organization_sink.bigquery_audit_logs_sink.writer_identity + dataset_name = "1yr_org_audit_logs" + project_id = module.project.project_id + log_sink_writer_identity = module.bigquery_export.writer_identity + expiration_days = 365 } -# Cloud Storage log sink. -resource "google_logging_organization_sink" "storage_audit_logs_sink" { - name = "storage-audit-logs-sink" - org_id = var.org_id - include_children = true - filter = "logName:\"logs/cloudaudit.googleapis.com\"" - destination = "storage.googleapis.com/${module.storage_destination.bucket.name}" +module "storage_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + destination_uri = module.storage_destination.destination_uri + filter = "logName:\"logs/cloudaudit.googleapis.com\"" + log_sink_name = "storage-audit-logs-sink" + parent_resource_type = "organization" + parent_resource_id = var.org_id + unique_writer_identity = true + include_children = true } module "storage_destination" { - source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" - version = "~> 1.7.0" - - name = "7yr-org-audit-logs" - project_id = module.project.project_id - location = "us-central1" - storage_class = "COLDLINE" - - lifecycle_rules = [{ - action = { - type = "Delete" - } - condition = { - age = 7 * 365 # 7 years - with_state = "ANY" - } - }] - - iam_members = [ - { - role = "roles/storage.objectViewer" - member = "group:${var.auditors_group}" - }, - ] + source = "terraform-google-modules/log-export/google//modules/storage" + version = "~> 5.0.0" + + storage_bucket_name = "7yr-org-audit-logs" + project_id = module.project.project_id + log_sink_writer_identity = module.storage_export.writer_identity + storage_class = "COLDLINE" + expiration_days = 7 * 365 + retention_policy = { + is_locked = true + retention_period_days = 6 * 365 + } } -# Deploy sink member as separate resource otherwise Terraform will return error: -# `The "for_each" value depends on resource attributes that cannot be determined -# until apply, so Terraform cannot predict how many instances will be created.` -resource "google_storage_bucket_iam_member" "storage_sink_member" { - bucket = module.storage_destination.bucket.name - role = "roles/storage.objectCreator" - member = google_logging_organization_sink.storage_audit_logs_sink.writer_identity +resource "google_project_iam_member" "logs_viewers_auditors" { + for_each = toset([ + "roles/bigquery.user", + "roles/storage.objectViewer", + ]) + project = module.project.project_id + role = each.key + member = "group:${var.auditors_group}" } # IAM permissions to grant log Auditors iam.securityReviewer role to view the logs. diff --git a/templates/tfengine/components/audit/main.tf b/templates/tfengine/components/audit/main.tf index 27d616b9b..5c539606b 100644 --- a/templates/tfengine/components/audit/main.tf +++ b/templates/tfengine/components/audit/main.tf @@ -40,84 +40,65 @@ resource "google_{{.parent_type}}_iam_audit_config" "config" { } } -# BigQuery log sink. -resource "google_logging_{{.parent_type}}_sink" "bigquery_audit_logs_sink" { - name = "bigquery-audit-logs-sink" - {{$parent_field}} = {{$parent_var}} - include_children = true - filter = "{{$filter}}" - destination = "bigquery.googleapis.com/projects/${module.project.project_id}/datasets/${module.bigquery_destination.bigquery_dataset.dataset_id}" +module "bigquery_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + log_sink_name = "bigquery-audit-logs-sink" + destination_uri = "${module.bigquery_destination.destination_uri}" + filter = "{{$filter}}" + parent_resource_type = "{{.parent_type}}" + parent_resource_id = {{$parent_var}} + unique_writer_identity = true + include_children = true } module "bigquery_destination" { - source = "terraform-google-modules/bigquery/google" - version = "~> 4.3.0" - - dataset_id = "{{.logs_bigquery_dataset.dataset_id}}" - project_id = module.project.project_id - location = "{{.bigquery_location}}" - default_table_expiration_ms = 365 * 8.64 * pow(10, 7) # 365 days - access = [ - { - role = "roles/bigquery.dataOwner", - special_group = "projectOwners" - }, - { - role = "roles/bigquery.dataViewer", - group_by_email = var.auditors_group - }, - ] -} + source = "terraform-google-modules/log-export/google//modules/bigquery" + version = "~> 5.0.0" -resource "google_project_iam_member" "bigquery_sink_member" { - project = module.bigquery_destination.bigquery_dataset.project - role = "roles/bigquery.dataEditor" - member = google_logging_{{.parent_type}}_sink.bigquery_audit_logs_sink.writer_identity + dataset_name = "{{.logs_bigquery_dataset.dataset_id}}" + project_id = module.project.project_id + log_sink_writer_identity = "${module.bigquery_export.writer_identity}" + expiration_days = 365 } -# Cloud Storage log sink. -resource "google_logging_{{.parent_type}}_sink" "storage_audit_logs_sink" { - name = "storage-audit-logs-sink" - {{$parent_field}} = {{$parent_var}} - include_children = true - filter = "{{$filter}}" - destination = "storage.googleapis.com/${module.storage_destination.bucket.name}" +module "storage_export" { + source = "terraform-google-modules/log-export/google" + version = "~> 5.0.0" + + destination_uri = "${module.storage_destination.destination_uri}" + filter = "{{$filter}}" + log_sink_name = "storage-audit-logs-sink" + parent_resource_type = "{{.parent_type}}" + parent_resource_id = {{$parent_var}} + unique_writer_identity = true + include_children = true } module "storage_destination" { - source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" - version = "~> 1.7.0" - - name = "{{.logs_storage_bucket.name}}" - project_id = module.project.project_id - location = "{{.storage_location}}" - storage_class = "COLDLINE" - - lifecycle_rules = [{ - action = { - type = "Delete" - } - condition = { - age = 7 * 365 # 7 years - with_state = "ANY" - } - }] - - iam_members = [ - { - role = "roles/storage.objectViewer" - member = "group:${var.auditors_group}" - }, - ] + source = "terraform-google-modules/log-export/google//modules/storage" + version = "~> 5.0.0" + + storage_bucket_name = "{{.logs_storage_bucket.name}}" + project_id = module.project.project_id + log_sink_writer_identity = "${module.storage_export.writer_identity}" + storage_class = "COLDLINE" + expiration_days = 7*365 + retention_policy = { + is_locked = true + retention_period_days = 6 * 365 + } } -# Deploy sink member as separate resource otherwise Terraform will return error: -# `The "for_each" value depends on resource attributes that cannot be determined -# until apply, so Terraform cannot predict how many instances will be created.` -resource "google_storage_bucket_iam_member" "storage_sink_member" { - bucket = module.storage_destination.bucket.name - role = "roles/storage.objectCreator" - member = google_logging_{{.parent_type}}_sink.storage_audit_logs_sink.writer_identity +resource "google_project_iam_member" "logs_viewers_auditors" { + for_each = toset([ + "roles/bigquery.user", + "roles/storage.objectViewer", + ]) + project = module.project.project_id + role = each.key + member = "group:${var.auditors_group}" } # IAM permissions to grant log Auditors iam.securityReviewer role to view the logs. From 5994a48283617a7f856c44f36468abc281c7ca5a Mon Sep 17 00:00:00 2001 From: umairidris Date: Fri, 16 Oct 2020 20:21:02 -0400 Subject: [PATCH 2/5] add locations back --- examples/tfengine/generated/folder_foundation/audit/main.tf | 4 +++- examples/tfengine/generated/org_foundation/audit/main.tf | 4 +++- templates/tfengine/components/audit/main.tf | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/tfengine/generated/folder_foundation/audit/main.tf b/examples/tfengine/generated/folder_foundation/audit/main.tf index b73661802..3b35f81c7 100644 --- a/examples/tfengine/generated/folder_foundation/audit/main.tf +++ b/examples/tfengine/generated/folder_foundation/audit/main.tf @@ -65,9 +65,9 @@ module "bigquery_export" { source = "terraform-google-modules/log-export/google" version = "~> 5.0.0" + log_sink_name = "bigquery-audit-logs-sink" destination_uri = module.bigquery_destination.destination_uri filter = "logName:\"logs/cloudaudit.googleapis.com\" OR logName=\"logs/forseti\" OR logName=\"logs/application\"" - log_sink_name = "bigquery-audit-logs-sink" parent_resource_type = "folder" parent_resource_id = var.folder unique_writer_identity = true @@ -80,6 +80,7 @@ module "bigquery_destination" { dataset_name = "1yr_folder_audit_logs" project_id = module.project.project_id + location = "us-east1" log_sink_writer_identity = module.bigquery_export.writer_identity expiration_days = 365 } @@ -103,6 +104,7 @@ module "storage_destination" { storage_bucket_name = "7yr-folder-audit-logs" project_id = module.project.project_id + location = "us-central1" log_sink_writer_identity = module.storage_export.writer_identity storage_class = "COLDLINE" expiration_days = 7 * 365 diff --git a/examples/tfengine/generated/org_foundation/audit/main.tf b/examples/tfengine/generated/org_foundation/audit/main.tf index 186c4da6d..71cfa148f 100644 --- a/examples/tfengine/generated/org_foundation/audit/main.tf +++ b/examples/tfengine/generated/org_foundation/audit/main.tf @@ -64,9 +64,9 @@ module "bigquery_export" { source = "terraform-google-modules/log-export/google" version = "~> 5.0.0" + log_sink_name = "bigquery-audit-logs-sink" destination_uri = module.bigquery_destination.destination_uri filter = "logName:\"logs/cloudaudit.googleapis.com\"" - log_sink_name = "bigquery-audit-logs-sink" parent_resource_type = "organization" parent_resource_id = var.org_id unique_writer_identity = true @@ -79,6 +79,7 @@ module "bigquery_destination" { dataset_name = "1yr_org_audit_logs" project_id = module.project.project_id + location = "us-east1" log_sink_writer_identity = module.bigquery_export.writer_identity expiration_days = 365 } @@ -102,6 +103,7 @@ module "storage_destination" { storage_bucket_name = "7yr-org-audit-logs" project_id = module.project.project_id + location = "us-central1" log_sink_writer_identity = module.storage_export.writer_identity storage_class = "COLDLINE" expiration_days = 7 * 365 diff --git a/templates/tfengine/components/audit/main.tf b/templates/tfengine/components/audit/main.tf index 5c539606b..f26125fc7 100644 --- a/templates/tfengine/components/audit/main.tf +++ b/templates/tfengine/components/audit/main.tf @@ -59,6 +59,7 @@ module "bigquery_destination" { dataset_name = "{{.logs_bigquery_dataset.dataset_id}}" project_id = module.project.project_id + location = "{{.bigquery_location}}" log_sink_writer_identity = "${module.bigquery_export.writer_identity}" expiration_days = 365 } @@ -82,6 +83,7 @@ module "storage_destination" { storage_bucket_name = "{{.logs_storage_bucket.name}}" project_id = module.project.project_id + location = "{{.storage_location}}" log_sink_writer_identity = "${module.storage_export.writer_identity}" storage_class = "COLDLINE" expiration_days = 7*365 From 8bb5056df3c10c3aff839f77339a1dbb996b90aa Mon Sep 17 00:00:00 2001 From: umairidris Date: Mon, 19 Oct 2020 12:14:46 -0400 Subject: [PATCH 3/5] space --- templates/tfengine/components/audit/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tfengine/components/audit/main.tf b/templates/tfengine/components/audit/main.tf index f26125fc7..8feadc141 100644 --- a/templates/tfengine/components/audit/main.tf +++ b/templates/tfengine/components/audit/main.tf @@ -86,7 +86,7 @@ module "storage_destination" { location = "{{.storage_location}}" log_sink_writer_identity = "${module.storage_export.writer_identity}" storage_class = "COLDLINE" - expiration_days = 7*365 + expiration_days = 7 * 365 retention_policy = { is_locked = true retention_period_days = 6 * 365 From 72f73e209ccc23f4c5a7939ffb5bfa7730ad7f61 Mon Sep 17 00:00:00 2001 From: umairidris Date: Mon, 19 Oct 2020 12:16:45 -0400 Subject: [PATCH 4/5] comment --- templates/tfengine/components/audit/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/tfengine/components/audit/main.tf b/templates/tfengine/components/audit/main.tf index 8feadc141..411470178 100644 --- a/templates/tfengine/components/audit/main.tf +++ b/templates/tfengine/components/audit/main.tf @@ -77,6 +77,9 @@ module "storage_export" { include_children = true } +// 6 years minimum audit log retention is required for HIPAA alignment. +// Thus, we lock retention policy to be at least 6 years +// and set the actual expiry to be greater than this amount (7 years). module "storage_destination" { source = "terraform-google-modules/log-export/google//modules/storage" version = "~> 5.0.0" From 9321d745b439fbe6c6edcd5b08f31c6ae82b354c Mon Sep 17 00:00:00 2001 From: umairidris Date: Mon, 19 Oct 2020 12:27:42 -0400 Subject: [PATCH 5/5] regen --- examples/tfengine/generated/folder_foundation/audit/main.tf | 3 +++ examples/tfengine/generated/org_foundation/audit/main.tf | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/tfengine/generated/folder_foundation/audit/main.tf b/examples/tfengine/generated/folder_foundation/audit/main.tf index 3b35f81c7..88bc9ecd2 100644 --- a/examples/tfengine/generated/folder_foundation/audit/main.tf +++ b/examples/tfengine/generated/folder_foundation/audit/main.tf @@ -98,6 +98,9 @@ module "storage_export" { include_children = true } +// 6 years minimum audit log retention is required for HIPAA alignment. +// Thus, we lock retention policy to be at least 6 years +// and set the actual expiry to be greater than this amount (7 years). module "storage_destination" { source = "terraform-google-modules/log-export/google//modules/storage" version = "~> 5.0.0" diff --git a/examples/tfengine/generated/org_foundation/audit/main.tf b/examples/tfengine/generated/org_foundation/audit/main.tf index 71cfa148f..179119648 100644 --- a/examples/tfengine/generated/org_foundation/audit/main.tf +++ b/examples/tfengine/generated/org_foundation/audit/main.tf @@ -97,6 +97,9 @@ module "storage_export" { include_children = true } +// 6 years minimum audit log retention is required for HIPAA alignment. +// Thus, we lock retention policy to be at least 6 years +// and set the actual expiry to be greater than this amount (7 years). module "storage_destination" { source = "terraform-google-modules/log-export/google//modules/storage" version = "~> 5.0.0"