From 0ef66cdb4417e577f15f825fff6a5dbdf28963ec Mon Sep 17 00:00:00 2001 From: Antonio Irizarry Date: Fri, 25 Apr 2025 00:37:22 -0400 Subject: [PATCH 1/2] [AWS, AWS-GOV] Added Boolean for Audit Log Delivery Closes #148 --- aws-gov/tf/main.tf | 1 + aws-gov/tf/modules/sra/databricks_account.tf | 3 +- .../audit_log_delivery/main.tf | 39 ++++++++++++------- .../audit_log_delivery/variables.tf | 5 +++ aws-gov/tf/modules/sra/variables.tf | 6 +++ aws/tf/main.tf | 23 +++++------ aws/tf/modules/sra/databricks_account.tf | 5 ++- .../audit_log_delivery/main.tf | 35 +++++++++++------ .../audit_log_delivery/variables.tf | 5 +++ .../classic_cluster/main.tf | 9 ++++- aws/tf/modules/sra/variables.tf | 6 +++ 11 files changed, 96 insertions(+), 41 deletions(-) diff --git a/aws-gov/tf/main.tf b/aws-gov/tf/main.tf index 7823b7bb..dccfafdb 100644 --- a/aws-gov/tf/main.tf +++ b/aws-gov/tf/main.tf @@ -19,6 +19,7 @@ module "sra" { # REQUIRED: network_configuration = "isolated" // Network (custom or isolated), see README.md for more information. metastore_exists = false // If a regional metastore exists set to true. + audit_log_delivery_exists = false // If audit log delivery is already configured. # REQUIRED IF USING ISOLATED NETWORK: vpc_cidr_range = "10.0.0.0/18" // Please re-define the subsequent subnet ranges if the VPC CIDR range is updated. diff --git a/aws-gov/tf/modules/sra/databricks_account.tf b/aws-gov/tf/modules/sra/databricks_account.tf index 23a4e483..f43921dd 100644 --- a/aws-gov/tf/modules/sra/databricks_account.tf +++ b/aws-gov/tf/modules/sra/databricks_account.tf @@ -67,7 +67,8 @@ module "log_delivery" { providers = { databricks = databricks.mws } - + + audit_log_delivery_exists = var.audit_log_delivery_exists databricks_account_id = var.databricks_account_id resource_prefix = var.resource_prefix databricks_gov_shard = var.databricks_gov_shard diff --git a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf index 575a3312..da1612d3 100644 --- a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf +++ b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf @@ -2,6 +2,7 @@ # S3 Bucket resource "aws_s3_bucket" "logdelivery" { + count = var.audit_log_delivery_exists ? 0 : 1 bucket = "${var.resource_prefix}-log-delivery" force_destroy = true tags = { @@ -12,7 +13,8 @@ resource "aws_s3_bucket" "logdelivery" { # S3 Public Access Block resource "aws_s3_bucket_public_access_block" "logdelivery" { - bucket = aws_s3_bucket.logdelivery.id + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id block_public_acls = true block_public_policy = true ignore_public_acls = true @@ -22,7 +24,8 @@ resource "aws_s3_bucket_public_access_block" "logdelivery" { # S3 Bucket Versioning resource "aws_s3_bucket_versioning" "logdelivery_versioning" { - bucket = aws_s3_bucket.logdelivery.id + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id versioning_configuration { status = "Disabled" } @@ -30,20 +33,22 @@ resource "aws_s3_bucket_versioning" "logdelivery_versioning" { # Bucket Policy Data Source data "databricks_aws_bucket_policy" "logdelivery" { - full_access_role = aws_iam_role.logdelivery.arn - bucket = aws_s3_bucket.logdelivery.bucket + count = var.audit_log_delivery_exists ? 0 : 1 + full_access_role = aws_iam_role.logdelivery[count.index].arn + bucket = aws_s3_bucket.logdelivery[count.index].bucket } # Bucket Policy resource "aws_s3_bucket_policy" "logdelivery" { - bucket = aws_s3_bucket.logdelivery.id + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id policy = jsonencode({ "Version" : "2012-10-17", "Statement" : [ { "Effect" : "Allow", "Principal" : { - "AWS" : [aws_iam_role.logdelivery.arn] + "AWS" : [aws_iam_role.logdelivery[count.index].arn] }, "Action" : "s3:GetBucketLocation", "Resource" : "arn:aws-us-gov:s3:::${var.resource_prefix}-log-delivery" @@ -51,7 +56,7 @@ resource "aws_s3_bucket_policy" "logdelivery" { { "Effect" : "Allow", "Principal" : { - "AWS" : [aws_iam_role.logdelivery.arn] + "AWS" : [aws_iam_role.logdelivery[count.index].arn] }, "Action" : [ "s3:PutObject", @@ -69,7 +74,7 @@ resource "aws_s3_bucket_policy" "logdelivery" { { "Effect" : "Allow", "Principal" : { - "AWS" : [aws_iam_role.logdelivery.arn] + "AWS" : [aws_iam_role.logdelivery[count.index].arn] }, "Action" : "s3:ListBucket", "Resource" : "arn:aws-us-gov:s3:::${var.resource_prefix}-log-delivery" @@ -84,6 +89,7 @@ resource "aws_s3_bucket_policy" "logdelivery" { # Assume Role data "aws_iam_policy_document" "passrole_for_log_delivery" { + count = var.audit_log_delivery_exists ? 0 : 1 statement { effect = "Allow" actions = ["sts:AssumeRole"] @@ -101,9 +107,10 @@ data "aws_iam_policy_document" "passrole_for_log_delivery" { # IAM Role resource "aws_iam_role" "logdelivery" { + count = var.audit_log_delivery_exists ? 0 : 1 name = "${var.resource_prefix}-log-delivery-role" description = "(${var.resource_prefix}) UsageDelivery role" - assume_role_policy = data.aws_iam_policy_document.passrole_for_log_delivery.json + assume_role_policy = data.aws_iam_policy_document.passrole_for_log_delivery[count.index].json tags = { Name = "${var.resource_prefix}-logdelivery" Project = var.resource_prefix @@ -112,17 +119,19 @@ resource "aws_iam_role" "logdelivery" { # Wait for Role resource "time_sleep" "wait" { + count = var.audit_log_delivery_exists ? 0 : 1 depends_on = [ - aws_iam_role.logdelivery + aws_iam_role.logdelivery[count.index] ] create_duration = "10s" } # Log Credential resource "databricks_mws_credentials" "log_writer" { + count = var.audit_log_delivery_exists ? 0 : 1 account_id = var.databricks_account_id credentials_name = "Usage Delivery" - role_arn = aws_iam_role.logdelivery.arn + role_arn = aws_iam_role.logdelivery[count.index].arn depends_on = [ time_sleep.wait ] @@ -130,16 +139,18 @@ resource "databricks_mws_credentials" "log_writer" { # Log Storage Configuration resource "databricks_mws_storage_configurations" "log_bucket" { + count = var.audit_log_delivery_exists ? 0 : 1 account_id = var.databricks_account_id storage_configuration_name = "Usage Logs" - bucket_name = aws_s3_bucket.logdelivery.bucket + bucket_name = aws_s3_bucket.logdelivery[count.index].bucket } # Log Delivery resource "databricks_mws_log_delivery" "audit_logs" { + count = var.audit_log_delivery_exists ? 0 : 1 account_id = var.databricks_account_id - credentials_id = databricks_mws_credentials.log_writer.credentials_id - storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + credentials_id = databricks_mws_credentials.log_writer[count.index].credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket[count.index].storage_configuration_id delivery_path_prefix = "audit-logs" config_name = "Audit Logs" log_type = "AUDIT_LOGS" diff --git a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf index f80801e9..42659525 100644 --- a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf +++ b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf @@ -1,3 +1,8 @@ +variable "audit_log_delivery_exists" { + description = "If audit log delivery is already configured" + type = bool +} + variable "databricks_account_id" { description = "ID of the Databricks account." type = string diff --git a/aws-gov/tf/modules/sra/variables.tf b/aws-gov/tf/modules/sra/variables.tf index a255f40d..b8c703b2 100644 --- a/aws-gov/tf/modules/sra/variables.tf +++ b/aws-gov/tf/modules/sra/variables.tf @@ -3,6 +3,12 @@ variable "admin_user" { type = string } +variable "audit_log_delivery_exists" { + description = "If audit log delivery is already configured" + type = bool + default = false +} + variable "availability_zones" { description = "List of AWS availability zones." type = list(string) diff --git a/aws/tf/main.tf b/aws/tf/main.tf index ebcf5678..2403d286 100644 --- a/aws/tf/main.tf +++ b/aws/tf/main.tf @@ -5,19 +5,20 @@ module "sra" { aws = aws } - databricks_account_id = var.databricks_account_id - client_id = var.client_id - client_secret = var.client_secret - aws_account_id = var.aws_account_id - region = var.region - region_name = var.region_name[var.region] - region_bucket_name = var.region_bucket_name[var.region] - admin_user = var.admin_user - resource_prefix = var.resource_prefix + databricks_account_id = var.databricks_account_id + client_id = var.client_id + client_secret = var.client_secret + aws_account_id = var.aws_account_id + region = var.region + region_name = var.region_name[var.region] + region_bucket_name = var.region_bucket_name[var.region] + admin_user = var.admin_user + resource_prefix = var.resource_prefix # REQUIRED: - network_configuration = "isolated" # Network (custom or isolated), see README.md for more information. - metastore_exists = false # If a regional metastore exists set to true. + network_configuration = "isolated" # Network (custom or isolated), see README.md for more information. + metastore_exists = false # If a regional metastore exists set to true. + audit_log_delivery_exists = false # If audit log delivery is already configured. # REQUIRED - IF USING ISOLATED NETWORK: vpc_cidr_range = "10.0.0.0/18" # Please re-define the subsequent subnet ranges if the VPC CIDR range is updated. diff --git a/aws/tf/modules/sra/databricks_account.tf b/aws/tf/modules/sra/databricks_account.tf index 7b534d54..1d511590 100644 --- a/aws/tf/modules/sra/databricks_account.tf +++ b/aws/tf/modules/sra/databricks_account.tf @@ -67,6 +67,7 @@ module "log_delivery" { databricks = databricks.mws } - databricks_account_id = var.databricks_account_id - resource_prefix = var.resource_prefix + audit_log_delivery_exists = var.audit_log_delivery_exists + databricks_account_id = var.databricks_account_id + resource_prefix = var.resource_prefix } \ No newline at end of file diff --git a/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf b/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf index f1854942..d440aeac 100644 --- a/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf +++ b/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf @@ -2,6 +2,7 @@ # S3 Bucket resource "aws_s3_bucket" "logdelivery" { + count = var.audit_log_delivery_exists ? 0 : 1 bucket = "${var.resource_prefix}-log-delivery" force_destroy = true tags = { @@ -12,7 +13,8 @@ resource "aws_s3_bucket" "logdelivery" { # S3 Public Access Block resource "aws_s3_bucket_public_access_block" "logdelivery" { - bucket = aws_s3_bucket.logdelivery.id + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id block_public_acls = true block_public_policy = true ignore_public_acls = true @@ -22,7 +24,8 @@ resource "aws_s3_bucket_public_access_block" "logdelivery" { # S3 Bucket Versioning resource "aws_s3_bucket_versioning" "logdelivery_versioning" { - bucket = aws_s3_bucket.logdelivery.id + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id versioning_configuration { status = "Disabled" } @@ -30,27 +33,31 @@ resource "aws_s3_bucket_versioning" "logdelivery_versioning" { # Bucket Policy Data Source data "databricks_aws_bucket_policy" "logdelivery" { - full_access_role = aws_iam_role.logdelivery.arn - bucket = aws_s3_bucket.logdelivery.bucket + count = var.audit_log_delivery_exists ? 0 : 1 + full_access_role = aws_iam_role.logdelivery[count.index].arn + bucket = aws_s3_bucket.logdelivery[count.index].bucket } # Bucket Policy resource "aws_s3_bucket_policy" "logdelivery" { - bucket = aws_s3_bucket.logdelivery.id - policy = data.databricks_aws_bucket_policy.logdelivery.json + count = var.audit_log_delivery_exists ? 0 : 1 + bucket = aws_s3_bucket.logdelivery[count.index].id + policy = data.databricks_aws_bucket_policy.logdelivery[count.index].json } # Assume Role data "databricks_aws_assume_role_policy" "logdelivery" { + count = var.audit_log_delivery_exists ? 0 : 1 external_id = var.databricks_account_id for_log_delivery = true } # IAM Role resource "aws_iam_role" "logdelivery" { + count = var.audit_log_delivery_exists ? 0 : 1 name = "${var.resource_prefix}-log-delivery-role" description = "(${var.resource_prefix}) UsageDelivery role" - assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery.json + assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery[count.index].json tags = { Name = "${var.resource_prefix}-logdelivery" Project = var.resource_prefix @@ -59,16 +66,18 @@ resource "aws_iam_role" "logdelivery" { # Wait for Role resource "time_sleep" "wait" { + count = var.audit_log_delivery_exists ? 0 : 1 depends_on = [ - aws_iam_role.logdelivery + aws_iam_role.logdelivery[count.index] ] create_duration = "10s" } # Log Credential resource "databricks_mws_credentials" "log_writer" { + count = var.audit_log_delivery_exists ? 0 : 1 credentials_name = "Usage Delivery" - role_arn = aws_iam_role.logdelivery.arn + role_arn = aws_iam_role.logdelivery[count.index].arn depends_on = [ time_sleep.wait ] @@ -76,16 +85,18 @@ resource "databricks_mws_credentials" "log_writer" { # Log Storage Configuration resource "databricks_mws_storage_configurations" "log_bucket" { + count = var.audit_log_delivery_exists ? 0 : 1 account_id = var.databricks_account_id storage_configuration_name = "Usage Logs" - bucket_name = aws_s3_bucket.logdelivery.bucket + bucket_name = aws_s3_bucket.logdelivery[count.index].bucket } # Log Delivery resource "databricks_mws_log_delivery" "audit_logs" { + count = var.audit_log_delivery_exists ? 0 : 1 account_id = var.databricks_account_id - credentials_id = databricks_mws_credentials.log_writer.credentials_id - storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + credentials_id = databricks_mws_credentials.log_writer[count.index].credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket[count.index].storage_configuration_id delivery_path_prefix = "audit-logs" config_name = "Audit Logs" log_type = "AUDIT_LOGS" diff --git a/aws/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf b/aws/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf index d286f110..c1c7a821 100644 --- a/aws/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf +++ b/aws/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf @@ -1,3 +1,8 @@ +variable "audit_log_delivery_exists" { + description = "If audit log delivery is already configured" + type = bool +} + variable "databricks_account_id" { description = "ID of the Databricks account." type = string diff --git a/aws/tf/modules/sra/databricks_workspace/classic_cluster/main.tf b/aws/tf/modules/sra/databricks_workspace/classic_cluster/main.tf index d336517d..06b26b2b 100644 --- a/aws/tf/modules/sra/databricks_workspace/classic_cluster/main.tf +++ b/aws/tf/modules/sra/databricks_workspace/classic_cluster/main.tf @@ -10,7 +10,7 @@ resource "databricks_cluster" "example" { cluster_name = "Shared Classic Compute Plane Cluster" data_security_mode = "USER_ISOLATION" spark_version = data.databricks_spark_version.latest_lts.id - node_type_id = "i3.large" + node_type_id = "m5n.large" autotermination_minutes = 10 autoscale { @@ -18,6 +18,13 @@ resource "databricks_cluster" "example" { max_workers = 2 } + aws_attributes { + availability = "ON_DEMAND" + ebs_volume_count = 1 + ebs_volume_size = 32 # Size in GB, adjust as needed + ebs_volume_type = "GENERAL_PURPOSE_SSD" + } + # Derby Metastore configs spark_conf = { "spark.hadoop.datanucleus.autoCreateTables" : "true", diff --git a/aws/tf/modules/sra/variables.tf b/aws/tf/modules/sra/variables.tf index 1456c330..02e90c3c 100644 --- a/aws/tf/modules/sra/variables.tf +++ b/aws/tf/modules/sra/variables.tf @@ -3,6 +3,12 @@ variable "admin_user" { type = string } +variable "audit_log_delivery_exists" { + description = "If audit log delivery is already configured" + type = bool + default = false +} + variable "availability_zones" { description = "List of AWS availability zones." type = list(string) From c86e0c333e647793b35e2a6021c6502bdd929610 Mon Sep 17 00:00:00 2001 From: Antonio Irizarry Date: Fri, 25 Apr 2025 00:40:15 -0400 Subject: [PATCH 2/2] [AWS, AWS-GOV] Added Boolean for Audit Log Delivery --- .../modules/sra/databricks_account/audit_log_delivery/main.tf | 2 +- .../modules/sra/databricks_account/audit_log_delivery/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf index da1612d3..7293fe54 100644 --- a/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf +++ b/aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf @@ -121,7 +121,7 @@ resource "aws_iam_role" "logdelivery" { resource "time_sleep" "wait" { count = var.audit_log_delivery_exists ? 0 : 1 depends_on = [ - aws_iam_role.logdelivery[count.index] + aws_iam_role.logdelivery ] create_duration = "10s" } diff --git a/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf b/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf index d440aeac..71ebe8b7 100644 --- a/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf +++ b/aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf @@ -68,7 +68,7 @@ resource "aws_iam_role" "logdelivery" { resource "time_sleep" "wait" { count = var.audit_log_delivery_exists ? 0 : 1 depends_on = [ - aws_iam_role.logdelivery[count.index] + aws_iam_role.logdelivery ] create_duration = "10s" }