[AWS, AWS-GOV] Added Boolean for Audit Log Delivery

aws-gov/tf/main.tf

@@ -19,6 +19,7 @@ module "sra" {
   # REQUIRED:
   network_configuration          = "isolated" // Network (custom or isolated), see README.md for more information.
   metastore_exists               = false      // If a regional metastore exists set to true.
+  audit_log_delivery_exists      = false      // If audit log delivery is already configured.
 
   # REQUIRED IF USING ISOLATED NETWORK:
   vpc_cidr_range                           = "10.0.0.0/18" // Please re-define the subsequent subnet ranges if the VPC CIDR range is updated.

aws-gov/tf/modules/sra/databricks_account.tf

@@ -67,7 +67,8 @@ module "log_delivery" {
   providers = {
     databricks = databricks.mws
   }
-
+
+  audit_log_delivery_exists      = var.audit_log_delivery_exists
   databricks_account_id          = var.databricks_account_id
   resource_prefix                = var.resource_prefix
   databricks_gov_shard           = var.databricks_gov_shard

aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/main.tf

@@ -2,6 +2,7 @@
 
 # S3 Bucket
 resource "aws_s3_bucket" "logdelivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   bucket        = "${var.resource_prefix}-log-delivery"
   force_destroy = true
   tags = {
@@ -12,7 +13,8 @@ resource "aws_s3_bucket" "logdelivery" {
 
 # S3 Public Access Block
 resource "aws_s3_bucket_public_access_block" "logdelivery" {
-  bucket                  = aws_s3_bucket.logdelivery.id
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket                  = aws_s3_bucket.logdelivery[count.index].id
   block_public_acls       = true
   block_public_policy     = true
   ignore_public_acls      = true
@@ -22,36 +24,39 @@ resource "aws_s3_bucket_public_access_block" "logdelivery" {
 
 # S3 Bucket Versioning
 resource "aws_s3_bucket_versioning" "logdelivery_versioning" {
-  bucket = aws_s3_bucket.logdelivery.id
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket = aws_s3_bucket.logdelivery[count.index].id
   versioning_configuration {
     status = "Disabled"
   }
 }
 
 # Bucket Policy Data Source
 data "databricks_aws_bucket_policy" "logdelivery" {
-  full_access_role = aws_iam_role.logdelivery.arn
-  bucket           = aws_s3_bucket.logdelivery.bucket
+  count = var.audit_log_delivery_exists ? 0 : 1
+  full_access_role = aws_iam_role.logdelivery[count.index].arn
+  bucket           = aws_s3_bucket.logdelivery[count.index].bucket
 }
 
 # Bucket Policy
 resource "aws_s3_bucket_policy" "logdelivery" {
-  bucket = aws_s3_bucket.logdelivery.id
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket = aws_s3_bucket.logdelivery[count.index].id
   policy = jsonencode({
     "Version" : "2012-10-17",
     "Statement" : [
       {
         "Effect" : "Allow",
         "Principal" : {
-          "AWS" : [aws_iam_role.logdelivery.arn]
+          "AWS" : [aws_iam_role.logdelivery[count.index].arn]
         },
         "Action" : "s3:GetBucketLocation",
         "Resource" : "arn:aws-us-gov:s3:::${var.resource_prefix}-log-delivery"
       },
       {
         "Effect" : "Allow",
         "Principal" : {
-          "AWS" : [aws_iam_role.logdelivery.arn]
+          "AWS" : [aws_iam_role.logdelivery[count.index].arn]
         },
         "Action" : [
           "s3:PutObject",
@@ -69,7 +74,7 @@ resource "aws_s3_bucket_policy" "logdelivery" {
       {
         "Effect" : "Allow",
         "Principal" : {
-          "AWS" : [aws_iam_role.logdelivery.arn]
+          "AWS" : [aws_iam_role.logdelivery[count.index].arn]
         },
         "Action" : "s3:ListBucket",
         "Resource" : "arn:aws-us-gov:s3:::${var.resource_prefix}-log-delivery"
@@ -84,6 +89,7 @@ resource "aws_s3_bucket_policy" "logdelivery" {
 
 # Assume Role
 data "aws_iam_policy_document" "passrole_for_log_delivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   statement {
     effect  = "Allow"
     actions = ["sts:AssumeRole"]
@@ -101,9 +107,10 @@ data "aws_iam_policy_document" "passrole_for_log_delivery" {
 
 # IAM Role
 resource "aws_iam_role" "logdelivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   name               = "${var.resource_prefix}-log-delivery-role"
   description        = "(${var.resource_prefix}) UsageDelivery role"
-  assume_role_policy = data.aws_iam_policy_document.passrole_for_log_delivery.json
+  assume_role_policy = data.aws_iam_policy_document.passrole_for_log_delivery[count.index].json
   tags = {
     Name    = "${var.resource_prefix}-logdelivery"
     Project = var.resource_prefix
@@ -112,6 +119,7 @@ resource "aws_iam_role" "logdelivery" {
 
 # Wait for Role
 resource "time_sleep" "wait" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   depends_on = [
     aws_iam_role.logdelivery
   ]
@@ -120,26 +128,29 @@ resource "time_sleep" "wait" {
 
 # Log Credential
 resource "databricks_mws_credentials" "log_writer" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   account_id       = var.databricks_account_id
   credentials_name = "Usage Delivery"
-  role_arn         = aws_iam_role.logdelivery.arn
+  role_arn         = aws_iam_role.logdelivery[count.index].arn
   depends_on = [
     time_sleep.wait
   ]
 }
 
 # Log Storage Configuration
 resource "databricks_mws_storage_configurations" "log_bucket" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   account_id                 = var.databricks_account_id
   storage_configuration_name = "Usage Logs"
-  bucket_name                = aws_s3_bucket.logdelivery.bucket
+  bucket_name                = aws_s3_bucket.logdelivery[count.index].bucket
 }
 
 # Log Delivery
 resource "databricks_mws_log_delivery" "audit_logs" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   account_id               = var.databricks_account_id
-  credentials_id           = databricks_mws_credentials.log_writer.credentials_id
-  storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id
+  credentials_id           = databricks_mws_credentials.log_writer[count.index].credentials_id
+  storage_configuration_id = databricks_mws_storage_configurations.log_bucket[count.index].storage_configuration_id
   delivery_path_prefix     = "audit-logs"
   config_name              = "Audit Logs"
   log_type                 = "AUDIT_LOGS"

aws-gov/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf

@@ -1,3 +1,8 @@
+variable "audit_log_delivery_exists" {
+  description = "If audit log delivery is already configured"
+  type        = bool
+}
+
 variable "databricks_account_id" {
   description = "ID of the Databricks account."
   type = string

aws-gov/tf/modules/sra/variables.tf

@@ -3,6 +3,12 @@ variable "admin_user" {
   type        = string
 }
 
+variable "audit_log_delivery_exists" {
+  description = "If audit log delivery is already configured"
+  type        = bool
+  default     = false
+}
+
 variable "availability_zones" {
   description = "List of AWS availability zones."
   type        = list(string)

aws/tf/main.tf

@@ -5,19 +5,20 @@ module "sra" {
     aws            = aws
   }
 
-  databricks_account_id = var.databricks_account_id
-  client_id             = var.client_id
-  client_secret         = var.client_secret
-  aws_account_id        = var.aws_account_id
-  region                = var.region
-  region_name           = var.region_name[var.region]
-  region_bucket_name    = var.region_bucket_name[var.region]
-  admin_user            = var.admin_user
-  resource_prefix       = var.resource_prefix
+  databricks_account_id     = var.databricks_account_id
+  client_id                 = var.client_id
+  client_secret             = var.client_secret
+  aws_account_id            = var.aws_account_id
+  region                    = var.region
+  region_name               = var.region_name[var.region]
+  region_bucket_name        = var.region_bucket_name[var.region]
+  admin_user                = var.admin_user
+  resource_prefix           = var.resource_prefix
 
   # REQUIRED:
-  network_configuration = "isolated" # Network (custom or isolated), see README.md for more information.
-  metastore_exists      = false      # If a regional metastore exists set to true.
+  network_configuration     = "isolated" # Network (custom or isolated), see README.md for more information.
+  metastore_exists          = false      # If a regional metastore exists set to true.
+  audit_log_delivery_exists = false      # If audit log delivery is already configured.
 
   # REQUIRED - IF USING ISOLATED NETWORK:
   vpc_cidr_range           = "10.0.0.0/18" # Please re-define the subsequent subnet ranges if the VPC CIDR range is updated.

aws/tf/modules/sra/databricks_account.tf

@@ -67,6 +67,7 @@ module "log_delivery" {
     databricks = databricks.mws
   }
 
-  databricks_account_id = var.databricks_account_id
-  resource_prefix       = var.resource_prefix
+  audit_log_delivery_exists = var.audit_log_delivery_exists
+  databricks_account_id     = var.databricks_account_id
+  resource_prefix           = var.resource_prefix
 }

aws/tf/modules/sra/databricks_account/audit_log_delivery/main.tf

@@ -2,6 +2,7 @@
 
 # S3 Bucket
 resource "aws_s3_bucket" "logdelivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   bucket        = "${var.resource_prefix}-log-delivery"
   force_destroy = true
   tags = {
@@ -12,7 +13,8 @@ resource "aws_s3_bucket" "logdelivery" {
 
 # S3 Public Access Block
 resource "aws_s3_bucket_public_access_block" "logdelivery" {
-  bucket                  = aws_s3_bucket.logdelivery.id
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket                  = aws_s3_bucket.logdelivery[count.index].id
   block_public_acls       = true
   block_public_policy     = true
   ignore_public_acls      = true
@@ -22,35 +24,40 @@ resource "aws_s3_bucket_public_access_block" "logdelivery" {
 
 # S3 Bucket Versioning
 resource "aws_s3_bucket_versioning" "logdelivery_versioning" {
-  bucket = aws_s3_bucket.logdelivery.id
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket = aws_s3_bucket.logdelivery[count.index].id
   versioning_configuration {
     status = "Disabled"
   }
 }
 
 # Bucket Policy Data Source
 data "databricks_aws_bucket_policy" "logdelivery" {
-  full_access_role = aws_iam_role.logdelivery.arn
-  bucket           = aws_s3_bucket.logdelivery.bucket
+  count = var.audit_log_delivery_exists ? 0 : 1
+  full_access_role = aws_iam_role.logdelivery[count.index].arn
+  bucket           = aws_s3_bucket.logdelivery[count.index].bucket
 }
 
 # Bucket Policy
 resource "aws_s3_bucket_policy" "logdelivery" {
-  bucket = aws_s3_bucket.logdelivery.id
-  policy = data.databricks_aws_bucket_policy.logdelivery.json
+  count = var.audit_log_delivery_exists ? 0 : 1
+  bucket = aws_s3_bucket.logdelivery[count.index].id
+  policy = data.databricks_aws_bucket_policy.logdelivery[count.index].json
 }
 
 # Assume Role
 data "databricks_aws_assume_role_policy" "logdelivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   external_id      = var.databricks_account_id
   for_log_delivery = true
 }
 
 # IAM Role
 resource "aws_iam_role" "logdelivery" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   name               = "${var.resource_prefix}-log-delivery-role"
   description        = "(${var.resource_prefix}) UsageDelivery role"
-  assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery.json
+  assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery[count.index].json
   tags = {
     Name    = "${var.resource_prefix}-logdelivery"
     Project = var.resource_prefix
@@ -59,6 +66,7 @@ resource "aws_iam_role" "logdelivery" {
 
 # Wait for Role
 resource "time_sleep" "wait" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   depends_on = [
     aws_iam_role.logdelivery
   ]
@@ -67,25 +75,28 @@ resource "time_sleep" "wait" {
 
 # Log Credential
 resource "databricks_mws_credentials" "log_writer" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   credentials_name = "Usage Delivery"
-  role_arn         = aws_iam_role.logdelivery.arn
+  role_arn         = aws_iam_role.logdelivery[count.index].arn
   depends_on = [
     time_sleep.wait
   ]
 }
 
 # Log Storage Configuration
 resource "databricks_mws_storage_configurations" "log_bucket" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   account_id                 = var.databricks_account_id
   storage_configuration_name = "Usage Logs"
-  bucket_name                = aws_s3_bucket.logdelivery.bucket
+  bucket_name                = aws_s3_bucket.logdelivery[count.index].bucket
 }
 
 # Log Delivery
 resource "databricks_mws_log_delivery" "audit_logs" {
+  count = var.audit_log_delivery_exists ? 0 : 1
   account_id               = var.databricks_account_id
-  credentials_id           = databricks_mws_credentials.log_writer.credentials_id
-  storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id
+  credentials_id           = databricks_mws_credentials.log_writer[count.index].credentials_id
+  storage_configuration_id = databricks_mws_storage_configurations.log_bucket[count.index].storage_configuration_id
   delivery_path_prefix     = "audit-logs"
   config_name              = "Audit Logs"
   log_type                 = "AUDIT_LOGS"

aws/tf/modules/sra/databricks_account/audit_log_delivery/variables.tf

@@ -1,3 +1,8 @@
+variable "audit_log_delivery_exists" {
+  description = "If audit log delivery is already configured"
+  type        = bool
+}
+
 variable "databricks_account_id" {
   description = "ID of the Databricks account."
   type        = string

aws/tf/modules/sra/databricks_workspace/classic_cluster/main.tf

@@ -10,14 +10,21 @@ resource "databricks_cluster" "example" {
   cluster_name            = "Shared Classic Compute Plane Cluster"
   data_security_mode      = "USER_ISOLATION"
   spark_version           = data.databricks_spark_version.latest_lts.id
-  node_type_id            = "i3.large"
+  node_type_id            = "m5n.large"
   autotermination_minutes = 10
 
   autoscale {
     min_workers = 1
     max_workers = 2
   }
 
+  aws_attributes {
+    availability           = "ON_DEMAND"
+    ebs_volume_count       = 1
+    ebs_volume_size        = 32  # Size in GB, adjust as needed
+    ebs_volume_type        = "GENERAL_PURPOSE_SSD"
+  }
+
   # Derby Metastore configs
   spark_conf = {
     "spark.hadoop.datanucleus.autoCreateTables" : "true",

aws/tf/modules/sra/variables.tf

@@ -3,6 +3,12 @@ variable "admin_user" {
   type        = string
 }
 
+variable "audit_log_delivery_exists" {
+  description = "If audit log delivery is already configured"
+  type        = bool
+  default     = false
+}
+
 variable "availability_zones" {
   description = "List of AWS availability zones."
   type        = list(string)