diff --git a/aws_datalake/modules/emr/README.md b/aws_datalake/modules/emr/README.md index 270a277..50d0845 100644 --- a/aws_datalake/modules/emr/README.md +++ b/aws_datalake/modules/emr/README.md @@ -54,6 +54,30 @@ Type: `string` Default: `"segment-data-lake"` +### emr\_cluster\_version + +Description: Version of emr cluster + +Type: `string` + +Default: `"6.5.0"` + +### additional\_applications + +Description: List of applications to install on the EMR cluster, besides Hadoop, Hive, and Spark. + +Type: `list(string)` + +Default: `[]` + +### key\_name + +Description: Amazon EC2 key pair that can be used to ssh to the master node as the user called hadoop. + +Type: `string` + +Default: `null` + ### core\_instance\_count Description: Number of Core Nodes @@ -102,6 +126,14 @@ Type: `string` Default: `""` +### additional\_master\_security\_groups + +Description: String containing a comma separated list of additional Amazon EC2 security group IDs for the master node. + +Type: `string` + +Default: `""` + ### slave\_security\_group Description: Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes. @@ -110,6 +142,14 @@ Type: `string` Default: `""` +### additional\_slave\_security\_groups + +Description: String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string. + +Type: `string` + +Default: `""` + ### tags Description: A map of tags to add to all resources. A vendor=segment tag will be added automatically (which is also used by the IAM policy to provide Segment access to submit jobs). @@ -142,6 +182,22 @@ Type: `string` Default: `"m5.xlarge"` +### ebs\_size + +Description: Volume size, in gibibytes (GiB) + +Type: `string` + +Default: `"64"` + +### ebs\_type + +Description: Volume type. Valid options are gp3, gp2, io1, standard, st1 and sc1. + +Type: `string` + +Default: `"gp2"` + ## Outputs The following outputs are exported: diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 5b1fa42..8938d89 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -3,27 +3,31 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { name = var.cluster_name release_label = "emr-${var.emr_cluster_version}" - applications = ["Hadoop", "Hive", "Spark"] + applications = concat(["Hadoop", "Hive", "Spark"], var.additional_applications) log_uri = "s3://${var.s3_bucket}/${var.emr_logs_s3_prefix}" ec2_attributes { subnet_id = var.subnet_id emr_managed_master_security_group = var.master_security_group + additional_master_security_groups = var.additional_master_security_groups emr_managed_slave_security_group = var.slave_security_group + additional_slave_security_groups = var.additional_slave_security_groups instance_profile = var.iam_emr_instance_profile + key_name = var.key_name } service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role + #unhealthy_node_replacement = var.unhealthy_node_replacement master_instance_group { instance_type = var.master_instance_type name = "master_group" ebs_config { - size = "64" - type = "gp2" + size = var.ebs_size + type = var.ebs_type volumes_per_instance = 1 } } @@ -34,8 +38,8 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { name = "core_group" ebs_config { - size = "64" - type = "gp2" + size = var.ebs_size + type = var.ebs_type volumes_per_instance = 1 } @@ -94,22 +98,7 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { EOF } - configurations_json = <