From d08ae966468463e2e381b79d01e52c25d3638922 Mon Sep 17 00:00:00 2001 From: Ken Michalak Date: Tue, 21 May 2024 13:30:05 -0500 Subject: [PATCH 1/2] add variables with defaults to AWS EMR module --- aws_datalake/modules/emr/README.md | 56 +++++++++++++++++++++++++++ aws_datalake/modules/emr/main.tf | 21 ++++++---- aws_datalake/modules/emr/variables.tf | 43 ++++++++++++++++++++ 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/aws_datalake/modules/emr/README.md b/aws_datalake/modules/emr/README.md index 270a277..50d0845 100644 --- a/aws_datalake/modules/emr/README.md +++ b/aws_datalake/modules/emr/README.md @@ -54,6 +54,30 @@ Type: `string` Default: `"segment-data-lake"` +### emr\_cluster\_version + +Description: Version of emr cluster + +Type: `string` + +Default: `"6.5.0"` + +### additional\_applications + +Description: List of applications to install on the EMR cluster, besides Hadoop, Hive, and Spark. + +Type: `list(string)` + +Default: `[]` + +### key\_name + +Description: Amazon EC2 key pair that can be used to ssh to the master node as the user called hadoop. + +Type: `string` + +Default: `null` + ### core\_instance\_count Description: Number of Core Nodes @@ -102,6 +126,14 @@ Type: `string` Default: `""` +### additional\_master\_security\_groups + +Description: String containing a comma separated list of additional Amazon EC2 security group IDs for the master node. + +Type: `string` + +Default: `""` + ### slave\_security\_group Description: Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes. @@ -110,6 +142,14 @@ Type: `string` Default: `""` +### additional\_slave\_security\_groups + +Description: String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string. + +Type: `string` + +Default: `""` + ### tags Description: A map of tags to add to all resources. A vendor=segment tag will be added automatically (which is also used by the IAM policy to provide Segment access to submit jobs). @@ -142,6 +182,22 @@ Type: `string` Default: `"m5.xlarge"` +### ebs\_size + +Description: Volume size, in gibibytes (GiB) + +Type: `string` + +Default: `"64"` + +### ebs\_type + +Description: Volume type. Valid options are gp3, gp2, io1, standard, st1 and sc1. + +Type: `string` + +Default: `"gp2"` + ## Outputs The following outputs are exported: diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 5b1fa42..71f5d5d 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -3,27 +3,31 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { name = var.cluster_name release_label = "emr-${var.emr_cluster_version}" - applications = ["Hadoop", "Hive", "Spark"] + applications = concat(["Hadoop", "Hive", "Spark"], var.additional_applications) log_uri = "s3://${var.s3_bucket}/${var.emr_logs_s3_prefix}" ec2_attributes { subnet_id = var.subnet_id emr_managed_master_security_group = var.master_security_group + additional_master_security_groups = var.additional_master_security_groups emr_managed_slave_security_group = var.slave_security_group + additional_slave_security_groups = var.additional_slave_security_groups instance_profile = var.iam_emr_instance_profile + key_name = var.key_name } service_role = var.iam_emr_service_role autoscaling_role = var.iam_emr_autoscaling_role + #unhealthy_node_replacement = var.unhealthy_node_replacement master_instance_group { instance_type = var.master_instance_type name = "master_group" ebs_config { - size = "64" - type = "gp2" + size = var.ebs_size + type = var.ebs_type volumes_per_instance = 1 } } @@ -34,8 +38,8 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { name = "core_group" ebs_config { - size = "64" - type = "gp2" + size = var.ebs_size + type = var.ebs_type volumes_per_instance = 1 } @@ -105,7 +109,8 @@ EOF { "Classification": "spark-hive-site", "Properties": { - "hive.metastore.client.factory.class":"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory" } + "hive.metastore.client.factory.class":"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory" + } } ] EOF @@ -121,8 +126,8 @@ resource "aws_emr_instance_group" "task" { instance_count = var.task_instance_count ebs_config { - size = "64" - type = "gp2" + size = var.ebs_size + type = var.ebs_type volumes_per_instance = 1 } diff --git a/aws_datalake/modules/emr/variables.tf b/aws_datalake/modules/emr/variables.tf index fd87b13..204cbec 100644 --- a/aws_datalake/modules/emr/variables.tf +++ b/aws_datalake/modules/emr/variables.tf @@ -14,12 +14,24 @@ variable "master_security_group" { default = "" } +variable "additional_master_security_groups" { + description = "String containing a comma separated list of additional Amazon EC2 security group IDs for the master node." + type = string + default = "" +} + variable "slave_security_group" { description = "Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes." type = string default = "" } +variable "additional_slave_security_groups" { + description = "String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string." + type = string + default = "" +} + variable "tags" { description = "A map of tags to add to all resources. A vendor=segment tag will be added automatically (which is also used by the IAM policy to provide Segment access to submit jobs)." type = map(string) @@ -53,6 +65,19 @@ variable "iam_emr_instance_profile" { type = string } +variable "key_name" { + description = "Amazon EC2 key pair that can be used to ssh to the master node as the user called hadoop." + type = string + default = null +} + +# FIXME requires aws provider v5 +#variable "unhealthy_node_replacement" { +# description = "Whether Amazon EMR should gracefully replace core nodes that have degraded within the cluster." +# type = bool +# default = false +#} + variable "master_instance_type" { description = "EC2 Instance Type for Master" type = string @@ -101,6 +126,24 @@ variable "emr_cluster_version" { default = "6.5.0" } +variable "additional_applications" { + description = "List of applications to install on the EMR cluster, besides Hadoop, Hive, and Spark." + type = list(string) + default = [] +} + +variable "ebs_size" { + description = "Volume size, in gibibytes (GiB)" + type = string + default = "64" +} + +variable "ebs_type" { + description = "Volume type. Valid options are gp3, gp2, io1, standard, st1 and sc1." + type = string + default = "gp2" +} + locals { tags = merge(tomap({"vendor" = "segment"}), var.tags) } From acb265e1408666c90d8788a77b53a16e754345bf Mon Sep 17 00:00:00 2001 From: Ken Michalak Date: Fri, 31 May 2024 11:07:10 -0500 Subject: [PATCH 2/2] add variable for configurations_json and fix disk space issue Without ssh access, there's no way to clean up Spark's job history, and the cleaner is disabled by default. This enables the cleaner with the default values and allows modifications. https://repost.aws/knowledge-center/core-node-emr-cluster-disk-space https://spark.apache.org/docs/latest/monitoring.html#spark-history-server-configuration-options --- aws_datalake/modules/emr/main.tf | 18 +---------------- aws_datalake/modules/emr/variables.tf | 29 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/aws_datalake/modules/emr/main.tf b/aws_datalake/modules/emr/main.tf index 71f5d5d..8938d89 100644 --- a/aws_datalake/modules/emr/main.tf +++ b/aws_datalake/modules/emr/main.tf @@ -98,23 +98,7 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { EOF } - configurations_json = <