diff --git a/README.md b/README.md index 13f32ea..0000709 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ provider "aws" { # Replace this with the AWS region your infrastructure is set up in. region = "us-west-2" - # Currently our modules require the older v2 AWS provider, as upgrading to v3 has notable breaking changes. - version = "~> 2" + # Currently our modules support both the version 2 and version 3 of the AWS provider. + version = "~> 3" } locals { @@ -103,8 +103,8 @@ module "iam" { # names. suffix = "-prod" - s3_bucket = "${aws_s3_bucket.segment_datalake_s3.id}" - external_ids = "${values(local.external_ids)}" + s3_bucket = aws_s3_bucket.segment_datalake_s3.id + external_ids = values(local.external_ids) } # Creates an EMR Cluster that Segment uses for performing the final ETL on your @@ -112,13 +112,13 @@ module "iam" { module "emr" { source = "git@github.com:segmentio/terraform-aws-data-lake//modules/emr?ref=v0.4.1" - s3_bucket = "${aws_s3_bucket.segment_datalake_s3.id}" + s3_bucket = aws_s3_bucket.segment_datalake_s3.id subnet_id = "subnet-XXX" # Replace this with the subnet ID you want the EMR cluster to run in. # LEAVE THIS AS-IS - iam_emr_autoscaling_role = "${module.iam.iam_emr_autoscaling_role}" - iam_emr_service_role = "${module.iam.iam_emr_service_role}" - iam_emr_instance_profile = "${module.iam.iam_emr_instance_profile}" + iam_emr_autoscaling_role = module.iam.iam_emr_autoscaling_role + iam_emr_service_role = module.iam.iam_emr_service_role + iam_emr_instance_profile = module.iam.iam_emr_instance_profile } ``` diff --git a/modules/emr/main.tf b/modules/emr/main.tf index c4f63db..26e4f49 100644 --- a/modules/emr/main.tf +++ b/modules/emr/main.tf @@ -1,25 +1,25 @@ # Creates an EMR cluster that will be used to transform and load events into the Data Lake. # https://www.terraform.io/docs/providers/aws/r/emr_cluster.html resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { - name = "${var.cluster_name}" + name = var.cluster_name release_label = "emr-5.27.0" applications = ["Hadoop", "Hive", "Spark"] log_uri = "s3://${var.s3_bucket}/${var.emr_logs_s3_prefix}" ec2_attributes { - subnet_id = "${var.subnet_id}" - emr_managed_master_security_group = "${var.master_security_group}" - emr_managed_slave_security_group = "${var.slave_security_group}" - instance_profile = "${var.iam_emr_instance_profile}" + subnet_id = var.subnet_id + emr_managed_master_security_group = var.master_security_group + emr_managed_slave_security_group = var.slave_security_group + instance_profile = var.iam_emr_instance_profile } - service_role = "${var.iam_emr_service_role}" - autoscaling_role = "${var.iam_emr_autoscaling_role}" + service_role = var.iam_emr_service_role + autoscaling_role = var.iam_emr_autoscaling_role master_instance_group { - instance_type = "${var.master_instance_type}" - name = "master_group" + instance_type = var.master_instance_type + name = var.master_instance_name ebs_config { size = "64" @@ -29,9 +29,9 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" { } core_instance_group { - instance_type = "${var.core_instance_type}" - instance_count = "${var.core_instance_count}" - name = "core_group" + instance_type = var.core_instance_type + instance_count = var.core_instance_count + name = var.core_instance_name ebs_config { size = "64" @@ -110,15 +110,15 @@ EOF ] EOF - tags = "${local.tags}" + tags = local.tags } resource "aws_emr_instance_group" "task" { - name = "task_group" - cluster_id = "${aws_emr_cluster.segment_data_lake_emr_cluster.id}" + name = var.task_group_name + cluster_id = aws_emr_cluster.segment_data_lake_emr_cluster.id - instance_type = "${var.task_instance_type}" - instance_count = "${var.task_instance_count}" + instance_type = var.task_instance_type + instance_count = var.task_instance_count ebs_config { size = "64" diff --git a/modules/emr/output.tf b/modules/emr/output.tf index 497b6cc..1388287 100644 --- a/modules/emr/output.tf +++ b/modules/emr/output.tf @@ -1,3 +1,3 @@ output "cluster_id" { - value = "${aws_emr_cluster.segment_data_lake_emr_cluster.id}" + value = aws_emr_cluster.segment_data_lake_emr_cluster.id } diff --git a/modules/emr/variables.tf b/modules/emr/variables.tf index 565111c..d39cc00 100644 --- a/modules/emr/variables.tf +++ b/modules/emr/variables.tf @@ -1,100 +1,119 @@ variable "s3_bucket" { description = "Name of the S3 bucket used by the Data Lake. The EMR cluster will be configured to store logs in this bucket." - type = "string" + type = string } variable "subnet_id" { description = "VPC subnet id where you want the job flow to launch. Cannot specify the cc1.4xlarge instance type for nodes of a job flow launched in a Amazon VPC." - type = "string" + type = string } variable "master_security_group" { description = "Identifier of the Amazon EC2 EMR-Managed security group for the master node." - type = "string" + type = string default = "" } variable "slave_security_group" { description = "Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes." - type = "string" + type = string default = "" } variable "tags" { description = "A map of tags to add to all resources. A vendor=segment tag will be added automatically (which is also used by the IAM policy to provide Segment access to submit jobs)." - type = "map" + type = map(string) default = {} } variable "cluster_name" { description = "Name of the EMR cluster that the module creates" - type = "string" + type = string default = "segment-data-lake" } +variable "master_instance_name" { + description = "Name of the master instance group." + type = string + default = "master_group" +} + +variable "core_instance_name" { + description = "Name of the core instance group." + type = string + default = "core_group" +} + +variable "task_group_name" { + description = "Name of the task group." + type = string + default = "task_group" +} + + variable "emr_logs_s3_prefix" { description = "Prefix for writing EMR cluster logs to S3. Make sure to include a trailing slash (/) when setting this." - type = "string" + type = string default = "logs/" } variable "iam_emr_service_role" { description = "Name of the EMR service role" - type = "string" + type = string } variable "iam_emr_autoscaling_role" { description = "Name of the EMR autoscaling role" - type = "string" + type = string } variable "iam_emr_instance_profile" { description = "Name of the EMR EC2 instance profile" - type = "string" + type = string } variable "master_instance_type" { description = "EC2 Instance Type for Master" - type = "string" + type = string default = "m5.xlarge" } variable "core_instance_type" { description = "EC2 Instance Type for Core Nodes" - type = "string" + type = string default = "m5.xlarge" } variable "task_instance_type" { description = "EC2 Instance Type for Task Nodes" - type = "string" + type = string default = "m5.xlarge" } variable "core_instance_count" { description = "Number of Core Nodes" - type = "string" + type = string default = "2" } variable "core_instance_max_count" { description = "Max number of Core Nodes used on autoscale" - type = "string" + type = string default = "4" } variable "task_instance_count" { description = "Number of instances of Task Nodes" - type = "string" + type = string default = "2" } variable "task_instance_max_count" { description = "Max number of Task Nodes used on autoscale" - type = "string" + type = string default = "4" } locals { - tags = "${merge(map("vendor", "segment"), var.tags)}" + tags = merge(map("vendor", "segment"), var.tags) } diff --git a/modules/glue/main.tf b/modules/glue/main.tf index 5f56492..b80aaf3 100644 --- a/modules/glue/main.tf +++ b/modules/glue/main.tf @@ -4,6 +4,6 @@ # makes the schema available to various tools like Athena, Spectrum, EMR etc. # https://www.terraform.io/docs/providers/aws/r/glue_catalog_database.html resource "aws_glue_catalog_database" "segment_data_lake_glue_catalog" { - name = "${var.name}" - description = "${var.description}" + name = var.name + description = var.description } diff --git a/modules/glue/output.tf b/modules/glue/output.tf index 0b14635..7562387 100644 --- a/modules/glue/output.tf +++ b/modules/glue/output.tf @@ -1,3 +1,3 @@ output "database_name" { - value = "${aws_glue_catalog_database.segment_data_lake_glue_catalog.name}" + value = aws_glue_catalog_database.segment_data_lake_glue_catalog.name } diff --git a/modules/glue/variables.tf b/modules/glue/variables.tf index 87aa8f6..6829dae 100644 --- a/modules/glue/variables.tf +++ b/modules/glue/variables.tf @@ -1,10 +1,10 @@ variable "name" { description = "The name of the database." - type = "string" + type = string } variable "description" { description = "Description of the database." - type = "string" + type = string default = "Segment Data Lake" } diff --git a/modules/iam/main.tf b/modules/iam/main.tf index 969ee0d..2d08fd7 100644 --- a/modules/iam/main.tf +++ b/modules/iam/main.tf @@ -3,8 +3,8 @@ resource "aws_iam_role" "segment_data_lake_iam_role" { name = "SegmentDataLakeRole${var.suffix}" description = "IAM Role used by Segment" - assume_role_policy = "${data.aws_iam_policy_document.segment_data_lake_assume_role_policy_document.json}" - tags = "${local.tags}" + assume_role_policy = data.aws_iam_policy_document.segment_data_lake_assume_role_policy_document.json + tags = local.tags } # Policy attached to the IAM role. @@ -20,7 +20,7 @@ data "aws_iam_policy_document" "segment_data_lake_assume_role_policy_document" { principals { type = "AWS" - identifiers = "${var.segment_aws_accounts}" + identifiers = var.segment_aws_accounts } effect = "Allow" @@ -28,7 +28,7 @@ data "aws_iam_policy_document" "segment_data_lake_assume_role_policy_document" { condition { test = "StringEquals" variable = "sts:ExternalId" - values = "${var.external_ids}" + values = var.external_ids } } } @@ -43,7 +43,7 @@ resource "aws_iam_policy" "segment_data_lake_policy" { name = "SegmentDataLakePolicy${var.suffix}" path = "/" description = "Gives access to resources in your Data Lake" - policy = "${data.aws_iam_policy_document.segment_data_lake_policy_document.json}" + policy = data.aws_iam_policy_document.segment_data_lake_policy_document.json } data "aws_iam_policy_document" "segment_data_lake_policy_document" { @@ -144,9 +144,9 @@ data "aws_iam_policy_document" "segment_data_lake_policy_document" { ] resources = [ - "${aws_iam_role.segment_emr_service_role.arn}", - "${aws_iam_role.segment_emr_instance_profile_role.arn}", - "${aws_iam_role.segment_emr_autoscaling_role.arn}", + aws_iam_role.segment_emr_service_role.arn, + aws_iam_role.segment_emr_instance_profile_role.arn, + aws_iam_role.segment_emr_autoscaling_role.arn, ] effect = "Allow" @@ -154,8 +154,8 @@ data "aws_iam_policy_document" "segment_data_lake_policy_document" { } resource "aws_iam_role_policy_attachment" "segment_data_lake_role_policy_attachment" { - role = "${aws_iam_role.segment_data_lake_iam_role.name}" - policy_arn = "${aws_iam_policy.segment_data_lake_policy.arn}" + role = aws_iam_role.segment_data_lake_iam_role.name + policy_arn = aws_iam_policy.segment_data_lake_policy.arn } # IAM role for EMR Service @@ -178,12 +178,12 @@ resource "aws_iam_role" "segment_emr_service_role" { } EOF - tags = "${local.tags}" + tags = local.tags } resource "aws_iam_role_policy" "segment_emr_service_policy" { name = "SegmentEMRServicePolicy${var.suffix}" - role = "${aws_iam_role.segment_emr_service_role.id}" + role = aws_iam_role.segment_emr_service_role.id policy = <