Upgrade to AWS provider 3.0 and add EMR resource names to be configurable

README.md

@@ -58,8 +58,8 @@ provider "aws" {
   # Replace this with the AWS region your infrastructure is set up in.
   region = "us-west-2"
 
-  # Currently our modules require the older v2 AWS provider, as upgrading to v3 has notable breaking changes.
-  version = "~> 2"
+  # Currently our modules support both the version 2 and version 3 of the AWS provider.
+  version = "~> 3"
 }
 
 locals {
@@ -103,22 +103,22 @@ module "iam" {
   # names.
   suffix = "-prod"
 
-  s3_bucket    = "${aws_s3_bucket.segment_datalake_s3.id}"
-  external_ids = "${values(local.external_ids)}"
+  s3_bucket    = aws_s3_bucket.segment_datalake_s3.id
+  external_ids = values(local.external_ids)
 }
 
 # Creates an EMR Cluster that Segment uses for performing the final ETL on your
 # data that lands in S3.
 module "emr" {
   source = "git@github.com:segmentio/terraform-aws-data-lake//modules/emr?ref=v0.4.1"
 
-  s3_bucket = "${aws_s3_bucket.segment_datalake_s3.id}"
+  s3_bucket = aws_s3_bucket.segment_datalake_s3.id
   subnet_id = "subnet-XXX" # Replace this with the subnet ID you want the EMR cluster to run in.
 
   # LEAVE THIS AS-IS
-  iam_emr_autoscaling_role = "${module.iam.iam_emr_autoscaling_role}"
-  iam_emr_service_role     = "${module.iam.iam_emr_service_role}"
-  iam_emr_instance_profile = "${module.iam.iam_emr_instance_profile}"
+  iam_emr_autoscaling_role = module.iam.iam_emr_autoscaling_role
+  iam_emr_service_role     = module.iam.iam_emr_service_role
+  iam_emr_instance_profile = module.iam.iam_emr_instance_profile
 }
 ```
 

modules/emr/main.tf

@@ -1,25 +1,25 @@
 # Creates an EMR cluster that will be used to transform and load events into the Data Lake.
 # https://www.terraform.io/docs/providers/aws/r/emr_cluster.html
 resource "aws_emr_cluster" "segment_data_lake_emr_cluster" {
-  name          = "${var.cluster_name}"
+  name          = var.cluster_name
   release_label = "emr-5.27.0"
   applications  = ["Hadoop", "Hive", "Spark"]
 
   log_uri = "s3://${var.s3_bucket}/${var.emr_logs_s3_prefix}"
 
   ec2_attributes {
-    subnet_id                         = "${var.subnet_id}"
-    emr_managed_master_security_group = "${var.master_security_group}"
-    emr_managed_slave_security_group  = "${var.slave_security_group}"
-    instance_profile                  = "${var.iam_emr_instance_profile}"
+    subnet_id                         = var.subnet_id
+    emr_managed_master_security_group = var.master_security_group
+    emr_managed_slave_security_group  = var.slave_security_group
+    instance_profile                  = var.iam_emr_instance_profile
   }
 
-  service_role     = "${var.iam_emr_service_role}"
-  autoscaling_role = "${var.iam_emr_autoscaling_role}"
+  service_role     = var.iam_emr_service_role
+  autoscaling_role = var.iam_emr_autoscaling_role
 
   master_instance_group {
-    instance_type = "${var.master_instance_type}"
-    name          = "master_group"
+    instance_type = var.master_instance_type
+    name          = var.master_instance_name
 
     ebs_config {
       size                 = "64"
@@ -29,9 +29,9 @@ resource "aws_emr_cluster" "segment_data_lake_emr_cluster" {
   }
 
   core_instance_group {
-    instance_type  = "${var.core_instance_type}"
-    instance_count = "${var.core_instance_count}"
-    name           = "core_group"
+    instance_type  = var.core_instance_type
+    instance_count = var.core_instance_count
+    name           = var.core_instance_name
 
     ebs_config {
       size                 = "64"
@@ -110,15 +110,15 @@ EOF
   ]
 EOF
 
-  tags = "${local.tags}"
+  tags = local.tags
 }
 
 resource "aws_emr_instance_group" "task" {
-  name       = "task_group"
-  cluster_id = "${aws_emr_cluster.segment_data_lake_emr_cluster.id}"
+  name       = var.task_group_name
+  cluster_id = aws_emr_cluster.segment_data_lake_emr_cluster.id
 
-  instance_type  = "${var.task_instance_type}"
-  instance_count = "${var.task_instance_count}"
+  instance_type  = var.task_instance_type
+  instance_count = var.task_instance_count
 
   ebs_config {
     size                 = "64"

modules/emr/output.tf

@@ -1,3 +1,3 @@
 output "cluster_id" {
-  value = "${aws_emr_cluster.segment_data_lake_emr_cluster.id}"
+  value = aws_emr_cluster.segment_data_lake_emr_cluster.id
 }

modules/emr/variables.tf

@@ -1,100 +1,119 @@
 variable "s3_bucket" {
   description = "Name of the S3 bucket used by the Data Lake. The EMR cluster will be configured to store logs in this bucket."
-  type        = "string"
+  type        = string
 }
 
 variable "subnet_id" {
   description = "VPC subnet id where you want the job flow to launch. Cannot specify the cc1.4xlarge instance type for nodes of a job flow launched in a Amazon VPC."
-  type        = "string"
+  type        = string
 }
 
 variable "master_security_group" {
   description = "Identifier of the Amazon EC2 EMR-Managed security group for the master node."
-  type        = "string"
+  type        = string
   default     = ""
 }
 
 variable "slave_security_group" {
   description = "Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes."
-  type        = "string"
+  type        = string
   default     = ""
 }
 
 variable "tags" {
   description = "A map of tags to add to all resources. A vendor=segment tag will be added automatically (which is also used by the IAM policy to provide Segment access to submit jobs)."
-  type        = "map"
+  type        = map(string)
   default     = {}
 }
 
 variable "cluster_name" {
   description = "Name of the EMR cluster that the module creates"
-  type        = "string"
+  type        = string
   default     = "segment-data-lake"
 }
 
+variable "master_instance_name" {
+  description = "Name of the master instance group."
+  type        = string
+  default     = "master_group"
+}
+
+variable "core_instance_name" {
+  description = "Name of the core instance group."
+  type        = string
+  default     = "core_group"
+}
+
+variable "task_group_name" {
+  description = "Name of the task group."
+  type        = string
+  default     = "task_group"
+}
+
+
 variable "emr_logs_s3_prefix" {
   description = "Prefix for writing EMR cluster logs to S3. Make sure to include a trailing slash (/) when setting this."
-  type        = "string"
+  type        = string
   default     = "logs/"
 }
 
 variable "iam_emr_service_role" {
   description = "Name of the EMR service role"
-  type        = "string"
+  type        = string
 }
 
 variable "iam_emr_autoscaling_role" {
   description = "Name of the EMR autoscaling role"
-  type        = "string"
+  type        = string
 }
 
 variable "iam_emr_instance_profile" {
   description = "Name of the EMR EC2 instance profile"
-  type        = "string"
+  type        = string
 }
 
 variable "master_instance_type" {
   description = "EC2 Instance Type for Master"
-  type        = "string"
+  type        = string
   default     = "m5.xlarge"
 }
 
 variable "core_instance_type" {
   description = "EC2 Instance Type for Core Nodes"
-  type        = "string"
+  type        = string
   default     = "m5.xlarge"
 }
 
 variable "task_instance_type" {
   description = "EC2 Instance Type for Task Nodes"
-  type        = "string"
+  type        = string
   default     = "m5.xlarge"
 }
 
 variable "core_instance_count" {
   description = "Number of Core Nodes"
-  type        = "string"
+  type        = string
   default     = "2"
 }
 
 variable "core_instance_max_count" {
   description = "Max number of Core Nodes used on autoscale"
-  type        = "string"
+  type        = string
   default     = "4"
 }
 
 variable "task_instance_count" {
   description = "Number of instances of Task Nodes"
-  type        = "string"
+  type        = string
   default     = "2"
 }
 
 variable "task_instance_max_count" {
   description = "Max number of Task Nodes used on autoscale"
-  type        = "string"
+  type        = string
   default     = "4"
 }
 
 locals {
-  tags = "${merge(map("vendor", "segment"), var.tags)}"
+  tags = merge(map("vendor", "segment"), var.tags)
 }

modules/glue/main.tf

@@ -4,6 +4,6 @@
 # makes the schema available to various tools like Athena, Spectrum, EMR etc.
 # https://www.terraform.io/docs/providers/aws/r/glue_catalog_database.html
 resource "aws_glue_catalog_database" "segment_data_lake_glue_catalog" {
-  name        = "${var.name}"
-  description = "${var.description}"
+  name        = var.name
+  description = var.description
 }

modules/glue/output.tf

@@ -1,3 +1,3 @@
 output "database_name" {
-  value = "${aws_glue_catalog_database.segment_data_lake_glue_catalog.name}"
+  value = aws_glue_catalog_database.segment_data_lake_glue_catalog.name
 }

modules/glue/variables.tf

@@ -1,10 +1,10 @@
 variable "name" {
   description = "The name of the database."
-  type        = "string"
+  type        = string
 }
 
 variable "description" {
   description = "Description of the database."
-  type        = "string"
+  type        = string
   default     = "Segment Data Lake"
 }

modules/iam/main.tf

@@ -3,8 +3,8 @@
 resource "aws_iam_role" "segment_data_lake_iam_role" {
   name               = "SegmentDataLakeRole${var.suffix}"
   description        = "IAM Role used by Segment"
-  assume_role_policy = "${data.aws_iam_policy_document.segment_data_lake_assume_role_policy_document.json}"
-  tags               = "${local.tags}"
+  assume_role_policy = data.aws_iam_policy_document.segment_data_lake_assume_role_policy_document.json
+  tags               = local.tags
 }
 
 # Policy attached to the IAM role.
@@ -20,15 +20,15 @@ data "aws_iam_policy_document" "segment_data_lake_assume_role_policy_document" {
 
     principals {
       type        = "AWS"
-      identifiers = "${var.segment_aws_accounts}"
+      identifiers = var.segment_aws_accounts
     }
 
     effect = "Allow"
 
     condition {
       test     = "StringEquals"
       variable = "sts:ExternalId"
-      values   = "${var.external_ids}"
+      values   = var.external_ids
     }
   }
 }
@@ -43,7 +43,7 @@ resource "aws_iam_policy" "segment_data_lake_policy" {
   name        = "SegmentDataLakePolicy${var.suffix}"
   path        = "/"
   description = "Gives access to resources in your Data Lake"
-  policy      = "${data.aws_iam_policy_document.segment_data_lake_policy_document.json}"
+  policy      = data.aws_iam_policy_document.segment_data_lake_policy_document.json
 }
 
 data "aws_iam_policy_document" "segment_data_lake_policy_document" {
@@ -144,18 +144,18 @@ data "aws_iam_policy_document" "segment_data_lake_policy_document" {
     ]
 
     resources = [
-      "${aws_iam_role.segment_emr_service_role.arn}",
-      "${aws_iam_role.segment_emr_instance_profile_role.arn}",
-      "${aws_iam_role.segment_emr_autoscaling_role.arn}",
+      aws_iam_role.segment_emr_service_role.arn,
+      aws_iam_role.segment_emr_instance_profile_role.arn,
+      aws_iam_role.segment_emr_autoscaling_role.arn,
     ]
 
     effect = "Allow"
   }
 }
 
 resource "aws_iam_role_policy_attachment" "segment_data_lake_role_policy_attachment" {
-  role       = "${aws_iam_role.segment_data_lake_iam_role.name}"
-  policy_arn = "${aws_iam_policy.segment_data_lake_policy.arn}"
+  role       = aws_iam_role.segment_data_lake_iam_role.name
+  policy_arn = aws_iam_policy.segment_data_lake_policy.arn
 }
 
 # IAM role for EMR Service
@@ -178,12 +178,12 @@ resource "aws_iam_role" "segment_emr_service_role" {
 }
 EOF
 
-  tags = "${local.tags}"
+  tags = local.tags
 }
 
 resource "aws_iam_role_policy" "segment_emr_service_policy" {
   name = "SegmentEMRServicePolicy${var.suffix}"
-  role = "${aws_iam_role.segment_emr_service_role.id}"
+  role = aws_iam_role.segment_emr_service_role.id
 
   policy = <<EOF
 {
@@ -281,17 +281,17 @@ resource "aws_iam_role" "segment_emr_instance_profile_role" {
 }
 EOF
 
-  tags = "${local.tags}"
+  tags = local.tags
 }
 
 resource "aws_iam_instance_profile" "segment_emr_instance_profile" {
-  name  = "SegmentEMRInstanceProfile${var.suffix}"
-  roles = ["${aws_iam_role.segment_emr_instance_profile_role.name}"]
+  name = "SegmentEMRInstanceProfile${var.suffix}"
+  role = aws_iam_role.segment_emr_instance_profile_role.name
 }
 
 resource "aws_iam_role_policy" "segment_emr_instance_profile_policy" {
   name = "SegmentEMRInstanceProfilePolicy${var.suffix}"
-  role = "${aws_iam_role.segment_emr_instance_profile_role.id}"
+  role = aws_iam_role.segment_emr_instance_profile_role.id
 
   policy = <<EOF
 {
@@ -386,12 +386,12 @@ resource "aws_iam_role" "segment_emr_autoscaling_role" {
 }
 EOF
 
-  tags = "${local.tags}"
+  tags = local.tags
 }
 
-resource "aws_iam_role_policy" "segmnet_emr_autoscaling_policy" {
+resource "aws_iam_role_policy" "segment_emr_autoscaling_policy" {
   name = "SegmentEMRAutoscalingPolicy${var.suffix}"
-  role = "${aws_iam_role.segment_emr_autoscaling_role.id}"
+  role = aws_iam_role.segment_emr_autoscaling_role.id
 
   policy = <<EOF
 {