From 6fea0dab2aa25a91d0d794942a5c184342924b48 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 23 Aug 2023 18:52:39 -0400 Subject: [PATCH] feat!: Update AWS provider to support `v5.0` and increase Terraform MSV to `1.0` (#42) --- .pre-commit-config.yaml | 2 +- README.md | 45 +- examples/complete/README.md | 35 +- .../complete/configs/s3_table_definition.json | 67 ++- examples/complete/main.tf | 449 +++++--------- examples/complete/outputs.tf | 55 +- examples/complete/versions.tf | 4 +- main.tf | 559 ++++++++++++++---- outputs.tf | 102 +++- variables.tf | 160 ++++- versions.tf | 6 +- 11 files changed, 960 insertions(+), 524 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5886a6..2ab13c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 + rev: v1.82.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 7f9f84f..e196e3b 100644 --- a/README.md +++ b/README.md @@ -302,16 +302,16 @@ Examples codified under the [`examples`](https://github.com/terraform-aws-module | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 4.17 | -| [time](#requirement\_time) | >=0.7.2 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 5.0 | +| [time](#requirement\_time) | >= 0.9 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.17 | -| [time](#provider\_time) | >=0.7.2 | +| [aws](#provider\_aws) | >= 5.0 | +| [time](#provider\_time) | >= 0.9 | ## Modules @@ -327,20 +327,46 @@ No modules. | [aws_dms_replication_instance.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_replication_instance) | resource | | [aws_dms_replication_subnet_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_replication_subnet_group) | resource | | [aws_dms_replication_task.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_replication_task) | resource | +| [aws_dms_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | +| [aws_iam_policy.access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.dms_access_for_endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.dms_cloudwatch_logs_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.dms_vpc_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.access_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [time_sleep.wait_for_dependency_resources](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.access_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.dms_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.dms_assume_role_redshift](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [access\_iam\_role\_description](#input\_access\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [access\_iam\_role\_name](#input\_access\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [access\_iam\_role\_path](#input\_access\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [access\_iam\_role\_permissions\_boundary](#input\_access\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [access\_iam\_role\_policies](#input\_access\_iam\_role\_policies) | Map of IAM role policy ARNs to attach to the IAM role | `map(string)` | `{}` | no | +| [access\_iam\_role\_tags](#input\_access\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [access\_iam\_role\_use\_name\_prefix](#input\_access\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`access_iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [access\_iam\_statements](#input\_access\_iam\_statements) | A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage | `any` | `{}` | no | +| [access\_kms\_key\_arns](#input\_access\_kms\_key\_arns) | A list of KMS key ARNs the access IAM role is permitted to decrypt | `list(string)` | `[]` | no | +| [access\_secret\_arns](#input\_access\_secret\_arns) | A list of SecretManager secret ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | +| [access\_source\_s3\_bucket\_arns](#input\_access\_source\_s3\_bucket\_arns) | A list of S3 bucket ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | +| [access\_target\_dynamodb\_table\_arns](#input\_access\_target\_dynamodb\_table\_arns) | A list of DynamoDB table ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | +| [access\_target\_elasticsearch\_arns](#input\_access\_target\_elasticsearch\_arns) | A list of Elasticsearch ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | +| [access\_target\_kinesis\_arns](#input\_access\_target\_kinesis\_arns) | A list of Kinesis ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | +| [access\_target\_s3\_bucket\_arns](#input\_access\_target\_s3\_bucket\_arns) | A list of S3 bucket ARNs the access IAM role is permitted to access | `list(string)` | `[]` | no | | [certificates](#input\_certificates) | Map of objects that define the certificates to be created | `map(any)` | `{}` | no | | [create](#input\_create) | Determines whether resources will be created | `bool` | `true` | no | +| [create\_access\_iam\_role](#input\_create\_access\_iam\_role) | Determines whether the ECS task definition IAM role should be created | `bool` | `true` | no | +| [create\_access\_policy](#input\_create\_access\_policy) | Determines whether the IAM policy should be created | `bool` | `true` | no | | [create\_iam\_roles](#input\_create\_iam\_roles) | Determines whether the required [DMS IAM resources](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Security.html#CHAP_Security.APIRole) will be created | `bool` | `true` | no | | [create\_repl\_subnet\_group](#input\_create\_repl\_subnet\_group) | Determines whether the replication subnet group will be created | `bool` | `true` | no | | [enable\_redshift\_target\_permissions](#input\_enable\_redshift\_target\_permissions) | Determines whether `redshift.amazonaws.com` is permitted access to assume the `dms-access-for-endpoint` role | `bool` | `false` | no | @@ -350,9 +376,9 @@ No modules. | [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the role | `string` | `null` | no | | [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to apply to the DMS IAM roles | `map(string)` | `{}` | no | | [repl\_instance\_allocated\_storage](#input\_repl\_instance\_allocated\_storage) | The amount of storage (in gigabytes) to be initially allocated for the replication instance. Min: 5, Max: 6144, Default: 50 | `number` | `null` | no | -| [repl\_instance\_allow\_major\_version\_upgrade](#input\_repl\_instance\_allow\_major\_version\_upgrade) | Indicates that major version upgrades are allowed | `bool` | `null` | no | +| [repl\_instance\_allow\_major\_version\_upgrade](#input\_repl\_instance\_allow\_major\_version\_upgrade) | Indicates that major version upgrades are allowed | `bool` | `true` | no | | [repl\_instance\_apply\_immediately](#input\_repl\_instance\_apply\_immediately) | Indicates whether the changes should be applied immediately or during the next maintenance window | `bool` | `null` | no | -| [repl\_instance\_auto\_minor\_version\_upgrade](#input\_repl\_instance\_auto\_minor\_version\_upgrade) | Indicates that minor engine upgrades will be applied automatically to the replication instance during the maintenance window | `bool` | `null` | no | +| [repl\_instance\_auto\_minor\_version\_upgrade](#input\_repl\_instance\_auto\_minor\_version\_upgrade) | Indicates that minor engine upgrades will be applied automatically to the replication instance during the maintenance window | `bool` | `true` | no | | [repl\_instance\_availability\_zone](#input\_repl\_instance\_availability\_zone) | The EC2 Availability Zone that the replication instance will be created in | `string` | `null` | no | | [repl\_instance\_class](#input\_repl\_instance\_class) | The compute and memory capacity of the replication instance as specified by the replication instance class | `string` | `null` | no | | [repl\_instance\_engine\_version](#input\_repl\_instance\_engine\_version) | The [engine version](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_ReleaseNotes.html) number of the replication instance | `string` | `null` | no | @@ -370,12 +396,16 @@ No modules. | [repl\_subnet\_group\_subnet\_ids](#input\_repl\_subnet\_group\_subnet\_ids) | A list of the EC2 subnet IDs for the subnet group | `list(string)` | `[]` | no | | [repl\_subnet\_group\_tags](#input\_repl\_subnet\_group\_tags) | A map of additional tags to apply to the replication subnet group | `map(string)` | `{}` | no | | [replication\_tasks](#input\_replication\_tasks) | Map of objects that define the replication tasks to be created | `any` | `{}` | no | +| [s3\_endpoints](#input\_s3\_endpoints) | Map of objects that define the S3 endpoints to be created | `any` | `{}` | no | | [tags](#input\_tags) | A map of tags to use on all resources | `map(string)` | `{}` | no | ## Outputs | Name | Description | |------|-------------| +| [access\_iam\_role\_arn](#output\_access\_iam\_role\_arn) | Access IAM role ARN | +| [access\_iam\_role\_name](#output\_access\_iam\_role\_name) | Access IAM role name | +| [access\_iam\_role\_unique\_id](#output\_access\_iam\_role\_unique\_id) | Stable and unique string identifying the access IAM role | | [certificates](#output\_certificates) | A map of maps containing the certificates created and their full output of attributes and values | | [dms\_access\_for\_endpoint\_iam\_role\_arn](#output\_dms\_access\_for\_endpoint\_iam\_role\_arn) | Amazon Resource Name (ARN) specifying the role | | [dms\_access\_for\_endpoint\_iam\_role\_id](#output\_dms\_access\_for\_endpoint\_iam\_role\_id) | Name of the IAM role | @@ -394,6 +424,7 @@ No modules. | [replication\_instance\_tags\_all](#output\_replication\_instance\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider `default_tags` configuration block | | [replication\_subnet\_group\_id](#output\_replication\_subnet\_group\_id) | The ID of the subnet group | | [replication\_tasks](#output\_replication\_tasks) | A map of maps containing the replication tasks created and their full output of attributes and values | +| [s3\_endpoints](#output\_s3\_endpoints) | A map of maps containing the S3 endpoints created and their full output of attributes and values | ## License diff --git a/examples/complete/README.md b/examples/complete/README.md index bd960b8..e6bb8bf 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -27,15 +27,15 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | >= 4.17 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 5.0 | | [random](#requirement\_random) | >= 3.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.17 | +| [aws](#provider\_aws) | >= 5.0 | | [random](#provider\_random) | >= 3.0 | ## Modules @@ -45,34 +45,26 @@ Note that this example may create resources which will incur monetary charges on | [dms\_aurora\_postgresql\_aurora\_mysql](#module\_dms\_aurora\_postgresql\_aurora\_mysql) | ../.. | n/a | | [dms\_default](#module\_dms\_default) | ../.. | n/a | | [dms\_disabled](#module\_dms\_disabled) | ../.. | n/a | -| [msk\_cluster](#module\_msk\_cluster) | clowdhaus/msk-kafka-cluster/aws | ~> 1.0 | -| [rds\_aurora](#module\_rds\_aurora) | terraform-aws-modules/rds-aurora/aws | ~> 6.0 | +| [msk\_cluster](#module\_msk\_cluster) | terraform-aws-modules/msk-kafka-cluster/aws | ~> 2.0 | +| [rds\_aurora](#module\_rds\_aurora) | terraform-aws-modules/rds-aurora/aws | ~> 8.0 | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.1 | -| [security\_group](#module\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | -| [vpc\_endpoint\_security\_group](#module\_vpc\_endpoint\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 3.0 | +| [secrets\_manager\_msk](#module\_secrets\_manager\_msk) | terraform-aws-modules/secrets-manager/aws | ~> 1.0 | +| [secrets\_manager\_mysql](#module\_secrets\_manager\_mysql) | terraform-aws-modules/secrets-manager/aws | ~> 1.0 | +| [secrets\_manager\_postgresql](#module\_secrets\_manager\_postgresql) | terraform-aws-modules/secrets-manager/aws | ~> 1.0 | +| [security\_group](#module\_security\_group) | terraform-aws-modules/security-group/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 5.0 | ## Resources | Name | Type | |------|------| -| [aws_iam_role.s3_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.secretsmanager_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_kms_key.aurora_credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_kms_key.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_rds_cluster_parameter_group.postgresql](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster_parameter_group) | resource | | [aws_s3_object.hr_data](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | -| [aws_secretsmanager_secret.aurora_credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | -| [aws_secretsmanager_secret.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | -| [aws_secretsmanager_secret_policy.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_policy) | resource | -| [aws_secretsmanager_secret_version.aurora_credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | -| [aws_secretsmanager_secret_version.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | | [aws_sns_topic.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource | | [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | ## Inputs @@ -100,6 +92,7 @@ No inputs. | [replication\_instance\_tags\_all](#output\_replication\_instance\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider `default_tags` configuration block | | [replication\_subnet\_group\_id](#output\_replication\_subnet\_group\_id) | The ID of the subnet group | | [replication\_tasks](#output\_replication\_tasks) | A map of maps containing the replication tasks created and their full output of attributes and values | +| [s3\_endpoints](#output\_s3\_endpoints) | A map of maps containing the S3 endpoints created and their full output of attributes and values | Apache-2.0 Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-aws-dms/blob/master/LICENSE). diff --git a/examples/complete/configs/s3_table_definition.json b/examples/complete/configs/s3_table_definition.json index dba412f..44105ab 100644 --- a/examples/complete/configs/s3_table_definition.json +++ b/examples/complete/configs/s3_table_definition.json @@ -1,35 +1,38 @@ { "TableCount": "1", - "Tables": [{ - "TableName": "employee", - "TablePath": "hr/employee/", - "TableOwner": "hr", - "TableColumns": [{ - "ColumnName": "Id", - "ColumnType": "INT8", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "LastName", - "ColumnType": "STRING", - "ColumnLength": "20" - }, - { - "ColumnName": "FirstName", - "ColumnType": "STRING", - "ColumnLength": "30" - }, - { - "ColumnName": "HireDate", - "ColumnType": "DATETIME" - }, - { - "ColumnName": "OfficeLocation", - "ColumnType": "STRING", - "ColumnLength": "20" - } - ], - "TableColumnsTotal": "5" - }] + "Tables": [ + { + "TableName": "employee", + "TablePath": "hr/employee/", + "TableOwner": "hr", + "TableColumns": [ + { + "ColumnName": "Id", + "ColumnType": "INT8", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "LastName", + "ColumnType": "STRING", + "ColumnLength": "20" + }, + { + "ColumnName": "FirstName", + "ColumnType": "STRING", + "ColumnLength": "30" + }, + { + "ColumnName": "HireDate", + "ColumnType": "DATETIME" + }, + { + "ColumnName": "OfficeLocation", + "ColumnType": "STRING", + "ColumnLength": "20" + } + ], + "TableColumnsTotal": "5" + } + ] } diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 47ff34d..dd3c44d 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -2,12 +2,18 @@ provider "aws" { region = local.region } +data "aws_availability_zones" "available" {} + locals { region = "us-east-1" - name = "dms-ex-${replace(basename(path.cwd), "_", "-")}" + name = "dms-ex-${basename(path.cwd)}" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) db_name = "example" db_username = "example" + db_password = "password123!" # do better! # MSK sasl_scram_credentials = { @@ -19,9 +25,6 @@ locals { replication_instance_event_categories = ["failure", "creation", "deletion", "maintenance", "failover", "low storage", "configuration change"] replication_task_event_categories = ["failure", "state change", "creation", "deletion", "configuration change"] - bucket_postfix = "${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}" - bucket_name = "${local.name}-s3-${local.bucket_postfix}" - tags = { Name = local.name Example = local.name @@ -29,10 +32,6 @@ locals { } } -data "aws_partition" "current" {} -data "aws_region" "current" {} -data "aws_caller_identity" "current" {} - ################################################################################ # DMS Module ################################################################################ @@ -55,7 +54,7 @@ module "dms_default" { # Subnet group repl_subnet_group_name = local.name repl_subnet_group_description = "DMS Subnet group for ${local.name}" - repl_subnet_group_subnet_ids = module.vpc.database_subnets + repl_subnet_group_subnet_ids = module.vpc.private_subnets # Instance repl_instance_class = "dms.t3.large" @@ -70,14 +69,14 @@ module "dms_aurora_postgresql_aurora_mysql" { # Subnet group repl_subnet_group_name = local.name repl_subnet_group_description = "DMS Subnet group for ${local.name}" - repl_subnet_group_subnet_ids = module.vpc.database_subnets + repl_subnet_group_subnet_ids = module.vpc.private_subnets # Instance repl_instance_allocated_storage = 64 repl_instance_auto_minor_version_upgrade = true repl_instance_allow_major_version_upgrade = true repl_instance_apply_immediately = true - repl_instance_engine_version = "3.4.5" + repl_instance_engine_version = "3.5.1" repl_instance_multi_az = true repl_instance_preferred_maintenance_window = "sun:10:30-sun:14:30" repl_instance_publicly_accessible = false @@ -85,49 +84,57 @@ module "dms_aurora_postgresql_aurora_mysql" { repl_instance_id = local.name repl_instance_vpc_security_group_ids = [module.security_group["replication-instance"].security_group_id] - endpoints = { + # Access role + create_access_iam_role = true + access_secret_arns = [ + module.secrets_manager_postgresql.secret_arn, + module.secrets_manager_mysql.secret_arn, + ] + access_source_s3_bucket_arns = [ + module.s3_bucket.s3_bucket_arn, + "${module.s3_bucket.s3_bucket_arn}/*" + ] + + # S3 Endpoints + s3_endpoints = { s3-source = { endpoint_id = "${local.name}-s3-source" endpoint_type = "source" engine_name = "s3" - ssl_mode = "none" - - s3_settings = { - bucket_folder = "sourcedata" - bucket_name = local.bucket_name # to avoid https://github.com/hashicorp/terraform/issues/4149 - data_format = "csv" - encryption_mode = "SSE_S3" - external_table_definition = file("configs/s3_table_definition.json") - service_access_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.name}-s3" # to avoid https://github.com/hashicorp/terraform/issues/4149 - } - tags = { EndpointType = "s3-source" } + bucket_folder = "sourcedata" + bucket_name = module.s3_bucket.s3_bucket_id + data_format = "csv" + ssl_mode = "none" + encryption_mode = "SSE_S3" + extra_connection_attributes = "" + external_table_definition = file("configs/s3_table_definition.json") + tags = { EndpointType = "s3-source" } } + } + # Endpoints + endpoints = { postgresql-destination = { database_name = local.db_name endpoint_id = "${local.name}-postgresql-destination" endpoint_type = "target" engine_name = "aurora-postgresql" - extra_connection_attributes = "heartbeatFrequency=1;" - username = local.db_username - password = module.rds_aurora["postgresql-source"].cluster_master_password - port = 5432 - server_name = module.rds_aurora["postgresql-source"].cluster_endpoint - ssl_mode = "none" - tags = { EndpointType = "postgresql-destination" } + extra_connection_attributes = "heartbeatFrequency=1;secretsManagerEndpointOverride=${module.vpc_endpoints.endpoints["secretsmanager"]["dns_entry"][0]["dns_name"]}" + secrets_manager_arn = module.secrets_manager_postgresql.secret_arn + + tags = { EndpointType = "postgresql-destination" } } postgresql-source = { - database_name = local.db_name - endpoint_id = "${local.name}-postgresql-source" - endpoint_type = "source" - engine_name = "aurora-postgresql" - secrets_manager_arn = aws_secretsmanager_secret_version.aurora_credentials.arn - secrets_manager_access_role_arn = aws_iam_role.secretsmanager_role.arn - extra_connection_attributes = "heartbeatFrequency=1;secretsManagerEndpointOverride=${module.vpc_endpoints.endpoints["secretsmanager"]["dns_entry"][0]["dns_name"]}" - ssl_mode = "none" - tags = { EndpointType = "postgresql-source" } + database_name = local.db_name + endpoint_id = "${local.name}-postgresql-source" + endpoint_type = "source" + engine_name = "aurora-postgresql" + extra_connection_attributes = "heartbeatFrequency=1;secretsManagerEndpointOverride=${module.vpc_endpoints.endpoints["secretsmanager"]["dns_entry"][0]["dns_name"]}" + secrets_manager_arn = module.secrets_manager_postgresql.secret_arn + + tags = { EndpointType = "postgresql-source" } } mysql-destination = { @@ -135,20 +142,16 @@ module "dms_aurora_postgresql_aurora_mysql" { endpoint_id = "${local.name}-mysql-destination" endpoint_type = "target" engine_name = "aurora" - extra_connection_attributes = "" - username = local.db_username - password = module.rds_aurora["mysql-destination"].cluster_master_password - port = 3306 - server_name = module.rds_aurora["mysql-destination"].cluster_endpoint - ssl_mode = "none" - tags = { EndpointType = "mysql-destination" } + extra_connection_attributes = "secretsManagerEndpointOverride=${module.vpc_endpoints.endpoints["secretsmanager"]["dns_entry"][0]["dns_name"]}" + secrets_manager_arn = module.secrets_manager_mysql.secret_arn + + tags = { EndpointType = "mysql-destination" } } kafka-destination = { endpoint_id = "${local.name}-kafka-destination" endpoint_type = "target" engine_name = "kafka" - ssl_mode = "none" kafka_settings = { # this https://github.com/hashicorp/terraform/issues/4149 requires the MSK cluster exists before applying @@ -168,12 +171,13 @@ module "dms_aurora_postgresql_aurora_mysql" { replication_tasks = { s3_import = { - replication_task_id = "${local.name}-s3-import" - migration_type = "full-load" - table_mappings = file("configs/table_mappings.json") - source_endpoint_key = "s3-source" - target_endpoint_key = "postgresql-destination" - tags = { Task = "S3-to-PostgreSQL" } + replication_task_id = "${local.name}-s3-import" + migration_type = "full-load" + replication_task_settings = file("configs/task_settings.json") + table_mappings = file("configs/table_mappings.json") + source_endpoint_key = "s3-source" + target_endpoint_key = "postgresql-destination" + tags = { Task = "S3-to-PostgreSQL" } } postgresql_mysql = { replication_task_id = "${local.name}-postgresql-to-mysql" @@ -196,15 +200,6 @@ module "dms_aurora_postgresql_aurora_mysql" { } event_subscriptions = { - # # Despite what the terraform docs say, this is not valid - you must supply a `source_type` - # all = { - # name = "all-events" - # enabled = true - # instance_event_subscription_keys = [local.name] - # task_event_subscription_keys = ["postgresql_mysql"] - # event_categories = distinct(concat(local.replication_instance_event_categories, local.replication_task_event_categories)) - # sns_topic_arn = aws_sns_topic.example.arn - # }, instance = { name = "instance-events" enabled = true @@ -248,53 +243,40 @@ resource "random_pet" "this" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" + version = "~> 5.0" name = local.name - cidr = "10.99.0.0/18" - - azs = ["${local.region}a", "${local.region}b", "${local.region}d"] # careful on which AZs support DMS VPC endpoint - public_subnets = ["10.99.0.0/24", "10.99.1.0/24", "10.99.2.0/24"] - private_subnets = ["10.99.3.0/24", "10.99.4.0/24", "10.99.5.0/24"] - database_subnets = ["10.99.7.0/24", "10.99.8.0/24", "10.99.9.0/24"] - - create_database_subnet_group = true - enable_nat_gateway = false # not required, using private VPC endpoint - single_nat_gateway = true - map_public_ip_on_launch = false + cidr = local.vpc_cidr - manage_default_security_group = true - default_security_group_ingress = [] - default_security_group_egress = [] + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 3)] + database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 6)] - enable_flow_log = true - flow_log_destination_type = "cloud-watch-logs" - create_flow_log_cloudwatch_log_group = true - create_flow_log_cloudwatch_iam_role = true - flow_log_max_aggregation_interval = 60 - flow_log_log_format = "$${version} $${account-id} $${interface-id} $${srcaddr} $${dstaddr} $${srcport} $${dstport} $${protocol} $${packets} $${bytes} $${start} $${end} $${action} $${log-status} $${vpc-id} $${subnet-id} $${instance-id} $${tcp-flags} $${type} $${pkt-srcaddr} $${pkt-dstaddr} $${region} $${az-id} $${sublocation-type} $${sublocation-id}" + enable_nat_gateway = true + single_nat_gateway = true - enable_dhcp_options = true - enable_dns_hostnames = true - dhcp_options_domain_name = data.aws_region.current.name == "us-east-1" ? "ec2.internal" : "${data.aws_region.current.name}.compute.internal" + create_database_subnet_group = true tags = local.tags } module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 3.0" - - vpc_id = module.vpc.vpc_id - security_group_ids = [module.vpc_endpoint_security_group.security_group_id] + version = "~> 5.0" + + vpc_id = module.vpc.vpc_id + create_security_group = true + security_group_name_prefix = "${local.name}-vpc-endpoints-" + security_group_description = "VPC endpoint security group" + security_group_rules = { + ingress_https = { + description = "HTTPS from VPC" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + } endpoints = { - dms = { - service = "dms" - private_dns_enabled = true - subnet_ids = [element(module.vpc.database_subnets, 0), element(module.vpc.database_subnets, 1)] # careful on which AZs support DMS VPC endpoint - tags = { Name = "dms-vpc-endpoint" } - } s3 = { service = "s3" service_type = "Gateway" @@ -303,57 +285,33 @@ module "vpc_endpoints" { } secretsmanager = { service_name = "com.amazonaws.${local.region}.secretsmanager" - subnet_ids = module.vpc.database_subnets + subnet_ids = module.vpc.private_subnets } } tags = local.tags } -module "vpc_endpoint_security_group" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" - - name = "${local.name}-vpc-endpoint" - description = "Security group for VPC endpoints" - vpc_id = module.vpc.vpc_id - - ingress_with_cidr_blocks = [ - { - from_port = 443 - to_port = 443 - protocol = "tcp" - description = "VPC Endpoints HTTPs for the VPC CIDR" - cidr_blocks = module.vpc.vpc_cidr_block - } - ] - - egress_cidr_blocks = [module.vpc.vpc_cidr_block] - egress_rules = ["all-all"] - - tags = local.tags -} - module "security_group" { source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" + version = "~> 5.0" # Creates multiple for_each = { postgresql-source = ["postgresql-tcp"] mysql-destination = ["mysql-tcp"] - replication-instance = ["postgresql-tcp", "mysql-tcp", "kafka-broker-tls-tcp"] - kafka-destination = ["kafka-broker-tls-tcp"] + replication-instance = ["postgresql-tcp", "mysql-tcp", "kafka-broker-sasl-scram-tcp"] + kafka-destination = ["kafka-broker-sasl-scram-tcp"] } name = "${local.name}-${each.key}" description = "Security group for ${each.key}" vpc_id = module.vpc.vpc_id - ingress_cidr_blocks = module.vpc.database_subnets_cidr_blocks + ingress_cidr_blocks = [module.vpc.vpc_cidr_block] ingress_rules = each.value - egress_cidr_blocks = [module.vpc.vpc_cidr_block] + egress_cidr_blocks = ["0.0.0.0/0"] egress_rules = ["all-all"] tags = local.tags @@ -361,7 +319,7 @@ module "security_group" { resource "aws_rds_cluster_parameter_group" "postgresql" { name = "${local.name}-postgresql" - family = "aurora-postgresql11" + family = "aurora-postgresql14" parameter { name = "rds.logical_replication" @@ -379,26 +337,28 @@ resource "aws_rds_cluster_parameter_group" "postgresql" { module "rds_aurora" { source = "terraform-aws-modules/rds-aurora/aws" - version = "~> 6.0" + version = "~> 8.0" # Creates multiple for_each = { postgresql-source = { engine = "aurora-postgresql" - engine_version = "11.12" + engine_version = "14.7" enabled_cloudwatch_logs_exports = ["postgresql"] }, mysql-destination = { engine = "aurora-mysql" - engine_version = "5.7.mysql_aurora.2.07.5" + engine_version = "8.0" enabled_cloudwatch_logs_exports = ["general", "error", "slowquery"] } } - name = "${local.name}-${each.key}" - database_name = local.db_name - master_username = local.db_username - apply_immediately = true + name = "${local.name}-${each.key}" + database_name = local.db_name + master_username = local.db_username + master_password = local.db_password + manage_master_user_password = false + apply_immediately = true engine = each.value.engine engine_version = each.value.engine_version @@ -408,10 +368,6 @@ module "rds_aurora" { skip_final_snapshot = true db_cluster_parameter_group_name = each.key == "postgresql-source" ? aws_rds_cluster_parameter_group.postgresql.id : null - enabled_cloudwatch_logs_exports = each.value.enabled_cloudwatch_logs_exports - monitoring_interval = 60 - create_monitoring_role = true - vpc_id = module.vpc.vpc_id subnets = module.vpc.database_subnets db_subnet_group_name = module.vpc.database_subnet_group_name @@ -423,8 +379,7 @@ module "rds_aurora" { } resource "aws_sns_topic" "example" { - name = local.name - kms_master_key_id = "alias/aws/sns" + name = local.name tags = local.tags } @@ -433,15 +388,9 @@ module "s3_bucket" { source = "terraform-aws-modules/s3-bucket/aws" version = "~> 3.1" - bucket = local.bucket_name - - attach_deny_insecure_transport_policy = true - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true + bucket_prefix = local.name + attach_deny_insecure_transport_policy = false server_side_encryption_configuration = { rule = { apply_server_side_encryption_by_default = { @@ -463,59 +412,18 @@ resource "aws_s3_object" "hr_data" { tags = local.tags } -resource "aws_iam_role" "s3_role" { - name = "${local.name}-s3" - description = "Role used to migrate data from S3 via DMS" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "DMSAssume" - Action = "sts:AssumeRole" - Effect = "Allow" - Principal = { - Service = "dms.${data.aws_partition.current.dns_suffix}" - } - }, - ] - }) - - inline_policy { - name = "${local.name}-s3" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "DMSRead" - Action = ["s3:GetObject"] - Effect = "Allow" - Resource = "${module.s3_bucket.s3_bucket_arn}/*" - }, - { - Sid = "DMSList" - Action = ["s3:ListBucket"] - Effect = "Allow" - Resource = module.s3_bucket.s3_bucket_arn - }, - ] - }) - } - - tags = local.tags -} - module "msk_cluster" { - source = "clowdhaus/msk-kafka-cluster/aws" - version = "~> 1.0" + source = "terraform-aws-modules/msk-kafka-cluster/aws" + version = "~> 2.0" name = local.name - kafka_version = "2.8.0" + kafka_version = "3.4.0" number_of_broker_nodes = 3 - broker_node_client_subnets = module.vpc.private_subnets - broker_node_ebs_volume_size = 20 + broker_node_client_subnets = module.vpc.private_subnets + broker_node_storage_info = { + ebs_storage_info = { volume_size = 20 } + } broker_node_instance_type = "kafka.t3.small" broker_node_security_groups = [module.security_group["kafka-destination"].security_group_id] @@ -529,123 +437,92 @@ module "msk_cluster" { "delete.topic.enable" = true } - client_authentication_sasl_scram = true + client_authentication = { + sasl = { scram = true } + } create_scram_secret_association = true - scram_secret_association_secret_arn_list = [aws_secretsmanager_secret.msk.arn] - - depends_on = [aws_secretsmanager_secret_version.msk] + scram_secret_association_secret_arn_list = [module.secrets_manager_msk.secret_arn] tags = local.tags } -resource "aws_kms_key" "msk" { - description = "KMS CMK for ${local.name}" - enable_key_rotation = true - - tags = local.tags -} +module "secrets_manager_msk" { + source = "terraform-aws-modules/secrets-manager/aws" + version = "~> 1.0" -resource "aws_secretsmanager_secret" "msk" { - name = "AmazonMSK_${local.name}_${random_pet.this.id}" + name_prefix = "AmazonMSK_${local.name}-" description = "Secret for ${local.name}" - kms_key_id = aws_kms_key.msk.key_id - - tags = local.tags -} -resource "aws_secretsmanager_secret_version" "msk" { - secret_id = aws_secretsmanager_secret.msk.id - secret_string = jsonencode(local.sasl_scram_credentials) -} - -resource "aws_secretsmanager_secret_policy" "msk" { - secret_arn = aws_secretsmanager_secret.msk.arn - policy = <<-POLICY - { - "Version" : "2012-10-17", - "Statement" : [ { - "Sid": "AWSKafkaResourcePolicy", - "Effect" : "Allow", - "Principal" : { - "Service" : "kafka.amazonaws.com" - }, - "Action" : "secretsmanager:getSecretValue", - "Resource" : "${aws_secretsmanager_secret.msk.arn}" - } ] + # Secret + recovery_window_in_days = 0 + secret_string = jsonencode(local.sasl_scram_credentials) + kms_key_id = aws_kms_key.this.id + + # Policy + create_policy = true + block_public_policy = true + policy_statements = { + read = { + sid = "AWSKafkaResourcePolicy" + principals = [{ + type = "Service" + identifiers = ["kafka.amazonaws.com"] + }] + actions = ["secretsmanager:GetSecretValue"] + resources = ["*"] + } } - POLICY + + tags = local.tags } -resource "aws_kms_key" "aurora_credentials" { +resource "aws_kms_key" "this" { description = "KMS CMK for ${local.name}" enable_key_rotation = true tags = local.tags } -resource "aws_secretsmanager_secret" "aurora_credentials" { - name = "rds_aurora_${local.name}_${random_pet.this.id}" - description = "Secret for ${local.name}" - kms_key_id = aws_kms_key.aurora_credentials.key_id +module "secrets_manager_postgresql" { + source = "terraform-aws-modules/secrets-manager/aws" + version = "~> 1.0" - tags = local.tags -} + name_prefix = "PostgreSQL-${local.name}-" + description = "Secret for ${local.name}" -resource "aws_secretsmanager_secret_version" "aurora_credentials" { - secret_id = aws_secretsmanager_secret.aurora_credentials.id + # Secret + recovery_window_in_days = 0 secret_string = jsonencode( { - username = module.rds_aurora["postgresql-source"].cluster_master_username - password = module.rds_aurora["postgresql-source"].cluster_master_password + username = local.db_username + password = local.db_password port = 5432 host = module.rds_aurora["postgresql-source"].cluster_endpoint } ) - depends_on = [module.rds_aurora] + kms_key_id = aws_kms_key.this.id + + tags = local.tags } -resource "aws_iam_role" "secretsmanager_role" { - name = "${local.name}-secretsmanager" - description = "Role used to read secretsmanager secret" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "DMSAssume" - Action = "sts:AssumeRole" - Effect = "Allow" - Principal = { - Service = "dms.${local.region}.amazonaws.com" - } - }, - ] - }) - - inline_policy { - name = "${local.name}-secretsmanager" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "DMSRead" - Action = "secretsmanager:GetSecretValue" - Effect = "Allow" - Resource = aws_secretsmanager_secret_version.aurora_credentials.arn - }, - { - Sid = "KMSRead" - Action = [ - "kms:Decrypt", - "kms:DescribeKey" - ] - Effect = "Allow" - Resource = aws_kms_key.aurora_credentials.arn - } - ] - }) - } +module "secrets_manager_mysql" { + source = "terraform-aws-modules/secrets-manager/aws" + version = "~> 1.0" + + name_prefix = "MySQL-${local.name}-" + description = "Secret for ${local.name}" + + # Secret + recovery_window_in_days = 0 + secret_string = jsonencode( + { + username = local.db_username + password = local.db_password + port = 3306 + host = module.rds_aurora["mysql-destination"].cluster_endpoint + } + ) + kms_key_id = aws_kms_key.this.id tags = local.tags } diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 2aa3b82..233bba8 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -1,5 +1,8 @@ -# IAM roles -### DMS Endpoint +################################################################################ +# IAM Roles +################################################################################ + +# DMS Endpoint output "dms_access_for_endpoint_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" value = module.dms_aurora_postgresql_aurora_mysql.dms_access_for_endpoint_iam_role_arn @@ -15,7 +18,7 @@ output "dms_access_for_endpoint_iam_role_unique_id" { value = module.dms_aurora_postgresql_aurora_mysql.dms_access_for_endpoint_iam_role_unique_id } -### DMS CloudWatch Logs +# DMS CloudWatch Logs output "dms_cloudwatch_logs_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" value = module.dms_aurora_postgresql_aurora_mysql.dms_cloudwatch_logs_iam_role_arn @@ -31,7 +34,7 @@ output "dms_cloudwatch_logs_iam_role_unique_id" { value = module.dms_aurora_postgresql_aurora_mysql.dms_cloudwatch_logs_iam_role_unique_id } -### DMS VPC +# DMS VPC output "dms_vpc_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" value = module.dms_aurora_postgresql_aurora_mysql.dms_vpc_iam_role_arn @@ -47,13 +50,19 @@ output "dms_vpc_iam_role_unique_id" { value = module.dms_aurora_postgresql_aurora_mysql.dms_vpc_iam_role_unique_id } +################################################################################ # Subnet group +################################################################################ + output "replication_subnet_group_id" { description = "The ID of the subnet group" value = module.dms_aurora_postgresql_aurora_mysql.replication_subnet_group_id } +################################################################################ # Instance +################################################################################ + output "replication_instance_arn" { description = "The Amazon Resource Name (ARN) of the replication instance" value = module.dms_aurora_postgresql_aurora_mysql.replication_instance_arn @@ -74,26 +83,48 @@ output "replication_instance_tags_all" { value = module.dms_aurora_postgresql_aurora_mysql.replication_instance_tags_all } -# Replication Tasks -output "replication_tasks" { - description = "A map of maps containing the replication tasks created and their full output of attributes and values" - value = module.dms_aurora_postgresql_aurora_mysql.replication_tasks -} +################################################################################ +# Endpoint +################################################################################ -# Endpoints output "endpoints" { description = "A map of maps containing the endpoints created and their full output of attributes and values" value = module.dms_aurora_postgresql_aurora_mysql.endpoints sensitive = true } -# Event Subscriptions +################################################################################ +# S3 Endpoint +################################################################################ + +output "s3_endpoints" { + description = "A map of maps containing the S3 endpoints created and their full output of attributes and values" + value = module.dms_aurora_postgresql_aurora_mysql.s3_endpoints + sensitive = true +} + +################################################################################ +# Replication Task +################################################################################ + +output "replication_tasks" { + description = "A map of maps containing the replication tasks created and their full output of attributes and values" + value = module.dms_aurora_postgresql_aurora_mysql.replication_tasks +} + +################################################################################ +# Event Subscription +################################################################################ + output "event_subscriptions" { description = "A map of maps containing the event subscriptions created and their full output of attributes and values" value = module.dms_aurora_postgresql_aurora_mysql.event_subscriptions } -# Certificates +################################################################################ +# Certificate +################################################################################ + output "certificates" { description = "A map of maps containing the certificates created and their full output of attributes and values" value = module.dms_aurora_postgresql_aurora_mysql.certificates diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index 849ab37..d7b00bc 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.17" + version = ">= 5.0" } random = { source = "hashicorp/random" diff --git a/main.tf b/main.tf index 786dafb..d21fd6f 100644 --- a/main.tf +++ b/main.tf @@ -1,11 +1,15 @@ -locals { - subnet_group_id = var.create && var.create_repl_subnet_group ? aws_dms_replication_subnet_group.this[0].id : var.repl_instance_subnet_group_id +data "aws_region" "current" {} +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} - partition = data.aws_partition.current.partition +locals { + account_id = data.aws_caller_identity.current.account_id dns_suffix = data.aws_partition.current.dns_suffix -} + partition = data.aws_partition.current.partition + region = data.aws_region.current.name -data "aws_partition" "current" {} + subnet_group_id = var.create && var.create_repl_subnet_group ? aws_dms_replication_subnet_group.this[0].id : var.repl_instance_subnet_group_id +} ################################################################################ # IAM Roles @@ -17,12 +21,28 @@ data "aws_iam_policy_document" "dms_assume_role" { count = var.create && var.create_iam_roles ? 1 : 0 statement { - actions = ["sts:AssumeRole"] + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] principals { identifiers = ["dms.${local.dns_suffix}"] type = "Service" } + + # https://docs.aws.amazon.com/dms/latest/userguide/cross-service-confused-deputy-prevention.html#cross-service-confused-deputy-prevention-dms-api + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:${local.partition}:dms:${local.region}:${local.account_id}:*"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [local.account_id] + } } } @@ -32,7 +52,10 @@ data "aws_iam_policy_document" "dms_assume_role_redshift" { source_policy_documents = [data.aws_iam_policy_document.dms_assume_role[0].json] statement { - actions = ["sts:AssumeRole"] + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] principals { identifiers = ["redshift.${local.dns_suffix}"] @@ -119,9 +142,9 @@ resource "aws_dms_replication_instance" "this" { count = var.create ? 1 : 0 allocated_storage = var.repl_instance_allocated_storage - auto_minor_version_upgrade = var.repl_instance_auto_minor_version_upgrade allow_major_version_upgrade = var.repl_instance_allow_major_version_upgrade apply_immediately = var.repl_instance_apply_immediately + auto_minor_version_upgrade = var.repl_instance_auto_minor_version_upgrade availability_zone = var.repl_instance_availability_zone engine_version = var.repl_instance_engine_version kms_key_arn = var.repl_instance_kms_key_arn @@ -136,9 +159,9 @@ resource "aws_dms_replication_instance" "this" { tags = merge(var.tags, var.repl_instance_tags) timeouts { - create = lookup(var.repl_instance_timeouts, "create", null) - update = lookup(var.repl_instance_timeouts, "update", null) - delete = lookup(var.repl_instance_timeouts, "delete", null) + create = try(var.repl_instance_timeouts.create, null) + update = try(var.repl_instance_timeouts.update, null) + delete = try(var.repl_instance_timeouts.delete, null) } depends_on = [time_sleep.wait_for_dependency_resources] @@ -151,137 +174,182 @@ resource "aws_dms_replication_instance" "this" { resource "aws_dms_endpoint" "this" { for_each = { for k, v in var.endpoints : k => v if var.create } - certificate_arn = try(aws_dms_certificate.this[each.value.certificate_key].certificate_arn, null) - database_name = lookup(each.value, "database_name", null) - endpoint_id = each.value.endpoint_id - endpoint_type = each.value.endpoint_type - engine_name = each.value.engine_name - extra_connection_attributes = lookup(each.value, "extra_connection_attributes", null) - kms_key_arn = lookup(each.value, "kms_key_arn", null) - password = lookup(each.value, "password", null) - port = lookup(each.value, "port", null) - server_name = lookup(each.value, "server_name", null) - service_access_role = lookup(each.value, "service_access_role", null) - ssl_mode = lookup(each.value, "ssl_mode", null) - username = lookup(each.value, "username", null) - secrets_manager_access_role_arn = lookup(each.value, "secrets_manager_access_role_arn", null) - secrets_manager_arn = lookup(each.value, "secrets_manager_arn", null) + certificate_arn = try(aws_dms_certificate.this[each.value.certificate_key].certificate_arn, null) + database_name = lookup(each.value, "database_name", null) # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Elasticsearch.html dynamic "elasticsearch_settings" { - for_each = length(lookup(each.value, "elasticsearch_settings", {})) == 0 ? [] : [each.value.elasticsearch_settings] + for_each = length(lookup(each.value, "elasticsearch_settings", [])) > 0 ? [each.value.elasticsearch_settings] : [] content { endpoint_uri = elasticsearch_settings.value.endpoint_uri - error_retry_duration = lookup(elasticsearch_settings.value, "error_retry_duration", null) - full_load_error_percentage = lookup(elasticsearch_settings.value, "full_load_error_percentage", null) - service_access_role_arn = lookup(elasticsearch_settings.value, "service_access_role_arn", null) + error_retry_duration = try(elasticsearch_settings.value.error_retry_duration, null) + full_load_error_percentage = try(elasticsearch_settings.value.full_load_error_percentage, null) + service_access_role_arn = lookup(elasticsearch_settings.value, "service_access_role_arn", aws_iam_role.access[0].arn) } } + endpoint_id = each.value.endpoint_id + endpoint_type = each.value.endpoint_type + engine_name = each.value.engine_name + extra_connection_attributes = try(each.value.extra_connection_attributes, null) + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kafka.html dynamic "kafka_settings" { - for_each = length(lookup(each.value, "kafka_settings", {})) == 0 ? [] : [each.value.kafka_settings] + for_each = length(lookup(each.value, "kafka_settings", [])) > 0 ? [each.value.kafka_settings] : [] content { broker = kafka_settings.value.broker - include_control_details = lookup(kafka_settings.value, "include_control_details", null) - include_null_and_empty = lookup(kafka_settings.value, "include_null_and_empty", null) - include_partition_value = lookup(kafka_settings.value, "include_partition_value", null) - include_table_alter_operations = lookup(kafka_settings.value, "include_table_alter_operations", null) - include_transaction_details = lookup(kafka_settings.value, "include_transaction_details", null) - message_format = lookup(kafka_settings.value, "message_format", null) - message_max_bytes = lookup(kafka_settings.value, "message_max_bytes", null) - no_hex_prefix = lookup(kafka_settings.value, "no_hex_prefix", null) - partition_include_schema_table = lookup(kafka_settings.value, "partition_include_schema_table", null) + include_control_details = try(kafka_settings.value.include_control_details, null) + include_null_and_empty = try(kafka_settings.value.include_null_and_empty, null) + include_partition_value = try(kafka_settings.value.include_partition_value, null) + include_table_alter_operations = try(kafka_settings.value.include_table_alter_operations, null) + include_transaction_details = try(kafka_settings.value.include_transaction_details, null) + message_format = try(kafka_settings.value.message_format, null) + message_max_bytes = try(kafka_settings.value.message_max_bytes, null) + no_hex_prefix = try(kafka_settings.value.no_hex_prefix, null) + partition_include_schema_table = try(kafka_settings.value.partition_include_schema_table, null) sasl_password = lookup(kafka_settings.value, "sasl_password", null) sasl_username = lookup(kafka_settings.value, "sasl_username", null) - security_protocol = lookup(kafka_settings.value, "security_protocol", null) + security_protocol = try(kafka_settings.value.security_protocol, null) ssl_ca_certificate_arn = lookup(kafka_settings.value, "ssl_ca_certificate_arn", null) ssl_client_certificate_arn = lookup(kafka_settings.value, "ssl_client_certificate_arn", null) ssl_client_key_arn = lookup(kafka_settings.value, "ssl_client_key_arn", null) ssl_client_key_password = lookup(kafka_settings.value, "ssl_client_key_password", null) - topic = lookup(kafka_settings.value, "topic", null) + topic = try(kafka_settings.value.topic, null) } } # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kinesis.html dynamic "kinesis_settings" { - for_each = length(lookup(each.value, "kinesis_settings", {})) == 0 ? [] : [each.value.kinesis_settings] + for_each = length(lookup(each.value, "kinesis_settings", [])) > 0 ? [each.value.kinesis_settings] : [] content { - include_control_details = lookup(kinesis_settings.value, "include_control_details", null) - include_null_and_empty = lookup(kinesis_settings.value, "include_null_and_empty", null) - include_partition_value = lookup(kinesis_settings.value, "include_partition_value", null) - include_table_alter_operations = lookup(kinesis_settings.value, "include_table_alter_operations", null) - include_transaction_details = lookup(kinesis_settings.value, "include_transaction_details", null) - message_format = lookup(kinesis_settings.value, "message_format", null) - partition_include_schema_table = lookup(kinesis_settings.value, "partition_include_schema_table", null) - service_access_role_arn = lookup(kinesis_settings.value, "service_access_role_arn", null) + include_control_details = try(kinesis_settings.value.include_control_details, null) + include_null_and_empty = try(kinesis_settings.value.include_null_and_empty, null) + include_partition_value = try(kinesis_settings.value.include_partition_value, null) + include_table_alter_operations = try(kinesis_settings.value.include_table_alter_operations, null) + include_transaction_details = try(kinesis_settings.value.include_transaction_details, null) + message_format = try(kinesis_settings.value.message_format, null) + partition_include_schema_table = try(kinesis_settings.value.partition_include_schema_table, null) + service_access_role_arn = lookup(kinesis_settings.value, "service_access_role_arn", aws_iam_role.access[0].arn) stream_arn = lookup(kinesis_settings.value, "stream_arn", null) } } + kms_key_arn = lookup(each.value, "kms_key_arn", null) + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.MongoDB.html dynamic "mongodb_settings" { - for_each = length(lookup(each.value, "mongodb_settings", {})) == 0 ? [] : [each.value.mongodb_settings] + for_each = length(lookup(each.value, "mongodb_settings", [])) > 0 ? [each.value.mongodb_settings] : [] content { - auth_mechanism = lookup(mongodb_settings.value, "auth_mechanism", null) - auth_source = lookup(mongodb_settings.value, "auth_source", null) - auth_type = lookup(mongodb_settings.value, "auth_type", null) - docs_to_investigate = lookup(mongodb_settings.value, "docs_to_investigate", null) - extract_doc_id = lookup(mongodb_settings.value, "extract_doc_id", null) - nesting_level = lookup(mongodb_settings.value, "nesting_level", null) + auth_mechanism = try(mongodb_settings.value.auth_mechanism, null) + auth_source = try(mongodb_settings.value.auth_source, null) + auth_type = try(mongodb_settings.value.auth_type, null) + docs_to_investigate = try(mongodb_settings.value.docs_to_investigate, null) + extract_doc_id = try(mongodb_settings.value.extract_doc_id, null) + nesting_level = try(mongodb_settings.value.nesting_level, null) } } - # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.S3.html - # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.S3.html - dynamic "s3_settings" { - for_each = length(lookup(each.value, "s3_settings", {})) == 0 ? [] : [each.value.s3_settings] + password = lookup(each.value, "password", null) + port = try(each.value.port, null) + + dynamic "redis_settings" { + for_each = length(lookup(each.value, "redis_settings", [])) > 0 ? [each.value.redis_settings] : [] content { - add_column_name = lookup(s3_settings.value, "add_column_name", null) - bucket_folder = lookup(s3_settings.value, "bucket_folder", null) - bucket_name = lookup(s3_settings.value, "bucket_name", null) - canned_acl_for_objects = lookup(s3_settings.value, "canned_acl_for_objects", null) - cdc_inserts_and_updates = lookup(s3_settings.value, "cdc_inserts_and_updates", null) - cdc_inserts_only = lookup(s3_settings.value, "cdc_inserts_only", null) - cdc_max_batch_interval = lookup(s3_settings.value, "cdc_max_batch_interval", null) - cdc_min_file_size = lookup(s3_settings.value, "cdc_min_file_size", null) - cdc_path = lookup(s3_settings.value, "cdc_path", null) - compression_type = lookup(s3_settings.value, "compression_type", null) - csv_delimiter = lookup(s3_settings.value, "csv_delimiter", null) - csv_no_sup_value = lookup(s3_settings.value, "csv_no_sup_value", null) - csv_null_value = lookup(s3_settings.value, "csv_null_value", null) - csv_row_delimiter = lookup(s3_settings.value, "csv_row_delimiter", null) - data_format = lookup(s3_settings.value, "data_format", null) - data_page_size = lookup(s3_settings.value, "data_page_size", null) - date_partition_delimiter = lookup(s3_settings.value, "date_partition_delimiter", null) - date_partition_enabled = lookup(s3_settings.value, "date_partition_enabled", null) - date_partition_sequence = lookup(s3_settings.value, "date_partition_sequence", null) - dict_page_size_limit = lookup(s3_settings.value, "dict_page_size_limit", null) - enable_statistics = lookup(s3_settings.value, "enable_statistics", null) - encoding_type = lookup(s3_settings.value, "encoding_type", null) - encryption_mode = lookup(s3_settings.value, "encryption_mode", null) - external_table_definition = lookup(s3_settings.value, "external_table_definition", null) - ignore_headers_row = lookup(s3_settings.value, "ignore_headers_row", null) - include_op_for_full_load = lookup(s3_settings.value, "include_op_for_full_load", null) - max_file_size = lookup(s3_settings.value, "max_file_size", null) - parquet_timestamp_in_millisecond = lookup(s3_settings.value, "parquet_timestamp_in_millisecond", null) - parquet_version = lookup(s3_settings.value, "parquet_version", null) - preserve_transactions = lookup(s3_settings.value, "preserve_transactions", null) - rfc_4180 = lookup(s3_settings.value, "rfc_4180", null) - row_group_length = lookup(s3_settings.value, "row_group_length", null) - server_side_encryption_kms_key_id = lookup(s3_settings.value, "server_side_encryption_kms_key_id", null) - service_access_role_arn = lookup(s3_settings.value, "service_access_role_arn", null) - timestamp_column_name = lookup(s3_settings.value, "timestamp_column_name", null) - use_csv_no_sup_value = lookup(s3_settings.value, "use_csv_no_sup_value", null) + auth_password = try(redis_settings.value.auth_password, null) + auth_type = redis_settings.value.auth_type + auth_user_name = try(redis_settings.value.auth_user_name, null) + port = try(redis_settings.value.port, 6379) + server_name = redis_settings.value.server_name + ssl_ca_certificate_arn = lookup(redis_settings.value, "ssl_ca_certificate_arn", null) + ssl_security_protocol = try(redis_settings.value.ssl_security_protocol, null) } } - tags = merge(var.tags, lookup(each.value, "tags", {})) + dynamic "redshift_settings" { + for_each = length(lookup(each.value, "redshift_settings", [])) > 0 ? [each.value.redshift_settings] : [] + + content { + bucket_folder = try(redshift_settings.value.bucket_folder, null) + bucket_name = lookup(redshift_settings.value, "bucket_name", null) + encryption_mode = try(redshift_settings.value.encryption_mode, null) + server_side_encryption_kms_key_id = lookup(redshift_settings.value, "server_side_encryption_kms_key_id", null) + service_access_role_arn = lookup(redshift_settings.value, "service_access_role_arn", "arn:${local.partition}:iam::${local.account_id}:role/dms-access-for-endpoint") + } + } + + secrets_manager_access_role_arn = lookup(each.value, "secrets_manager_arn", null) != null ? lookup(each.value, "secrets_manager_access_role_arn", aws_iam_role.access[0].arn) : null + secrets_manager_arn = lookup(each.value, "secrets_manager_arn", null) + server_name = lookup(each.value, "server_name", null) + service_access_role = lookup(each.value, "service_access_role", aws_iam_role.access[0].arn) + ssl_mode = try(each.value.ssl_mode, null) + username = try(each.value.username, null) + + tags = merge(var.tags, try(each.value.tags, {})) +} + +################################################################################ +# S3 Endpoint +################################################################################ + +resource "aws_dms_s3_endpoint" "this" { + for_each = { for k, v in var.s3_endpoints : k => v if var.create } + + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.S3.html + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.S3.html + certificate_arn = try(aws_dms_certificate.this[each.value.certificate_key].certificate_arn, null) + endpoint_id = each.value.endpoint_id + endpoint_type = each.value.endpoint_type + kms_key_arn = lookup(each.value, "kms_key_arn", null) + ssl_mode = try(each.value.ssl_mode, null) + + add_column_name = try(each.value.add_column_name, null) + add_trailing_padding_character = try(each.value.add_trailing_padding_character, null) + bucket_folder = try(each.value.bucket_folder, null) + bucket_name = each.value.bucket_name + canned_acl_for_objects = try(each.value.canned_acl_for_objects, null) + cdc_inserts_and_updates = try(each.value.cdc_inserts_and_updates, null) + cdc_inserts_only = try(each.value.cdc_inserts_only, null) + cdc_max_batch_interval = try(each.value.cdc_max_batch_interval, null) + cdc_min_file_size = try(each.value.cdc_min_file_size, null) + cdc_path = try(each.value.cdc_path, null) + compression_type = try(each.value.compression_type, null) + csv_delimiter = try(each.value.csv_delimiter, null) + csv_no_sup_value = try(each.value.csv_no_sup_value, null) + csv_null_value = try(each.value.csv_null_value, null) + csv_row_delimiter = try(each.value.csv_row_delimiter, null) + data_format = try(each.value.data_format, null) + data_page_size = try(each.value.data_page_size, null) + date_partition_delimiter = try(each.value.date_partition_delimiter, null) + date_partition_enabled = try(each.value.date_partition_enabled, null) + date_partition_sequence = try(each.value.date_partition_sequence, null) + date_partition_timezone = try(each.value.date_partition_timezone, null) + detach_target_on_lob_lookup_failure_parquet = try(each.value.detach_target_on_lob_lookup_failure_parquet, null) + dict_page_size_limit = try(each.value.dict_page_size_limit, null) + enable_statistics = try(each.value.enable_statistics, null) + encoding_type = try(each.value.encoding_type, null) + encryption_mode = try(each.value.encryption_mode, null) + expected_bucket_owner = try(each.value.expected_bucket_owner, null) + external_table_definition = try(each.value.external_table_definition, null) + ignore_header_rows = try(each.value.ignore_header_rows, null) + include_op_for_full_load = try(each.value.include_op_for_full_load, null) + max_file_size = try(each.value.max_file_size, null) + parquet_timestamp_in_millisecond = try(each.value.parquet_timestamp_in_millisecond, null) + parquet_version = try(each.value.parquet_version, null) + preserve_transactions = try(each.value.preserve_transactions, null) + rfc_4180 = try(each.value.rfc_4180, null) + row_group_length = try(each.value.row_group_length, null) + server_side_encryption_kms_key_id = lookup(each.value, "server_side_encryption_kms_key_id", null) + service_access_role_arn = lookup(each.value, "service_access_role_arn", aws_iam_role.access[0].arn) + timestamp_column_name = try(each.value.timestamp_column_name, null) + use_csv_no_sup_value = try(each.value.use_csv_no_sup_value, null) + use_task_start_time_for_full_load_timestamp = try(each.value.use_task_start_time_for_full_load_timestamp, null) + + tags = merge(var.tags, try(each.value.tags, {})) } ################################################################################ @@ -291,18 +359,18 @@ resource "aws_dms_endpoint" "this" { resource "aws_dms_replication_task" "this" { for_each = { for k, v in var.replication_tasks : k => v if var.create } - cdc_start_position = lookup(each.value, "cdc_start_position", null) - cdc_start_time = lookup(each.value, "cdc_start_time", null) + cdc_start_position = try(each.value.cdc_start_position, null) + cdc_start_time = try(each.value.cdc_start_time, null) migration_type = each.value.migration_type replication_instance_arn = aws_dms_replication_instance.this[0].replication_instance_arn replication_task_id = each.value.replication_task_id - replication_task_settings = lookup(each.value, "replication_task_settings", null) - table_mappings = lookup(each.value, "table_mappings", null) - source_endpoint_arn = aws_dms_endpoint.this[each.value.source_endpoint_key].endpoint_arn - target_endpoint_arn = aws_dms_endpoint.this[each.value.target_endpoint_key].endpoint_arn - start_replication_task = lookup(each.value, "start_replication_task", null) + replication_task_settings = try(each.value.replication_task_settings, null) + source_endpoint_arn = try(aws_dms_endpoint.this[each.value.source_endpoint_key].endpoint_arn, aws_dms_s3_endpoint.this[each.value.source_endpoint_key].endpoint_arn) + start_replication_task = try(each.value.start_replication_task, null) + table_mappings = try(each.value.table_mappings, null) + target_endpoint_arn = try(aws_dms_endpoint.this[each.value.target_endpoint_key].endpoint_arn, aws_dms_s3_endpoint.this[each.value.target_endpoint_key].endpoint_arn) - tags = merge(var.tags, lookup(each.value, "tags", {})) + tags = merge(var.tags, try(each.value.tags, {})) } ################################################################################ @@ -312,26 +380,30 @@ resource "aws_dms_replication_task" "this" { resource "aws_dms_event_subscription" "this" { for_each = { for k, v in var.event_subscriptions : k => v if var.create } + enabled = try(each.value.enabled, null) + event_categories = try(each.value.event_categories, null) name = each.value.name - enabled = lookup(each.value, "enabled", null) - event_categories = lookup(each.value, "event_categories", null) - source_type = lookup(each.value, "source_type", null) - source_ids = compact(concat([ - for instance in aws_dms_replication_instance.this[*] : - instance.replication_instance_id if lookup(each.value, "instance_event_subscription_keys", null) == var.repl_instance_id - ], [ - for task in aws_dms_replication_task.this[*] : - task.replication_task_id if contains(lookup(each.value, "task_event_subscription_keys", []), each.key) - ])) + sns_topic_arn = each.value.sns_topic_arn + + source_ids = compact(concat( + [ + for instance in aws_dms_replication_instance.this[*] : + instance.replication_instance_id if lookup(each.value, "instance_event_subscription_keys", null) == var.repl_instance_id + ], + [ + for task in aws_dms_replication_task.this[*] : + task.replication_task_id if contains(lookup(each.value, "task_event_subscription_keys", []), each.key) + ] + )) - sns_topic_arn = each.value.sns_topic_arn + source_type = try(each.value.source_type, null) - tags = merge(var.tags, lookup(each.value, "tags", {})) + tags = merge(var.tags, try(each.value.tags, {})) timeouts { - create = lookup(var.event_subscription_timeouts, "create", null) - update = lookup(var.event_subscription_timeouts, "update", null) - delete = lookup(var.event_subscription_timeouts, "delete", null) + create = try(var.event_subscription_timeouts.create, null) + update = try(var.event_subscription_timeouts.update, null) + delete = try(var.event_subscription_timeouts.delete, null) } } @@ -346,5 +418,246 @@ resource "aws_dms_certificate" "this" { certificate_pem = lookup(each.value, "certificate_pem", null) certificate_wallet = lookup(each.value, "certificate_wallet", null) - tags = merge(var.tags, lookup(each.value, "tags", {})) + tags = merge(var.tags, try(each.value.tags, {})) +} + +################################################################################ +# Access IAM Role +################################################################################ + +locals { + access_iam_role_name = try(coalesce(var.access_iam_role_name, var.repl_instance_id), "") + create_access_iam_role = var.create && var.create_access_iam_role + create_access_policy = local.create_access_iam_role && var.create_access_policy +} + +data "aws_iam_policy_document" "access_assume" { + count = local.create_access_iam_role ? 1 : 0 + + statement { + sid = "DMSAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + identifiers = [ + "dms.${local.dns_suffix}", + "dms.${local.region}.${local.dns_suffix}", + ] + type = "Service" + } + + # https://docs.aws.amazon.com/dms/latest/userguide/cross-service-confused-deputy-prevention.html#cross-service-confused-deputy-prevention-dms-api + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:${local.partition}:dms:${local.region}:${local.account_id}:*"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [local.account_id] + } + } +} + +resource "aws_iam_role" "access" { + count = local.create_access_iam_role ? 1 : 0 + + name = var.access_iam_role_use_name_prefix ? null : local.access_iam_role_name + name_prefix = var.access_iam_role_use_name_prefix ? "${local.access_iam_role_name}-" : null + path = var.access_iam_role_path + description = coalesce(var.access_iam_role_description, "Service access role") + + assume_role_policy = data.aws_iam_policy_document.access_assume[0].json + permissions_boundary = var.access_iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.access_iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "access_additional" { + for_each = { for k, v in var.access_iam_role_policies : k => v if local.create_access_iam_role } + + role = aws_iam_role.access[0].name + policy_arn = each.value +} + +data "aws_iam_policy_document" "access" { + count = local.create_access_policy ? 1 : 0 + + statement { + sid = "KMS" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + ] + resources = coalescelist( + var.access_kms_key_arns, + ["arn:${local.partition}:kms:${local.region}:${local.account_id}:key/*"] + ) + } + + # https://docs.aws.amazon.com/dms/latest/userguide/security_iam_secretsmanager.html + dynamic "statement" { + for_each = length(var.access_secret_arns) > 0 ? [1] : [] + + content { + sid = "SecretsManager" + actions = ["secretsmanager:GetSecretValue"] + resources = var.access_secret_arns + } + } + + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.S3.html#CHAP_Source.S3.Prerequisites + dynamic "statement" { + for_each = length(var.access_source_s3_bucket_arns) > 0 ? [1] : [] + + content { + sid = "S3Source" + actions = [ + "s3:ListBucket", + "s3:GetObject", + "S3:GetObjectVersion", + ] + resources = var.access_source_s3_bucket_arns + } + } + + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.S3.html#CHAP_Target.S3.Prerequisites + dynamic "statement" { + for_each = length(var.access_target_s3_bucket_arns) > 0 ? [1] : [] + + content { + sid = "S3Target" + actions = [ + "s3:ListBucket", + "s3:PutObject", + "s3:DeleteObject", + "s3:PutObjectTagging", + ] + resources = var.access_target_s3_bucket_arns + } + } + + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Elasticsearch.html#CHAP_Target.Elasticsearch.Prerequisites + dynamic "statement" { + for_each = length(var.access_target_elasticsearch_arns) > 0 ? [1] : [] + + content { + sid = "ElasticSearchTarget" + actions = [ + "es:ESHttpDelete", + "es:ESHttpGet", + "es:ESHttpHead", + "es:ESHttpPost", + "es:ESHttpPut", + ] + resources = var.access_target_elasticsearch_arns + } + } + + # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kinesis.html#CHAP_Target.Kinesis.Prerequisites + dynamic "statement" { + for_each = length(var.access_target_kinesis_arns) > 0 ? [1] : [] + + content { + sid = "KinesisTarget" + actions = [ + "kinesis:DescribeStream", + "kinesis:PutRecord", + "kinesis:PutRecords", + ] + resources = var.access_target_kinesis_arns + } + } + + dynamic "statement" { + for_each = length(var.access_target_dynamodb_table_arns) > 0 ? [1] : [] + + content { + sid = "DynamoDBList" + actions = ["dynamodb:ListTables"] + resources = ["*"] + } + } + + dynamic "statement" { + for_each = length(var.access_target_dynamodb_table_arns) > 0 ? [1] : [] + + content { + sid = "DynamoDBTarget" + actions = [ + "dynamodb:PutItem", + "dynamodb:CreateTable", + "dynamodb:DescribeTable", + "dynamodb:DeleteTable", + "dynamodb:DeleteItem", + "dynamodb:UpdateItem" + ] + resources = var.access_target_dynamodb_table_arns + } + } + + dynamic "statement" { + for_each = var.access_iam_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_policy" "access" { + count = local.create_access_policy ? 1 : 0 + + name = var.access_iam_role_use_name_prefix ? null : local.access_iam_role_name + name_prefix = var.access_iam_role_use_name_prefix ? "${local.access_iam_role_name}-" : null + description = coalesce(var.access_iam_role_description, "Service access role IAM policy") + policy = data.aws_iam_policy_document.access[0].json + + tags = merge(var.tags, var.access_iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "access" { + count = local.create_access_policy ? 1 : 0 + + role = aws_iam_role.access[0].name + policy_arn = aws_iam_policy.access[0].arn } diff --git a/outputs.tf b/outputs.tf index 8e78e8b..05cfad3 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,101 +1,151 @@ -# IAM roles -### DMS Endpoint +################################################################################ +# IAM Roles +################################################################################ + +# DMS Endpoint output "dms_access_for_endpoint_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" - value = element(concat(aws_iam_role.dms_access_for_endpoint[*].arn, [""]), 0) + value = try(aws_iam_role.dms_access_for_endpoint[0].arn, null) } output "dms_access_for_endpoint_iam_role_id" { description = "Name of the IAM role" - value = element(concat(aws_iam_role.dms_access_for_endpoint[*].id, [""]), 0) + value = try(aws_iam_role.dms_access_for_endpoint[0].id, null) } output "dms_access_for_endpoint_iam_role_unique_id" { description = "Stable and unique string identifying the role" - value = element(concat(aws_iam_role.dms_access_for_endpoint[*].unique_id, [""]), 0) + value = try(aws_iam_role.dms_access_for_endpoint[0].unique_id, null) } -### DMS CloudWatch Logs +# DMS CloudWatch Logs output "dms_cloudwatch_logs_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" - value = element(concat(aws_iam_role.dms_cloudwatch_logs_role[*].arn, [""]), 0) + value = try(aws_iam_role.dms_cloudwatch_logs_role[0].arn, null) } output "dms_cloudwatch_logs_iam_role_id" { description = "Name of the IAM role" - value = element(concat(aws_iam_role.dms_cloudwatch_logs_role[*].id, [""]), 0) + value = try(aws_iam_role.dms_cloudwatch_logs_role[0].id, null) } output "dms_cloudwatch_logs_iam_role_unique_id" { description = "Stable and unique string identifying the role" - value = element(concat(aws_iam_role.dms_cloudwatch_logs_role[*].unique_id, [""]), 0) + value = try(aws_iam_role.dms_cloudwatch_logs_role[0].unique_id, null) } -### DMS VPC +# DMS VPC output "dms_vpc_iam_role_arn" { description = "Amazon Resource Name (ARN) specifying the role" - value = element(concat(aws_iam_role.dms_vpc_role[*].arn, [""]), 0) + value = try(aws_iam_role.dms_vpc_role[0].arn, null) } output "dms_vpc_iam_role_id" { description = "Name of the IAM role" - value = element(concat(aws_iam_role.dms_vpc_role[*].id, [""]), 0) + value = try(aws_iam_role.dms_vpc_role[0].id, null) } output "dms_vpc_iam_role_unique_id" { description = "Stable and unique string identifying the role" - value = element(concat(aws_iam_role.dms_vpc_role[*].unique_id, [""]), 0) + value = try(aws_iam_role.dms_vpc_role[0].unique_id, null) } +################################################################################ # Subnet group +################################################################################ + output "replication_subnet_group_id" { description = "The ID of the subnet group" - value = element(concat(aws_dms_replication_subnet_group.this[*].id, [""]), 0) + value = try(aws_dms_replication_subnet_group.this[0].id, null) } +################################################################################ # Instance +################################################################################ + output "replication_instance_arn" { description = "The Amazon Resource Name (ARN) of the replication instance" - value = element(concat(aws_dms_replication_instance.this[*].replication_instance_arn, [""]), 0) + value = try(aws_dms_replication_instance.this[0].replication_instance_arn, null) } output "replication_instance_private_ips" { description = "A list of the private IP addresses of the replication instance" - value = element(concat(aws_dms_replication_instance.this[*].replication_instance_private_ips, [""]), 0) + value = try(aws_dms_replication_instance.this[0].replication_instance_private_ips, null) } output "replication_instance_public_ips" { description = "A list of the public IP addresses of the replication instance" - value = element(concat(aws_dms_replication_instance.this[*].replication_instance_public_ips, [""]), 0) + value = try(aws_dms_replication_instance.this[0].replication_instance_public_ips, null) } output "replication_instance_tags_all" { description = "A map of tags assigned to the resource, including those inherited from the provider `default_tags` configuration block" - value = element(concat(aws_dms_replication_instance.this[*].tags_all, [""]), 0) + value = try(aws_dms_replication_instance.this[0].tags_all, null) } -# Replication Tasks -output "replication_tasks" { - description = "A map of maps containing the replication tasks created and their full output of attributes and values" - value = aws_dms_replication_task.this -} +################################################################################ +# Endpoint +################################################################################ -# Endpoints output "endpoints" { description = "A map of maps containing the endpoints created and their full output of attributes and values" value = aws_dms_endpoint.this sensitive = true } -# Event Subscriptions +################################################################################ +# S3 Endpoint +################################################################################ + +output "s3_endpoints" { + description = "A map of maps containing the S3 endpoints created and their full output of attributes and values" + value = aws_dms_s3_endpoint.this + sensitive = true +} + +################################################################################ +# Replication Task +################################################################################ + +output "replication_tasks" { + description = "A map of maps containing the replication tasks created and their full output of attributes and values" + value = aws_dms_replication_task.this +} + +################################################################################ +# Event Subscription +################################################################################ + output "event_subscriptions" { description = "A map of maps containing the event subscriptions created and their full output of attributes and values" value = aws_dms_event_subscription.this } -# Certificates +################################################################################ +# Certificate +################################################################################ + output "certificates" { description = "A map of maps containing the certificates created and their full output of attributes and values" value = aws_dms_certificate.this sensitive = true } + +################################################################################ +# Access IAM Role +################################################################################ + +output "access_iam_role_name" { + description = "Access IAM role name" + value = try(aws_iam_role.access[0].name, null) +} + +output "access_iam_role_arn" { + description = "Access IAM role ARN" + value = try(aws_iam_role.access[0].arn, null) +} + +output "access_iam_role_unique_id" { + description = "Stable and unique string identifying the access IAM role" + value = try(aws_iam_role.access[0].unique_id, null) +} diff --git a/variables.tf b/variables.tf index 6d40945..5a48c1a 100644 --- a/variables.tf +++ b/variables.tf @@ -10,7 +10,12 @@ variable "tags" { default = {} } -# IAM roles +################################################################################ +# IAM Roles +# https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Security.html#CHAP_Security.APIRole +# Issue: https://github.com/hashicorp/terraform-provider-aws/issues/19580 +################################################################################ + variable "create_iam_roles" { description = "Determines whether the required [DMS IAM resources](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Security.html#CHAP_Security.APIRole) will be created" type = bool @@ -35,7 +40,10 @@ variable "enable_redshift_target_permissions" { default = false } +################################################################################ # Subnet group +################################################################################ + variable "create_repl_subnet_group" { description = "Determines whether the replication subnet group will be created" type = bool @@ -66,7 +74,10 @@ variable "repl_subnet_group_tags" { default = {} } +################################################################################ # Instance +################################################################################ + variable "repl_instance_allocated_storage" { description = "The amount of storage (in gigabytes) to be initially allocated for the replication instance. Min: 5, Max: 6144, Default: 50" type = number @@ -76,13 +87,13 @@ variable "repl_instance_allocated_storage" { variable "repl_instance_auto_minor_version_upgrade" { description = "Indicates that minor engine upgrades will be applied automatically to the replication instance during the maintenance window" type = bool - default = null + default = true } variable "repl_instance_allow_major_version_upgrade" { description = "Indicates that major version upgrades are allowed" type = bool - default = null + default = true } variable "repl_instance_apply_immediately" { @@ -163,22 +174,40 @@ variable "repl_instance_timeouts" { default = {} } -# Replication Tasks -variable "replication_tasks" { - description = "Map of objects that define the replication tasks to be created" +################################################################################ +# Endpoint +################################################################################ + +variable "endpoints" { + description = "Map of objects that define the endpoints to be created" type = any default = {} } +################################################################################ +# S3 Endpoint +################################################################################ -# Endpoints -variable "endpoints" { - description = "Map of objects that define the endpoints to be created" +variable "s3_endpoints" { + description = "Map of objects that define the S3 endpoints to be created" type = any default = {} } -# Event Subscriptions +################################################################################ +# Replication Task +################################################################################ + +variable "replication_tasks" { + description = "Map of objects that define the replication tasks to be created" + type = any + default = {} +} + +################################################################################ +# Event Subscription +################################################################################ + variable "event_subscriptions" { description = "Map of objects that define the event subscriptions to be created" type = any @@ -191,9 +220,118 @@ variable "event_subscription_timeouts" { default = {} } -# Certificates +################################################################################ +# Certificate +################################################################################ + variable "certificates" { description = "Map of objects that define the certificates to be created" type = map(any) default = {} } + +################################################################################ +# Access IAM Role +################################################################################ + +variable "create_access_iam_role" { + description = "Determines whether the ECS task definition IAM role should be created" + type = bool + default = true +} + +variable "access_iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "access_iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`access_iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "access_iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "access_iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "access_iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "access_iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +variable "access_iam_role_policies" { + description = "Map of IAM role policy ARNs to attach to the IAM role" + type = map(string) + default = {} +} + +variable "create_access_policy" { + description = "Determines whether the IAM policy should be created" + type = bool + default = true +} + +variable "access_iam_statements" { + description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" + type = any + default = {} +} + +variable "access_kms_key_arns" { + description = "A list of KMS key ARNs the access IAM role is permitted to decrypt" + type = list(string) + default = [] +} + +variable "access_secret_arns" { + description = "A list of SecretManager secret ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} + +variable "access_source_s3_bucket_arns" { + description = "A list of S3 bucket ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} + +variable "access_target_s3_bucket_arns" { + description = "A list of S3 bucket ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} + +variable "access_target_elasticsearch_arns" { + description = "A list of Elasticsearch ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} + +variable "access_target_kinesis_arns" { + description = "A list of Kinesis ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} + +variable "access_target_dynamodb_table_arns" { + description = "A list of DynamoDB table ARNs the access IAM role is permitted to access" + type = list(string) + default = [] +} diff --git a/versions.tf b/versions.tf index bd3e87b..497d5ab 100644 --- a/versions.tf +++ b/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = ">= 0.13.1" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.17" + version = ">= 5.0" } time = { source = "hashicorp/time" - version = ">=0.7.2" + version = ">= 0.9" } } }