-
Notifications
You must be signed in to change notification settings - Fork 118
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DO NOT MERGE Test aws integration on localstack API
- Loading branch information
Showing
10 changed files
with
338 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| version: "3.8" | ||
| services: | ||
| localstack: | ||
| container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" | ||
| image: localstack/localstack | ||
| hostname: localstack | ||
| ports: | ||
| - "4566:4566" # LocalStack Gateway | ||
| environment: | ||
| - SERVICES=sqs,sns | ||
| - DEBUG=1 | ||
| - DOCKER_HOST=unix:///var/run/docker.sock | ||
| - HOST_TMP_FOLDER=${TMPDIR} | ||
| - HOSTNAME_EXTERNAL=localstack | ||
| - S3_HOSTNAME=localstack | ||
| volumes: | ||
| - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" | ||
| - "/var/run/docker.sock:/var/run/docker.sock" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
173 changes: 173 additions & 0 deletions
173
test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| resource "aws_cloudwatch_metric_stream" "main" { | ||
| name = "my-metric-stream" | ||
| role_arn = aws_iam_role.metric_stream_to_firehose.arn | ||
| firehose_arn = aws_kinesis_firehose_delivery_stream.s3_stream.arn | ||
| output_format = "json" | ||
|
|
||
| include_filter { | ||
| namespace = "AWS/EC2" | ||
| metric_names = ["CPUUtilization", "NetworkOut"] | ||
| } | ||
|
|
||
| include_filter { | ||
| namespace = "AWS/EBS" | ||
| metric_names = [] | ||
| } | ||
| } | ||
|
|
||
| # https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html | ||
| data "aws_iam_policy_document" "streams_assume_role" { | ||
| statement { | ||
| effect = "Allow" | ||
|
|
||
| principals { | ||
| type = "Service" | ||
| identifiers = ["streams.metrics.cloudwatch.amazonaws.com"] | ||
| } | ||
|
|
||
| actions = [ | ||
| "sts:AssumeRole", | ||
| "iam:passRole", | ||
| "cloudwatch:PutMetricStream" | ||
| ] | ||
| } | ||
| } | ||
|
|
||
| resource "aws_iam_role" "metric_stream_to_firehose" { | ||
| name = "metric_stream_to_firehose_role" | ||
| assume_role_policy = data.aws_iam_policy_document.streams_assume_role.json | ||
| } | ||
|
|
||
| # https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html | ||
| data "aws_iam_policy_document" "metric_stream_to_firehose" { | ||
| statement { | ||
| effect = "Allow" | ||
|
|
||
| actions = [ | ||
| "firehose:PutRecord", | ||
| "firehose:PutRecordBatch", | ||
| ] | ||
|
|
||
| resources = [aws_kinesis_firehose_delivery_stream.s3_stream.arn] | ||
| } | ||
| } | ||
| resource "aws_iam_role_policy" "metric_stream_to_firehose" { | ||
| name = "default" | ||
| role = aws_iam_role.metric_stream_to_firehose.id | ||
| policy = data.aws_iam_policy_document.metric_stream_to_firehose.json | ||
| } | ||
|
|
||
| resource "aws_s3_bucket" "bucket" { | ||
| bucket = "metric-stream-test-bucket" | ||
| } | ||
|
|
||
| resource "aws_s3_bucket_acl" "bucket_acl" { | ||
| bucket = aws_s3_bucket.bucket.id | ||
| acl = "private" | ||
| } | ||
|
|
||
| data "aws_iam_policy_document" "firehose_assume_role" { | ||
| statement { | ||
| effect = "Allow" | ||
|
|
||
| principals { | ||
| type = "Service" | ||
| identifiers = ["firehose.amazonaws.com"] | ||
| } | ||
|
|
||
| actions = [ | ||
| "sts:AssumeRole", | ||
| "iam:passRole", | ||
| "cloudwatch:PutMetricStream" | ||
| ] | ||
| } | ||
| } | ||
|
|
||
| resource "aws_iam_role" "firehose_to_s3" { | ||
| assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json | ||
| } | ||
|
|
||
| data "aws_iam_policy_document" "firehose_to_s3" { | ||
| statement { | ||
| effect = "Allow" | ||
|
|
||
| actions = [ | ||
| "s3:AbortMultipartUpload", | ||
| "s3:GetBucketLocation", | ||
| "s3:GetObject", | ||
| "s3:ListBucket", | ||
| "s3:ListBucketMultipartUploads", | ||
| "s3:PutObject", | ||
| ] | ||
|
|
||
| resources = [ | ||
| aws_s3_bucket.bucket.arn, | ||
| "${aws_s3_bucket.bucket.arn}/*", | ||
| ] | ||
| } | ||
| } | ||
|
|
||
| resource "aws_iam_role_policy" "firehose_to_s3" { | ||
| name = "default" | ||
| role = aws_iam_role.firehose_to_s3.id | ||
| policy = data.aws_iam_policy_document.firehose_to_s3.json | ||
| } | ||
|
|
||
| resource "aws_kinesis_firehose_delivery_stream" "s3_stream" { | ||
| name = "metric-stream-test-stream" | ||
| destination = "s3" | ||
|
|
||
| s3_configuration { | ||
| role_arn = aws_iam_role.firehose_to_s3.arn | ||
| bucket_arn = aws_s3_bucket.bucket.arn | ||
| } | ||
| } | ||
|
|
||
| resource "aws_iam_user" "ecs_deployer" { | ||
| name = "ecs_deployer" | ||
| path = "*" | ||
| } | ||
|
|
||
| # The most important part is the iam:PassRole. With that, this user can give roles to ECS tasks. | ||
| # In theory the user can give the task Admin rights. To make sure that does not happen we restrict | ||
| # the user and allow him only to hand out roles in /ecs/ path. You still need to be careful not | ||
| # to have any roles in there with full admin rights, but no ECS task should have these rights! | ||
| resource "aws_iam_user_policy" "ecs_deployer_policy" { | ||
| name = "ecs_deployer_policy" | ||
| user = aws_iam_user.ecs_deployer.name | ||
| policy = jsonencode( | ||
| { | ||
| "Version" : "2012-10-17", | ||
| "Statement" : [ | ||
| { | ||
| "Effect" : "Allow", | ||
| "Action" : [ | ||
| "ecs:RegisterTaskDefinition", | ||
| "ecs:DescribeTaskDefinitions", | ||
| "ecs:ListTaskDefinitions", | ||
| "ecs:CreateService", | ||
| "ecs:UpdateService", | ||
| "ecs:DescribeServices", | ||
| "ecs:ListServices" | ||
| ], | ||
| "Resource" : "*" | ||
| }, | ||
| { | ||
| "Effect" : "Allow", | ||
| "Action" : [ | ||
| "cloudwatch:PutMetricStream" | ||
| ], | ||
| "Resource" : "*" | ||
| }, | ||
| { | ||
| "Effect" : "Allow", | ||
| "Action" : ["iam:PassRole"], | ||
| "Resource" : "*" | ||
| } | ||
| ] | ||
| }) | ||
| } | ||
|
|
||
| resource "aws_iam_access_key" "ecs_deployer" { | ||
| user = aws_iam_user.ecs_deployer.name | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.