From ae4c5428d64f2bb86c6c56aaa3ea5bec8c8a4d84 Mon Sep 17 00:00:00 2001 From: eiathom Date: Fri, 2 Feb 2024 18:10:52 +0000 Subject: [PATCH] [TailAware] Builds load-balancing service --- README.md | 4 +- loadbalancing-collector-configuration.yaml | 4 +- stack/cloudformation/main.yaml | 42 +++- stack/cloudformation/privateservice.yaml | 243 +++++++++++++++++++ stack/cloudformation/publicloadbalancer.yaml | 23 +- stack/cloudformation/secureprivatecloud.yaml | 37 ++- tailaware-collector-configuration.yaml | 4 +- 7 files changed, 345 insertions(+), 12 deletions(-) create mode 100644 stack/cloudformation/privateservice.yaml diff --git a/README.md b/README.md index 3b1848a..1425df2 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ aws configure * S3 * IAM (Read-Only) -# run checks on the template for proper syntax -cfn-lint --template template.yaml --region eu-west-1 --ignore-checks W +# run checks on the template +cfn-lint --template stack/cloudformation/main.yaml --region eu-west-1 # create the bucket for the stacks aws cloudformation create-stack \ diff --git a/loadbalancing-collector-configuration.yaml b/loadbalancing-collector-configuration.yaml index 09cced5..b1138ac 100644 --- a/loadbalancing-collector-configuration.yaml +++ b/loadbalancing-collector-configuration.yaml @@ -18,9 +18,9 @@ exporters: # we are going to export metric signals directly to the back-end # see: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings otlp/newrelic: - endpoint: ${OTLP_NEW_RELIC_EXPORTER_ENDPOINT} + endpoint: otlp.eu01.nr-data.net:443 headers: - api-key: ${NEW_RELIC_API_KEY} + api-key: ${TELEMETRY_BACKEND_API_KEY} # trace and log signals will be exported downstream to the tail-aware Collector (and then on to the telemetry back-end) # see: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/loadbalancingexporter diff --git a/stack/cloudformation/main.yaml b/stack/cloudformation/main.yaml index b4e04e1..4ca3fcd 100644 --- a/stack/cloudformation/main.yaml +++ b/stack/cloudformation/main.yaml @@ -22,6 +22,28 @@ Parameters: Type: Number Description: The port for HTTP traffic to be forwarded to Default: 4318 + LoadBalancingCollectorMaxCapacity: + Type: Number + Description: The maximum number of tasks the LoadBalancing Collector should scale out to + Default: 5 + LoadBalancingCollectorMinCapacity: + Type: Number + Description: The minimum number of tasks the LoadBalancing Collector should scale in to + Default: 1 + LoadBalancingCollectorScalingTargetPercentage: + Type: Number + Description: The average memory utilisation percentage used to begin scaling the LoadBalancing Collector + Default: 70 + LoadBalancingCollectorDesiredCount: + Type: Number + Description: The desired number of initial tasks + Default: 1 + LoadBalancingCollectorCpu: + Type: Number + Default: 256 + LoadBalancingCollectorMemory: + Type: Number + Default: 512 Resources: SecurePrivateCloud: @@ -38,6 +60,24 @@ Resources: TemplateURL: !Sub "https://s3.amazonaws.com/${BucketName}/publicloadbalancer.yaml" Parameters: VpcId: !GetAtt SecurePrivateCloud.Outputs.VpcId - SubnetIds: !GetAtt SecurePrivateCloud.Outputs.SubnetIds + PublicSubnetIds: !GetAtt SecurePrivateCloud.Outputs.PublicSubnetIds + HttpTrafficPort: !Ref HttpTrafficPort + HttpTargetPort: !Ref HttpTargetPort + + PrivateService: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: !Sub "https://s3.amazonaws.com/${BucketName}/privateservice.yaml" + Parameters: + VpcId: !GetAtt SecurePrivateCloud.Outputs.VpcId + PublicSubnetIds: !GetAtt SecurePrivateCloud.Outputs.PublicSubnetIds HttpTrafficPort: !Ref HttpTrafficPort HttpTargetPort: !Ref HttpTargetPort + PublicLoadBalancerSecurityGroupId: !GetAtt PublicLoadBalancer.Outputs.PublicLoadBalancerSecurityGroupId + HttpTargetGroupArn: !GetAtt PublicLoadBalancer.Outputs.HttpTargetGroupArn + LoadBalancingCollectorMaxCapacity: !Ref LoadBalancingCollectorMaxCapacity + LoadBalancingCollectorMinCapacity: !Ref LoadBalancingCollectorMinCapacity + LoadBalancingCollectorScalingTargetPercentage: !Ref LoadBalancingCollectorScalingTargetPercentage + LoadBalancingCollectorDesiredCount: !Ref LoadBalancingCollectorDesiredCount + LoadBalancingCollectorCpu: !Ref LoadBalancingCollectorCpu + LoadBalancingCollectorMemory: !Ref LoadBalancingCollectorMemory diff --git a/stack/cloudformation/privateservice.yaml b/stack/cloudformation/privateservice.yaml new file mode 100644 index 0000000..f8857df --- /dev/null +++ b/stack/cloudformation/privateservice.yaml @@ -0,0 +1,243 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Public-facing ELB listening on port 80 for path "/v1/*" + +Parameters: + VpcId: + Type: AWS::EC2::VPC::Id + Description: The VPC ID where the load balancer will be deployed + Default: vpc-123456789 + PublicLoadBalancerSecurityGroupId: + Type: AWS::EC2::SecurityGroup::Id + Description: The SecurityGroup Id associated to the public load balancer + Default: Default + PrivateSubnetIds: + Type: List + Description: The Subnets (multiple for HA) configured for internal traffic only + Default: subnet-4256274569,subnet-82458245802475 + HttpTargetGroupArn: + Type: String + Description: The ARN of the HTTP target group + Default: Default + HttpTrafficPort: + Type: Number + Description: The port for HTTP traffic + Default: 80 + HttpTargetPort: + Type: Number + Description: The port for HTTP traffic to be forwarded to + Default: 4318 + OtelCollectorNamespaceName: + Type: String + Description: The name of the namespace for service discovery (discovery of Collectors hostnames) + Default: otelcollector + LoadBalancingCollectorMaxCapacity: + Type: Number + Description: The maximum number of tasks the LoadBalancing Collector should scale out to + Default: 5 + LoadBalancingCollectorMinCapacity: + Type: Number + Description: The minimum number of tasks the LoadBalancing Collector should scale in to + Default: 1 + LoadBalancingCollectorScalingTargetPercentage: + Type: Number + Description: The average memory utilisation percentage used to begin scaling the LoadBalancing Collector + Default: 70 + LoadBalancingCollectorServiceName: + Type: String + Description: The name of the load balancing collector service + Default: loadbalancingcollector + ImageName: + Type: String + Description: The image to leverage for the task + Default: "otel/opentelemetry-collector:0.89.0" + LoadBalancingCollectorDesiredCount: + Type: Number + Description: The desired number of initial tasks + Default: 1 + # 1024 == 1 CPU + LoadBalancingCollectorCpu: + Type: Number + Default: 256 + LoadBalancingCollectorMemory: + Type: Number + Default: 512 + TelemetryBackendApiKeySsmKeyName: + Type: AWS::SSM::Parameter::Name + Description: Name of the SSM secret parameter key whose value is a telemetry backend API key + Default: "/otel/collector/configuration/telemetry-backend-api-key" + # value of the SSM variable is: env:COLLECTOR_CONFIGURATION + LoadBalancingCollectorConfMap: + Type: AWS::SSM::Parameter::Value + Description: Name of the SSM parameter key whose value points to the location of the Collector configuration + Default: "/otel/collector/configuration/loadbalancing-collector-conf-map-type" + LoadBalancingCollectorConfigurationYaml: + Type: AWS::SSM::Parameter::Value + Description: Name of the SSM parameter key whose value is the actual configuration + Default: "/otel/collector/configuration/loadbalancing-collector-configuration" + +Resources: + PrivateServiceSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allow traffic from public ALB to private ECS + VpcId: !Ref VpcId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: !Ref HttpTrafficPort + ToPort: !Ref HttpTrafficPort + SourceSecurityGroupId: !Ref PublicLoadBalancerSecurityGroupId + + OtelCollectorCluster: + Type: AWS::ECS::Cluster + + OtelCollectorCloudMapNamespace: + Type: AWS::ServiceDiscovery::PrivateDnsNamespace + Properties: + Name: !Sub "${OtelCollectorNamespaceName}.local" + Vpc: !Ref VpcId + + # LoadBalancingCollector: Scalable Service + LoadBalancingCollectorDiscovery: + Type: AWS::ServiceDiscovery::Service + Properties: + Name: loadbalancingcollector + NamespaceId: !Ref OtelCollectorCloudMapNamespace + + ECSTaskRole: + Type: AWS::IAM::Role + Properties: + Description: Allows ECS tasks to call AWS services on your behalf + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: "" + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: "sts:AssumeRole" + Policies: + - PolicyName: AwsTelemetryPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "logs:PutLogEvents" + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:DescribeLogStreams" + - "logs:DescribeLogGroups" + - "xray:PutTraceSegments" + - "xray:PutTelemetryRecords" + - "xray:GetSamplingRules" + - "xray:GetSamplingTargets" + - "xray:GetSamplingStatisticSummaries" + - "ssm:GetParameters" + Resource: "*" + + ECSTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: "" + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: "sts:AssumeRole" + ManagedPolicyArns: + - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + - "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" + - "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess" + + LoadBalancingCollectorTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Ref LoadBalancingCollectorServiceName + Cpu: !Ref LoadBalancingCollectorCpu + Memory: !Ref LoadBalancingCollectorMemory + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: !GetAtt ECSTaskExecutionRole.Arn + TaskRoleArn: !GetAtt ECSTaskRole.Arn + ContainerDefinitions: + - Name: !Ref LoadBalancingCollectorServiceName + Image: !Ref ImageName + Cpu: !Ref LoadBalancingCollectorCpu + Memory: !Ref LoadBalancingCollectorMemory + PortMappings: + - ContainerPort: !Ref HttpTargetPort + Protocol: tcp + AppProtocol: http + Command: + - "--config" + - !Sub "${LoadBalancingCollectorConfMap}" + Secrets: + - Name: TELEMETRY_BACKEND_API_KEY + ValueFrom: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${TelemetryBackendApiKeySsmKeyName}" + Environment: + - Name: COLLECTOR_CONFIGURATION + Value: !Ref LoadBalancingCollectorConfigurationYaml + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: "True" + awslogs-group: !Ref LoadBalancingCollectorServiceName + awslogs-region: !Ref "AWS::Region" + awslogs-stream-prefix: otel + + LoadBalancingCollector: + Type: AWS::ECS::Service + Properties: + ServiceName: !Ref LoadBalancingCollectorServiceName + Cluster: !Ref OtelCollectorCluster + TaskDefinition: !Ref LoadBalancingCollectorTaskDefinition + LaunchType: FARGATE + PropagateTags: SERVICE + SchedulingStrategy: REPLICA + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 75 + ServiceRegistries: + - RegistryArn: !GetAtt LoadBalancingCollectorDiscovery.Arn + DesiredCount: !Ref LoadBalancingCollectorDesiredCount + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: DISABLED + Subnets: !Ref PrivateSubnetIds + SecurityGroups: + - !Ref PrivateServiceSecurityGroup + LoadBalancers: + - ContainerName: !Ref LoadBalancingCollectorServiceName + ContainerPort: !Ref HttpTargetPort + TargetGroupArn: !Ref HttpTargetGroupArn + Tags: + - Key: Name + Value: !Join ["-", [!Ref LoadBalancingCollectorServiceName, "service"]] + + LoadBalancingCollectorScalingTarget: + Type: "AWS::ApplicationAutoScaling::ScalableTarget" + Properties: + MaxCapacity: !Ref LoadBalancingCollectorMaxCapacity + MinCapacity: !Ref LoadBalancingCollectorMinCapacity + ResourceId: !Sub "service/${OtelCollectorCluster}/${LoadBalancingCollector}" + ScalableDimension: "ecs:service:DesiredCount" + ServiceNamespace: ecs + + # scaling the load balancing collector service task based on average memory utilisation + LoadBalancingCollectorScalingPolicy: + Type: "AWS::ApplicationAutoScaling::ScalingPolicy" + Properties: + PolicyName: LoadBalancingCollectorAutoScalingPolicy + PolicyType: TargetTrackingScaling + ScalingTargetId: !Ref LoadBalancingCollectorScalingTarget + TargetTrackingScalingPolicyConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: "ECSServiceAverageMemoryUtilization" + TargetValue: !Ref LoadBalancingCollectorScalingTargetPercentage + # see: https://docs.aws.amazon.com/autoscaling/application/userguide/target-tracking-scaling-policy-overview.html#target-tracking-cooldown + # in seconds, defaulting to both being 5 minutes of cooldown (this should be tuned after many load generation verification runs) + ScaleInCooldown: 300 + ScaleOutCooldown: 300 diff --git a/stack/cloudformation/publicloadbalancer.yaml b/stack/cloudformation/publicloadbalancer.yaml index 5b0754f..d4e9885 100644 --- a/stack/cloudformation/publicloadbalancer.yaml +++ b/stack/cloudformation/publicloadbalancer.yaml @@ -6,7 +6,7 @@ Parameters: Type: AWS::EC2::VPC::Id Description: The VPC ID where the load balancer will be deployed Default: vpc-2465842045826 - SubnetIds: + PublicSubnetIds: Type: List Description: The Subnets (multiple for HA) configured to allow internet access Default: subnet-27465297465,subnet-265924692 @@ -20,6 +20,8 @@ Parameters: Default: 4318 Resources: + # Security groups act as a virtual firewall for your ELB to control inbound and outbound traffic + # In this case, the security group allows inbound traffic on TCP port 80 from anywhere (0.0.0.0/0) (which is necessary for a public-facing load balancer) PublicLoadBalancerSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: @@ -36,11 +38,14 @@ Resources: - Key: Name Value: public-load-balancer-security-group + # The Scheme is set to internet-facing to make it publicly accessible + # It's associated with subnets (that should be in different Availability Zones for high availability) + # These subnets must be connected to the internet (typically through an Internet Gateway in the VPC) PublicLoadBalancer: Type: AWS::ElasticLoadBalancingV2::LoadBalancer Properties: Name: public-load-balancer - Subnets: !Ref SubnetIds + Subnets: !Ref PublicSubnetIds SecurityGroups: - Ref: PublicLoadBalancerSecurityGroup Scheme: internet-facing @@ -50,6 +55,9 @@ Resources: - Key: Name Value: public-load-balancer + # A listener checks for connection requests to the load balancer based on the protocol and port + # Here, it listens on port 80 for HTTP traffic + # The default action is configured to return a 404 (Not Found) fixed response, effectively serving as a fallback if no rules match the incoming request PublicLoadBalancerListener: Type: AWS::ElasticLoadBalancingV2::Listener Properties: @@ -63,6 +71,9 @@ Resources: ContentType: "text/plain" MessageBody: "Not Found" + # Listener Rules define how the incoming traffic is routed + # In this case, there's a rule to forward requests with a path starting with "/v1/" to a specific target group + # This is where you define the criteria for forwarding requests to specific targets (e.g., EC2 instances, ECS services) ListenerRuleForV1Path: Type: AWS::ElasticLoadBalancingV2::ListenerRule Properties: @@ -75,6 +86,8 @@ Resources: - Type: forward TargetGroupArn: !Ref HttpTargetGroup + # Target groups are used to route requests to one or more registered targets + # This template defines a target group with targets (e.g., EC2 instances, ECS services) that listen on port 4318 using HTTP protocol (within the specified VPC) HttpTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: @@ -102,3 +115,9 @@ Outputs: LoadBalancerDNSName: Description: DNS name of the public load balancer Value: !GetAtt PublicLoadBalancer.DNSName + PublicLoadBalancerSecurityGroupId: + Description: SecurityGroups Id associated to the public load balancer + Value: !GetAtt PublicLoadBalancerSecurityGroup.GroupId + HttpTargetGroupArn: + Description: The ARN of the HTTP target group + Value: !GetAtt HttpTargetGroup.TargetGroupArn diff --git a/stack/cloudformation/secureprivatecloud.yaml b/stack/cloudformation/secureprivatecloud.yaml index d9426f1..32dbd6d 100644 --- a/stack/cloudformation/secureprivatecloud.yaml +++ b/stack/cloudformation/secureprivatecloud.yaml @@ -8,8 +8,12 @@ Parameters: Default: secure-private-cloud PublicSubnetName: Type: String - Description: The name of the Subnet resource(s) + Description: The name of the public Subnet resource(s) Default: public-subnet + PrivateSubnetName: + Type: String + Description: The name of the private Subnet resource(s) + Default: private-subnet Resources: # Defines a Virtual Private Cloud (VPC) with a CIDR block of 10.0.0.0/16 @@ -53,6 +57,30 @@ Resources: - Key: Name Value: !Join ["-", [!Ref PublicSubnetName, "2"]] + # These subnets are part of the VPC and are configured in different Availability Zones for high availability + # They are meant for resources that need to be not accessible from the internet, like a instances or services + PrivateSubnetOne: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref SecureVpc + CidrBlock: 10.0.3.0/24 + AvailabilityZone: !Select [0, !GetAZs ''] + Tags: + - Key: Name + Value: !Join ["-", [!Ref PrivateSubnetName, "1"]] + + # These subnets are part of the VPC and are configured in different Availability Zones for high availability + # They are meant for resources that need to be not accessible from the internet, like a instances or services + PrivateSubnetTwo: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref SecureVpc + CidrBlock: 10.0.4.0/24 + AvailabilityZone: !Select [1, !GetAZs ''] + Tags: + - Key: Name + Value: !Join ["-", [!Ref PrivateSubnetName, "2"]] + # Provides a route for traffic between your VPC and the internet # It's necessary for any resource in the public subnets to be accessible from the internet InternetGateway: @@ -105,6 +133,9 @@ Outputs: VpcId: Description: ID of the VPC Value: !GetAtt SecureVpc.VpcId - SubnetIds: - Description: ID(s) of the Subnet(s) + PublicSubnetIds: + Description: ID(s) of the public Subnet(s) Value: !Join [",", [!Ref PublicSubnetOne, !Ref PublicSubnetTwo]] + PrivateSubnetIds: + Description: ID(s) of the private Subnet(s) + Value: !Join [",", [!Ref PrivateSubnetOne, !Ref PrivateSubnetTwo]] diff --git a/tailaware-collector-configuration.yaml b/tailaware-collector-configuration.yaml index f44e64d..2462343 100644 --- a/tailaware-collector-configuration.yaml +++ b/tailaware-collector-configuration.yaml @@ -49,9 +49,9 @@ exporters: # example telemetry back-end # see: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings otlp/newrelic: - endpoint: ${OTLP_NEW_RELIC_EXPORTER_ENDPOINT} + endpoint: otlp.eu01.nr-data.net:443 headers: - api-key: ${NEW_RELIC_API_KEY} + api-key: ${TELEMETRY_BACKEND_API_KEY} service: pipelines: