Merge pull request #6 from eiathom/TailAware-builds-load-balancing-se…

…rvice

[TailAware] Builds load-balancing service

README.md

@@ -51,8 +51,8 @@ aws configure
 * S3
 * IAM (Read-Only)
 
-# run checks on the template for proper syntax
-cfn-lint --template template.yaml --region eu-west-1 --ignore-checks W
+# run checks on the template
+cfn-lint --template stack/cloudformation/main.yaml --region eu-west-1
 
 # create the bucket for the stacks
 aws cloudformation create-stack \

loadbalancing-collector-configuration.yaml

@@ -18,9 +18,9 @@ exporters:
   # we are going to export metric signals directly to the back-end
   # see: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings
   otlp/newrelic:
-    endpoint: ${OTLP_NEW_RELIC_EXPORTER_ENDPOINT}
+    endpoint: otlp.eu01.nr-data.net:443
     headers:
-      api-key: ${NEW_RELIC_API_KEY}
+      api-key: ${TELEMETRY_BACKEND_API_KEY}
 
   # trace and log signals will be exported downstream to the tail-aware Collector (and then on to the telemetry back-end)
   # see: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/loadbalancingexporter

stack/cloudformation/main.yaml

@@ -22,6 +22,28 @@ Parameters:
     Type: Number
     Description: The port for HTTP traffic to be forwarded to
     Default: 4318
+  LoadBalancingCollectorMaxCapacity:
+    Type: Number
+    Description: The maximum number of tasks the LoadBalancing Collector should scale out to
+    Default: 5
+  LoadBalancingCollectorMinCapacity:
+    Type: Number
+    Description: The minimum number of tasks the LoadBalancing Collector should scale in to
+    Default: 1
+  LoadBalancingCollectorScalingTargetPercentage:
+    Type: Number
+    Description: The average memory utilisation percentage used to begin scaling the LoadBalancing Collector
+    Default: 70
+  LoadBalancingCollectorDesiredCount:
+    Type: Number
+    Description: The desired number of initial tasks
+    Default: 1
+  LoadBalancingCollectorCpu:
+    Type: Number
+    Default: 256
+  LoadBalancingCollectorMemory:
+    Type: Number
+    Default: 512
 
 Resources:
   SecurePrivateCloud:
@@ -38,6 +60,24 @@ Resources:
       TemplateURL: !Sub "https://s3.amazonaws.com/${BucketName}/publicloadbalancer.yaml"
       Parameters:
         VpcId: !GetAtt SecurePrivateCloud.Outputs.VpcId
-        SubnetIds: !GetAtt SecurePrivateCloud.Outputs.SubnetIds
+        PublicSubnetIds: !GetAtt SecurePrivateCloud.Outputs.PublicSubnetIds
+        HttpTrafficPort: !Ref HttpTrafficPort
+        HttpTargetPort: !Ref HttpTargetPort
+
+  PrivateService:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      TemplateURL: !Sub "https://s3.amazonaws.com/${BucketName}/privateservice.yaml"
+      Parameters:
+        VpcId: !GetAtt SecurePrivateCloud.Outputs.VpcId
+        PublicSubnetIds: !GetAtt SecurePrivateCloud.Outputs.PublicSubnetIds
         HttpTrafficPort: !Ref HttpTrafficPort
         HttpTargetPort: !Ref HttpTargetPort
+        PublicLoadBalancerSecurityGroupId: !GetAtt PublicLoadBalancer.Outputs.PublicLoadBalancerSecurityGroupId
+        HttpTargetGroupArn: !GetAtt PublicLoadBalancer.Outputs.HttpTargetGroupArn
+        LoadBalancingCollectorMaxCapacity: !Ref LoadBalancingCollectorMaxCapacity
+        LoadBalancingCollectorMinCapacity: !Ref LoadBalancingCollectorMinCapacity
+        LoadBalancingCollectorScalingTargetPercentage: !Ref LoadBalancingCollectorScalingTargetPercentage
+        LoadBalancingCollectorDesiredCount: !Ref LoadBalancingCollectorDesiredCount
+        LoadBalancingCollectorCpu: !Ref LoadBalancingCollectorCpu
+        LoadBalancingCollectorMemory: !Ref LoadBalancingCollectorMemory

stack/cloudformation/privateservice.yaml

@@ -0,0 +1,243 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Public-facing ELB listening on port 80 for path "/v1/*"
+
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+    Description: The VPC ID where the load balancer will be deployed
+    Default: vpc-123456789
+  PublicLoadBalancerSecurityGroupId:
+    Type: AWS::EC2::SecurityGroup::Id
+    Description: The SecurityGroup Id associated to the public load balancer
+    Default: Default
+  PrivateSubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: The Subnets (multiple for HA) configured for internal traffic only
+    Default: subnet-4256274569,subnet-82458245802475
+  HttpTargetGroupArn:
+    Type: String
+    Description: The ARN of the HTTP target group
+    Default: Default
+  HttpTrafficPort:
+    Type: Number
+    Description: The port for HTTP traffic
+    Default: 80
+  HttpTargetPort:
+    Type: Number
+    Description: The port for HTTP traffic to be forwarded to
+    Default: 4318
+  OtelCollectorNamespaceName:
+    Type: String
+    Description: The name of the namespace for service discovery (discovery of Collectors hostnames)
+    Default: otelcollector
+  LoadBalancingCollectorMaxCapacity:
+    Type: Number
+    Description: The maximum number of tasks the LoadBalancing Collector should scale out to
+    Default: 5
+  LoadBalancingCollectorMinCapacity:
+    Type: Number
+    Description: The minimum number of tasks the LoadBalancing Collector should scale in to
+    Default: 1
+  LoadBalancingCollectorScalingTargetPercentage:
+    Type: Number
+    Description: The average memory utilisation percentage used to begin scaling the LoadBalancing Collector
+    Default: 70
+  LoadBalancingCollectorServiceName:
+    Type: String
+    Description: The name of the load balancing collector service
+    Default: loadbalancingcollector
+  ImageName:
+    Type: String
+    Description: The image to leverage for the task
+    Default: "otel/opentelemetry-collector:0.89.0"
+  LoadBalancingCollectorDesiredCount:
+    Type: Number
+    Description: The desired number of initial tasks
+    Default: 1
+  # 1024 == 1 CPU
+  LoadBalancingCollectorCpu:
+    Type: Number
+    Default: 256
+  LoadBalancingCollectorMemory:
+    Type: Number
+    Default: 512
+  TelemetryBackendApiKeySsmKeyName:
+    Type: AWS::SSM::Parameter::Name
+    Description: Name of the SSM secret parameter key whose value is a telemetry backend API key
+    Default: "/otel/collector/configuration/telemetry-backend-api-key"
+  # value of the SSM variable is: env:COLLECTOR_CONFIGURATION
+  LoadBalancingCollectorConfMap:
+    Type: AWS::SSM::Parameter::Value<String>
+    Description: Name of the SSM parameter key whose value points to the location of the Collector configuration
+    Default: "/otel/collector/configuration/loadbalancing-collector-conf-map-type"
+  LoadBalancingCollectorConfigurationYaml:
+    Type: AWS::SSM::Parameter::Value<String>
+    Description: Name of the SSM parameter key whose value is the actual configuration
+    Default: "/otel/collector/configuration/loadbalancing-collector-configuration"
+
+Resources:
+  PrivateServiceSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Allow traffic from public ALB to private ECS
+      VpcId: !Ref VpcId
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: !Ref HttpTrafficPort
+          ToPort: !Ref HttpTrafficPort
+          SourceSecurityGroupId: !Ref PublicLoadBalancerSecurityGroupId
+
+  OtelCollectorCluster:
+    Type: AWS::ECS::Cluster
+
+  OtelCollectorCloudMapNamespace:
+    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
+    Properties:
+      Name: !Sub "${OtelCollectorNamespaceName}.local"
+      Vpc: !Ref VpcId
+
+  # LoadBalancingCollector: Scalable Service
+  LoadBalancingCollectorDiscovery:
+    Type: AWS::ServiceDiscovery::Service
+    Properties:
+      Name: loadbalancingcollector
+      NamespaceId: !Ref OtelCollectorCloudMapNamespace
+
+  ECSTaskRole:
+    Type: AWS::IAM::Role
+    Properties:
+      Description: Allows ECS tasks to call AWS services on your behalf
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: ""
+            Effect: Allow
+            Principal:
+              Service: ecs-tasks.amazonaws.com
+            Action: "sts:AssumeRole"
+      Policies:
+        - PolicyName: AwsTelemetryPolicy
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - "logs:PutLogEvents"
+                  - "logs:CreateLogGroup"
+                  - "logs:CreateLogStream"
+                  - "logs:DescribeLogStreams"
+                  - "logs:DescribeLogGroups"
+                  - "xray:PutTraceSegments"
+                  - "xray:PutTelemetryRecords"
+                  - "xray:GetSamplingRules"
+                  - "xray:GetSamplingTargets"
+                  - "xray:GetSamplingStatisticSummaries"
+                  - "ssm:GetParameters"
+                Resource: "*"
+
+  ECSTaskExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: ""
+            Effect: Allow
+            Principal:
+              Service: ecs-tasks.amazonaws.com
+            Action: "sts:AssumeRole"
+      ManagedPolicyArns:
+        - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+        - "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
+        - "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess"
+
+  LoadBalancingCollectorTaskDefinition:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      Family: !Ref LoadBalancingCollectorServiceName
+      Cpu: !Ref LoadBalancingCollectorCpu
+      Memory: !Ref LoadBalancingCollectorMemory
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      ExecutionRoleArn: !GetAtt ECSTaskExecutionRole.Arn
+      TaskRoleArn: !GetAtt ECSTaskRole.Arn
+      ContainerDefinitions:
+        - Name: !Ref LoadBalancingCollectorServiceName
+          Image: !Ref ImageName
+          Cpu: !Ref LoadBalancingCollectorCpu
+          Memory: !Ref LoadBalancingCollectorMemory
+          PortMappings:
+            - ContainerPort: !Ref HttpTargetPort
+              Protocol: tcp
+              AppProtocol: http
+          Command:
+            - "--config"
+            - !Sub "${LoadBalancingCollectorConfMap}"
+          Secrets:
+            - Name: TELEMETRY_BACKEND_API_KEY
+              ValueFrom: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${TelemetryBackendApiKeySsmKeyName}"
+          Environment:
+            - Name: COLLECTOR_CONFIGURATION
+              Value: !Ref LoadBalancingCollectorConfigurationYaml
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-create-group: "True"
+              awslogs-group: !Ref LoadBalancingCollectorServiceName
+              awslogs-region: !Ref "AWS::Region"
+              awslogs-stream-prefix: otel
+
+  LoadBalancingCollector:
+    Type: AWS::ECS::Service
+    Properties:
+      ServiceName: !Ref LoadBalancingCollectorServiceName
+      Cluster: !Ref OtelCollectorCluster
+      TaskDefinition: !Ref LoadBalancingCollectorTaskDefinition
+      LaunchType: FARGATE
+      PropagateTags: SERVICE
+      SchedulingStrategy: REPLICA
+      DeploymentConfiguration:
+        MaximumPercent: 200
+        MinimumHealthyPercent: 75
+      ServiceRegistries:
+        - RegistryArn: !GetAtt LoadBalancingCollectorDiscovery.Arn
+      DesiredCount: !Ref LoadBalancingCollectorDesiredCount
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: DISABLED
+          Subnets: !Ref PrivateSubnetIds
+          SecurityGroups:
+            - !Ref PrivateServiceSecurityGroup
+      LoadBalancers:
+        - ContainerName: !Ref LoadBalancingCollectorServiceName
+          ContainerPort: !Ref HttpTargetPort
+          TargetGroupArn: !Ref HttpTargetGroupArn
+      Tags:
+        - Key: Name
+          Value: !Join ["-", [!Ref LoadBalancingCollectorServiceName, "service"]]
+
+  LoadBalancingCollectorScalingTarget:
+    Type: "AWS::ApplicationAutoScaling::ScalableTarget"
+    Properties:
+      MaxCapacity: !Ref LoadBalancingCollectorMaxCapacity
+      MinCapacity: !Ref LoadBalancingCollectorMinCapacity
+      ResourceId: !Sub "service/${OtelCollectorCluster}/${LoadBalancingCollector}"
+      ScalableDimension: "ecs:service:DesiredCount"
+      ServiceNamespace: ecs
+
+  # scaling the load balancing collector service task based on average memory utilisation
+  LoadBalancingCollectorScalingPolicy:
+    Type: "AWS::ApplicationAutoScaling::ScalingPolicy"
+    Properties:
+      PolicyName: LoadBalancingCollectorAutoScalingPolicy
+      PolicyType: TargetTrackingScaling
+      ScalingTargetId: !Ref LoadBalancingCollectorScalingTarget
+      TargetTrackingScalingPolicyConfiguration:
+        PredefinedMetricSpecification:
+          PredefinedMetricType: "ECSServiceAverageMemoryUtilization"
+        TargetValue: !Ref LoadBalancingCollectorScalingTargetPercentage
+        # see: https://docs.aws.amazon.com/autoscaling/application/userguide/target-tracking-scaling-policy-overview.html#target-tracking-cooldown
+        # in seconds, defaulting to both being 5 minutes of cooldown (this should be tuned after many load generation verification runs)
+        ScaleInCooldown: 300
+        ScaleOutCooldown: 300

stack/cloudformation/publicloadbalancer.yaml

@@ -6,7 +6,7 @@ Parameters:
     Type: AWS::EC2::VPC::Id
     Description: The VPC ID where the load balancer will be deployed
     Default: vpc-2465842045826
-  SubnetIds:
+  PublicSubnetIds:
     Type: List<AWS::EC2::Subnet::Id>
     Description: The Subnets (multiple for HA) configured to allow internet access
     Default: subnet-27465297465,subnet-265924692
@@ -20,6 +20,8 @@ Parameters:
     Default: 4318
 
 Resources:
+  # Security groups act as a virtual firewall for your ELB to control inbound and outbound traffic
+  # In this case, the security group allows inbound traffic on TCP port 80 from anywhere (0.0.0.0/0) (which is necessary for a public-facing load balancer)
   PublicLoadBalancerSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
@@ -36,11 +38,14 @@ Resources:
         - Key: Name
           Value: public-load-balancer-security-group
 
+  # The Scheme is set to internet-facing to make it publicly accessible
+  # It's associated with subnets (that should be in different Availability Zones for high availability)
+  # These subnets must be connected to the internet (typically through an Internet Gateway in the VPC)
   PublicLoadBalancer:
     Type: AWS::ElasticLoadBalancingV2::LoadBalancer
     Properties:
       Name: public-load-balancer
-      Subnets: !Ref SubnetIds
+      Subnets: !Ref PublicSubnetIds
       SecurityGroups:
         - Ref: PublicLoadBalancerSecurityGroup
       Scheme: internet-facing
@@ -50,6 +55,9 @@ Resources:
         - Key: Name
           Value: public-load-balancer
 
+  # A listener checks for connection requests to the load balancer based on the protocol and port
+  # Here, it listens on port 80 for HTTP traffic
+  # The default action is configured to return a 404 (Not Found) fixed response, effectively serving as a fallback if no rules match the incoming request
   PublicLoadBalancerListener:
     Type: AWS::ElasticLoadBalancingV2::Listener
     Properties:
@@ -63,6 +71,9 @@ Resources:
             ContentType: "text/plain"
             MessageBody: "Not Found"
 
+  # Listener Rules define how the incoming traffic is routed
+  # In this case, there's a rule to forward requests with a path starting with "/v1/" to a specific target group
+  # This is where you define the criteria for forwarding requests to specific targets (e.g., EC2 instances, ECS services)
   ListenerRuleForV1Path:
     Type: AWS::ElasticLoadBalancingV2::ListenerRule
     Properties:
@@ -75,6 +86,8 @@ Resources:
         - Type: forward
           TargetGroupArn: !Ref HttpTargetGroup
 
+  # Target groups are used to route requests to one or more registered targets
+  # This template defines a target group with targets (e.g., EC2 instances, ECS services) that listen on port 4318 using HTTP protocol (within the specified VPC)
   HttpTargetGroup:
     Type: AWS::ElasticLoadBalancingV2::TargetGroup
     Properties:
@@ -102,3 +115,9 @@ Outputs:
   LoadBalancerDNSName:
     Description: DNS name of the public load balancer
     Value: !GetAtt PublicLoadBalancer.DNSName
+  PublicLoadBalancerSecurityGroupId:
+    Description: SecurityGroups Id associated to the public load balancer
+    Value: !GetAtt PublicLoadBalancerSecurityGroup.GroupId
+  HttpTargetGroupArn:
+    Description: The ARN of the HTTP target group
+    Value: !GetAtt HttpTargetGroup.TargetGroupArn