From 49a1acfdb4b34c4394447a6241979564504db2b0 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 21 Jul 2022 16:47:17 +0200 Subject: [PATCH 01/29] Modified the CF to configure Jenkins using Docker --- .../amazon-ec2-spot-cicd-workshop.yaml | 46 +++++++++---------- .../container/Dockerfile | 9 ++++ .../container/casc.yaml | 24 ++++++++++ .../container/docker-compose.yml | 16 +++++++ .../container/plugins.txt | 22 +++++++++ 5 files changed, 92 insertions(+), 25 deletions(-) create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index 0b738405..2253d7a9 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -811,32 +811,28 @@ Resources: #!/bin/bash # Install all pending updates to the system yum -y update - # Configure YUM to be able to access official Jenkins RPM packages - wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo - # Import the Jenkins repository public key - rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key - # Configure YUM to be able to access contributed Maven RPM packages - wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo - # Update the release version in the Maven repository configuration for this mainline release of Amazon Linux - sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo - # Install the Java 8 SDK, Git, Jenkins and Maven - yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel git apache-maven - yum -y install jenkins-2.138.4-1.1 --nogpgcheck - # Set the default version of java to run out of the Java 8 SDK path (required by Jenkins) - update-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java - update-alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac - # Restore the pre-staged copy of JENKINS_HOME used by this workshop - mv /var/lib/jenkins /var/lib/jenkins_orig - mkdir /var/lib/jenkins - chown jenkins:jenkins /var/lib/jenkins - wget -q -O /var/lib/jenkins/jenkins_home.tar.gz https://s3-us-west-2.amazonaws.com/amazon-ec2-spot-cicd-workshop/jenkins_home.tar.gz - tar -zxf /var/lib/jenkins/jenkins_home.tar.gz -C /var/lib/jenkins/ - # Reset the password for the spotcicdworkshop user in Jenkins to the password defined as the JenkinsAdminPassword - echo -n '${JenkinsAdminPassword}{spot}' | sha256sum | awk '{print $1;}' | xargs -I {} sed -i 's/jbcrypt:$2a$10$21qSY20aYvtAFeWA0yL8AezRox5bExGclhtaFatOokyYWe7CxbRfm/spot:{}/' /var/lib/jenkins/users/spotcicdworkshop/config.xml + # Install Docker + yum install -y docker + usermod -a -G docker ec2-user + newgrp docker + systemctl enable docker.service + systemctl start docker.service + # Install Docker Compose + yum install python3-pip -y + pip3 install docker-compose + # Download files from GitHub + mkdir -p /usr/share/jenkins + cd /usr/share/jenkins + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt + # Reset the password defined as the JenkinsAdminPassword + sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml # Configure the Jenkins Location - curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/203.0.113.0/{}/' /var/lib/jenkins/jenkins.model.JenkinsLocationConfiguration.xml - # Start the Jenkins service - service jenkins start + curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml + # Start Jenkins + docker-compose up --build -d JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 Type: AWS::ElasticLoadBalancingV2::LoadBalancer diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile new file mode 100644 index 00000000..f107833a --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile @@ -0,0 +1,9 @@ +FROM jenkins/jenkins:2.346.2-lts-jdk11 + +ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false +ENV CASC_JENKINS_CONFIG /usr/share/jenkins/ref/casc.yaml + +COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt +RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt + +COPY casc.yaml /usr/share/jenkins/ref/casc.yaml \ No newline at end of file diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml b/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml new file mode 100644 index 00000000..ae36ae02 --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml @@ -0,0 +1,24 @@ +jenkins: + securityRealm: + local: + allowsSignup: false + users: + - id: ${JENKINS_ADMIN_ID} + password: ${JENKINS_ADMIN_PASSWORD} + authorizationStrategy: + globalMatrix: + permissions: + - "USER:Overall/Administer:admin" + - "GROUP:Overall/Read:authenticated" + remotingSecurity: + enabled: true + +security: + queueItemAuthenticator: + authenticators: + - global: + strategy: triggeringUsersAuthorizationStrategy + +unclassified: + location: + url: http://localhost:8080/ diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml new file mode 100644 index 00000000..4f2bd787 --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.2' + +services: + jenkins: + image: flexco-cicd/jenkins + privileged: true + user: root + build: . + environment: + JENKINS_ADMIN_ID: admin + JENKINS_ADMIN_PASSWORD: JenkinsAdminPassword + volumes: + - ${HOME}/jenkins_home:/var/jenkins_home + ports: + - 8080:8080 + restart: always diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt new file mode 100644 index 00000000..3368c790 --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -0,0 +1,22 @@ +ant:latest +antisamy-markup-formatter:latest +build-timeout:latest +cloudbees-folder:latest +configuration-as-code:latest +credentials-binding:latest +email-ext:latest +git:latest +github-branch-source:latest +gradle:latest +ldap:latest +mailer:latest +matrix-auth:latest +pam-auth:latest +pipeline-github-lib:latest +pipeline-stage-view:latest +ssh-slaves:latest +timestamper:latest +workflow-aggregator:latest +ws-cleanup:latest +authorize-project:latest +ec2-fleet:2.5.1 \ No newline at end of file From 66830b52232727ddc3f5fc0dce8d9e7d3a8a0623 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Fri, 22 Jul 2022 17:55:00 +0200 Subject: [PATCH 02/29] Fixed problems with the CF for the CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop.yaml | 59 +++++++++++++------ .../container/plugins.txt | 42 ++++++------- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index 2253d7a9..bd39a1f4 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -21,6 +21,15 @@ Parameters: MinLength: 8 NoEcho: true + AmazonLinux2LatestAmiId: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 + + ECSAMI: + Description: AMI ID + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + Resources: IAMRoleAMILookupLambdaExecution: # IAM Role that allows the AMILookupLambdaFunction to look up the latest Amazon Linux AMI and write logs to CloudWatch Logs Type: AWS::IAM::Role @@ -57,7 +66,6 @@ Resources: DependsOn: - DynamoDBTestEnvironmentTable - IAMRoleTestEnvironmentCloudFormation - # DependedOn: TestEnvironmentLambdaFuntion Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -98,7 +106,6 @@ Resources: IAMRoleTestEnvironmentCloudFormation: Type: AWS::IAM::Role DependsOn: AMILookupLambdaFunction - #DependedOn: None Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -164,7 +171,6 @@ Resources: DependsOn: - DeploymentArtifactsS3Bucket - TestEnvironmentLambdaFunction - # DependedOn: InstanceProfileJenkins Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -188,11 +194,32 @@ Resources: - Effect: Allow Action: lambda:invokeFunction Resource: !GetAtt TestEnvironmentLambdaFunction.Arn + - Effect: Allow + Action: + - ec2:DescribeSpotFleetInstances + - ec2:ModifySpotFleetRequest + - ec2:CreateTags + - ec2:DescribeRegions + - ec2:DescribeInstances + - ec2:TerminateInstances + - ec2:DescribeInstanceStatus + - ec2:DescribeSpotFleetRequests + Resource: "*" + - Effect: Allow + Action: + - autoscaling:DescribeAutoScalingGroups + - autoscaling:UpdateAutoScalingGroup + Resource: "*" + - Effect: Allow + Action: + - iam:ListInstanceProfiles + - iam:ListRoles + - iam:PassRole + Resource: "*" InstanceProfileJenkins: Type: AWS::IAM::InstanceProfile DependsOn: IAMRoleJenkins - # DependedOn: JenkinsOnDemandEC2Instance Properties: Path: "/" Roles: @@ -200,8 +227,6 @@ Resources: IAMUserJenkins: Type: AWS::IAM::User - # DependsOn: None - # DependedOn: None Properties: Policies: - PolicyName: SpotAgentPolicy @@ -259,8 +284,6 @@ Resources: IAMRoleECS: Type: AWS::IAM::Role - # DependsOn: None - # DependedOn: InstanceProfileECS Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -788,7 +811,6 @@ Resources: - InstanceProfileJenkins - SecurityGroupJenkins - SubnetPublicA - # DependedOn: JenkinsMasterALBTargetGroupEC2 Properties: BlockDeviceMappings: - DeviceName: "/dev/xvda" @@ -797,7 +819,7 @@ Resources: VolumeSize: 8 VolumeType: gp2 IamInstanceProfile: !Ref InstanceProfileJenkins - ImageId: !GetAtt EC2AMILookupCustomResource.Id + ImageId: !Ref AmazonLinux2LatestAmiId InstanceType: t3.medium KeyName: !Ref KeyPair SecurityGroupIds: @@ -812,14 +834,13 @@ Resources: # Install all pending updates to the system yum -y update # Install Docker - yum install -y docker + amazon-linux-extras install docker + chkconfig docker on + service docker start usermod -a -G docker ec2-user - newgrp docker - systemctl enable docker.service - systemctl start docker.service # Install Docker Compose - yum install python3-pip -y - pip3 install docker-compose + curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose # Download files from GitHub mkdir -p /usr/share/jenkins cd /usr/share/jenkins @@ -965,7 +986,7 @@ Resources: IamInstanceProfile: #Arn: !GetAtt InstanceProfileJenkins.Arn Name: !Ref InstanceProfileJenkins - ImageId: !GetAtt EC2AMILookupCustomResource.Id + ImageId: !Ref AmazonLinux2LatestAmiId InstanceType: t3.medium KeyName: !Ref KeyPair SecurityGroupIds: @@ -1018,7 +1039,7 @@ Resources: IamInstanceProfile: #Arn: !GetAtt InstanceProfileJenkins.Arn Name: !Ref InstanceProfileJenkins - ImageId: !GetAtt EC2AMILookupCustomResource.Id + ImageId: !Ref AmazonLinux2LatestAmiId InstanceType: t3.small KeyName: !Ref KeyPair SecurityGroupIds: @@ -1128,7 +1149,7 @@ Resources: VolumeType: gp2 IamInstanceProfile: Name: !Ref InstanceProfileECS - ImageId: !GetAtt ECSAMILookupCustomResource.Id + ImageId: !Ref ECSAMI InstanceType: t3.medium KeyName: !Ref KeyPair SecurityGroupIds: diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt index 3368c790..cb383d61 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -1,22 +1,22 @@ -ant:latest -antisamy-markup-formatter:latest -build-timeout:latest -cloudbees-folder:latest -configuration-as-code:latest -credentials-binding:latest -email-ext:latest -git:latest -github-branch-source:latest -gradle:latest -ldap:latest -mailer:latest -matrix-auth:latest -pam-auth:latest -pipeline-github-lib:latest -pipeline-stage-view:latest -ssh-slaves:latest -timestamper:latest -workflow-aggregator:latest -ws-cleanup:latest -authorize-project:latest +ant:475.vf34069fef73c +antisamy-markup-formatter:2.7 +build-timeout:1.21 +cloudbees-folder:6.729.v2b_9d1a_74d673 +configuration-as-code:1512.vb_79d418d5fc8 +credentials-binding:523.vd859a_4b_122e6 +email-ext:2.90 +git:4.11.3 +github-branch-source:1677.v731f745ea_0cf +gradle:1.39.4 +ldap:2.10 +mailer:438.v02c7f0a_12fa_4 +matrix-auth:3.1.5 +pam-auth:1.8 +pipeline-github-lib:38.v445716ea_edda_ +pipeline-stage-view:2.24 +ssh-slaves:1.834.v622da_57f702c +timestamper:1.18 +workflow-aggregator:590.v6a_d052e5a_a_b_5 +ws-cleanup:0.42 +authorize-project:1.4.0 ec2-fleet:2.5.1 \ No newline at end of file From 9ba5f2a8ccebb9148391d87dabfb316617f22ca7 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 27 Jul 2022 18:07:15 +0200 Subject: [PATCH 03/29] Updated the instructions for theh CI/CD Workshop lab 1 --- content/amazon-ec2-spot-cicd-workshop/lab1.md | 123 +++++++++++------- .../amazon-ec2-spot-cicd-workshop.yaml | 11 +- .../container/docker-compose.yml | 4 +- .../jobs/seed.tar.gz | Bin 0 -> 17678 bytes 4 files changed, 89 insertions(+), 49 deletions(-) create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz diff --git a/content/amazon-ec2-spot-cicd-workshop/lab1.md b/content/amazon-ec2-spot-cicd-workshop/lab1.md index 13f88e32..8b0df14e 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab1.md +++ b/content/amazon-ec2-spot-cicd-workshop/lab1.md @@ -2,36 +2,73 @@ title = "Lab 1: Reduce the cost of builds using Amazon EC2 Spot Fleet" weight = 20 +++ -The Jenkins server that was launched by the CloudFormation template has a number of build projects preconfigured. However by default, all builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: +By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: * When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and * The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. -## PROVISION A SPOT FLEET FOR YOUR BUILD AGENTS -Before configuring the EC2 Fleet Jenkins Plugin, create a Spot Fleet that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); optimize for lowest cost; determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. +## Setting Up environment variables -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Click on the **Request Spot Instances** button; -3. At the first screen of the Spot instance launch wizard: - 1. Under the Tell us your application or task need heading, ensure that the **Load balancing workloads** option selected (Note: do NOT select the Flexible workloads option as this will deploy a Spot Fleet with weightings applied to some of the EC2 instance types which will adversely impact how the plugin scales out); - 2. In the Configure your instances section, select the **JenkinsBuildAgentLaunchTemplate** template from the Launch template dropdown (if this option is not present in the dropdown, please verify that the CloudFormation template that you launched during the Workshop Preparation has deployed successfully). Change the Network to be the **Spot CICD Workshop VPC**. After making this selection, enable the check boxes for all three Availability Zones and then select the **Amazon EC2 Spot CICD Workshop Public Subnet** associated with each availability zone as the subnet to launch instances in; - 3. At the Tell us how much capacity you need section, keep the Total target capacity at **1** instance and the Optional On-Demand portion set to **0**, and then tick the **Maintain target capacity** checkbox. Once selected, leave the Interruption behavior set to **Terminate**; - 4. While the new Spot Instances wizard makes some good recommendations to you on how best to add diversity to the fleet that you're creating, the recommended instance types provide more resources than we strictly need in this workshop - so to the right of the Fleet request settings heading, clear the tick from **Apply recommendations** checkbox. Click on each of the **Remove** links associated with the all of the instance types initially defined to remove them from the fleet configuration. Then click on the **Select instance types** button and add the **t2.medium**, **t2.large**, **t3.medium** and **t3.large** instance types to the fleet definition (Hint: you may need to adjust the **vCPUs** and **Memory (GiB)** filters to reveal all of these instance types). Once the checkboxes for the required instance types have been ticked, click on the **Select** button. Once you have the four desired instance types listed in the fleet request, select the **Lowest Price** Fleet allocation strategy (since we’re interested in keeping cost to an absolute minimum for this use case); - 5. Review the Your fleet request as a glance section - it should indicate that your Fleet strength is Strong as a result of being able to draw instances from 12 instance pools, and your Estimated price should indicate that you're expecting to make a 70% saving compared to the cost of equivalent on-demand resources; - 6. Lastly, click on the **Launch** button. -4. Make a note of the Request ID of the Spot Fleet that you’ve just created. +```bash +export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); +export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); +export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); +export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); +export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); +export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +``` -## CREATE A SECRET KEY AND ACCESS KEY FOR THE PLUGIN -The CloudFormation template that you deployed in Lab 1 created an IAM User called **SpotCICDWorkshopJenkins**. Jenkins will use this IAM User to control the spot fleet used for your build slaves. Generate a secret key and access key for this user. +## Provision an Auto Scaling Group for your build agents +Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **IAM** console and click on the **Users** option from the left frame (or [click here](https://console.aws.amazon.com/iam/home?region=eu-west-1#/users)); -2. Click on the **SpotCICDWorkshopJenkins** user; -3. Click on the **Security credentials** tab, then click on the **Create access key** button – Make a note of the Access key ID and Secret access key, then click the **Close** button. -{{% /expand%}} +First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: + +```bash +cat < ~/asg-policy.json +{ + "LaunchTemplate":{ + "LaunchTemplateSpecification":{ + "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", + "Version":"1" + }, + "Overrides":[ + { + "InstanceType":"t2.large" + }, + { + "InstanceType":"t3.large" + }, + { + "InstanceType":"m4.large" + }, + { + "InstanceType":"m5.large" + }, + { + "InstanceType":"c5.large" + }, + { + "InstanceType":"c4.large" + } + ] + }, + "InstancesDistribution":{ + "OnDemandBaseCapacity": 0, + "OnDemandPercentageAboveBaseCapacity": 0, + "SpotAllocationStrategy":"capacity-optimized" + } +} +EoF +``` + +Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. + +```bash +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +``` -## SIGN IN TO YOUR JENKINS SERVER +## Sign-in to Jenkins The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **spotcicdworkshop** as the Username and the password that you supplied to the CloudFormation template as the password. {{%expand "Click to reveal detailed instructions" %}} 1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); @@ -42,23 +79,17 @@ The CloudFormation template deployed during the Workshop Preparation stage deplo 2. Enter in the password that you supplied to the CloudFormation template as the Password. {{% /expand%}} -## CONFIGURE THE EC2 FLEET JENKINS PLUGIN -The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll first need to supply the IAM Access Key ID and Secret Key that you created so that the plugin can find your Spot Fleet request. You'll then need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). +## Configure the EC2 Fleet Jenkins plugin +The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. {{%expand "Click to reveal detailed instructions" %}} -1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Configure System** link; -2. Scroll all the way down to the bottom of the page and under the **Cloud** section, click on the **Add a new cloud button**, followed by the **Amazon SpotFleet** option; -3. Under the Spot Fleet Configuration section, click on the **Add** button next to the AWS Credentials [sic] dropdown, then click on the **Jenkins** option. This will pop up a new **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: - 1. Change the Kind to **AWS Credentials**; - 2. Change the Scope to **System (Jenkins and nodes only)** – you don’t want your builds to have access to these credentials! - 3. At the ID field, enter **SpotCICDWorkshopJenkins**; - 4. At the Access Key ID and Secret Access Key fields, enter in the information that you gathered earlier; - 5. Click on the **Add** button; -4. Once the sub-form disappears, select your Access Key ID from the AWS Credentials dropdown - the plugin will then issue a request to the AWS APIs and populate the list of regions; -5. Select **eu-west-1** from the Region dropdown - the plugin will now attempt to obtain a list of Spot Fleet requests made in the selected region; -6. Select the Request Id of the Spot Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; +1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; +2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; +3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; +4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; +6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; 7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: 1. Change the Kind to **SSH Username with private key**; 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; @@ -67,32 +98,32 @@ When configuring the plugin, think about how you could force build processes to 5. Click on the **Add** button. 8. Select the ec2-user option from the Credentials dropdown; 9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; -10. Mark the **Connect Private** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); -11. Change the Label field to be **spot-agents** - you'll shortly reconfigure your build job to run on slave instances featuring this label; -12. Set the Max Idle Minutes Before Scaledown to **5**. As AWS launched per-second billing in 2017, there's no need to keep a build agent running for too much longer than it's required; +10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); +11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; +12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; 13. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); 14. Finally, click on the **Save** button. -Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your Spot fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. +Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. {{% /expand%}} -## RECONFIGURE YOUR BUILD JOBS TO USE THE NEW SPOT INSTANCE(S) -As alluded to in the previous section, you'll need to reconfigure your build jobs so that they are executed on the build agents running in your Spot fleet (again, use the **spot-agents** label). In addition, configure each job to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. +## Configure a build job to use the new Spot instance(s) +As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. {{%expand "Click to reveal detailed instructions" %}} -1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins deployment: +1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); 3. Click on the **Save** button towards the bottom of the screen. {{% /expand%}} -## TEST SPOT BUILDS AND SCALE-OUT -Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your Spot Fleet scales out when there are build jobs queued for more than a few minutes. +## Test Spot Builds and Scale-out +Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. -1. Back at the Jenkins home page, first click on the **ENABLE AUTO REFRESH** link that's located towards to top-right corner of the screen - this will enable a full refresh of the Jenkins user interface every 10 seconds allowing you to see regular updates to the status of each build. Next, click on the **Schedule a Build** icon (which looks like a play symbol superimposed over a clock) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; +1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; 2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; 3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; 4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). -## PROCEED TO LAB 2 -Once you've verified that builds are succeeding and that your Spot Fleet is capable of scaling out to handle queued build jobs, you may proceed with [Lab 2](/amazon-ec2-spot-cicd-workshop/lab2.html). +## Proceed to lab 2 +Once you've verified that builds are succeeding and that your ASG is capable of scaling out to handle queued build jobs, you may proceed with [Lab 2](/amazon-ec2-spot-cicd-workshop/lab2.html). diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index bd39a1f4..bf7b9e02 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -408,6 +408,8 @@ Resources: Tags: - Key: Name Value: Amazon EC2 Spot CICD Workshop Private Subnet A + - Key: Type + Value: Private SubnetPrivateB: # ... and the second of three subnets... Type: AWS::EC2::Subnet @@ -421,6 +423,8 @@ Resources: Tags: - Key: Name Value: Amazon EC2 Spot CICD Workshop Private Subnet B + - Key: Type + Value: Private SubnetPrivateC: # ... and the third of three subnets defined within the VPC Type: AWS::EC2::Subnet @@ -434,6 +438,8 @@ Resources: Tags: - Key: Name Value: Amazon EC2 Spot CICD Workshop Private Subnet C + - Key: Type + Value: Private InternetGateway: # Create an Internet Gateway in order to allow EC2 instances to be accessible via the Internet Type: AWS::EC2::InternetGateway @@ -816,7 +822,7 @@ Resources: - DeviceName: "/dev/xvda" Ebs: DeleteOnTermination: "true" - VolumeSize: 8 + VolumeSize: 50 VolumeType: gp2 IamInstanceProfile: !Ref InstanceProfileJenkins ImageId: !Ref AmazonLinux2LatestAmiId @@ -848,6 +854,9 @@ Resources: wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz + tar -xvf seed.tar.gz + cp -r jobs/* /jenkins_home/jobs/ # Reset the password defined as the JenkinsAdminPassword sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml # Configure the Jenkins Location diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml index 4f2bd787..1417dcd9 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml @@ -7,10 +7,10 @@ services: user: root build: . environment: - JENKINS_ADMIN_ID: admin + JENKINS_ADMIN_ID: spotcicdworkshop JENKINS_ADMIN_PASSWORD: JenkinsAdminPassword volumes: - - ${HOME}/jenkins_home:/var/jenkins_home + - /jenkins_home:/var/jenkins_home ports: - 8080:8080 restart: always diff --git a/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz b/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..784b2bdf9feb3e40e57eb35e7b0c18efb7cf3122 GIT binary patch literal 17678 zcmb9CbyQUEA3ctWNJ&dd3J6FD(ulwyih#68i7+4{AtBw264IbjQj&sz2uSD9-QC?i zwR(iD|Z&>-!>yMsynxCB?o2x+MKBUptt+0+KNt!g0||H+p?mLw$rj% z7hCrURIP>T%MId5EcU|Jw1T9{#bfdCr15ZGEBsE$qfV^89Ew^NTbUHej3;A|@>MNJ?00PXNrj+jLHp(v8N#@?3giKq(r!>yR!;n+h|Q zIX7`0Mwa1=#Zi5 zcY%au({}Xs4|+dP74o2&d8o{J!?L@y?6$4n?-wJjEbJXkvdK#A0M^RF5(Nz-!ds~h zFYQsqqZ43V^k|YeQ<*a==7DtyhF1kC$QG_eyL?N z+nlgm$a_W+OXx$|zEHdP_7|bH?mEb^flBVvkCGACqkUFES`|z284Jua!l)E+`-ys^ zlZ1ZWI)c#Fxwo=Lp<|zUlR|-YA$rmVHRwrw=S{e|sr{p=a3xVT_iiDbfjdX*7P0Eb zZGRi2`<}6I9>;6`evsuAv2nkC#yu=l!H{L++D%1vahGe{ZyqQzE0~K;iCI-zFAN9S z9_-NFBfAxrSWsyK24lEh86b6f`GI?xpn3B5P66Yq!`2gOwwXJ~Y?v zh&M9jkx;b1EMY*3IqfGPWdCVgR1MZ_pWX30d;9J&Z?N;T zz;fs_=!c4FO?YHY?k2?b3Z5YyxP~Q;&s4{^nEl&&Hml_~4Er|rliHRSTw}@hF zS`95aqL}RWX*KhrPTZDf4QDmuqo^ocSRXdqoOV4}4r>0xRYYKzI`-nTE7PtI^hFI` zZO>ab2vb2VeA0fm_#w6w%HIx(YX39{+~S1u->A`(1AJD8g~!KbtY_+h{7487jT#>O z1ma4;+yUYUID~wAC2<0EIR4lHw;F;*L4rCSOMd~vh5%zMQ01`)gcd~LP`-O0q5WU) zviFi4VYE13$^nQ_CI9P?vZFhhLKB(W$YLgJ>i|O+B#8K!Jzq}@*yf?Mjs*D8%2vZ> z0Q==?jh+d5xAJU~5qAo~FC74*5`gH<+vau(G443wuQ%H`)pTob&&RbeAVU2JI3a4f zlo#gsL`lv?WcLHOSMq(?lj1po-z@hgB2{iQi`r2LklMCGk4iuJ`j@bRz6&3>CvYvP z6`?&9`4$6SSxU=`R8E;SKK+bQ)x+LDBgujjd%eiF9Q;TH&NqBZFGvPs2+etiy`Gd5 zsIc*FcPuuG>D@F9vt*pP~|Aj_VE@! z3znz8+v;!Y*sP4$z}8iQ^zlTEN?eKK!!?+wR-$-C__c1Uzhu{YuuNK-u@al`C#c}; zRSBYC@g=FF;Nv#}`A)s|aii4)Cw}~PWuXjhRXbYbE2@^hVFY>zD-C9SFAn-KM+5t1 z0r#Z{_>|UqU9cxnnBSA29gTK~eC1#-1$2W%k2EeJ>g+^Vj_$}n%jd;7%~7kRFSWn) zcpe$uBda*DuP3L0h;Y|)9+Kt>17zZH5S@zGf?}iNbY&biD&~|v-0ymPRh=w8ob!>> zcNPBFBd^Cjj}2+#26CKg3rJ6hC*JAk<+d=~jU`57N=Nn2m`g_)AEn>*{kqfmdfS&6F-v89z|#VQ$;B;}O5+8mgC6JqXY9k>ZzXDis1 ziQAXYqn`wb10GOKVQDx$FVGhnci$JwsZ!f4wwYi^vPkx*%M$F zv_TCJc*m889_UiI{^DWxDwqqX>Hct!Cl#0~0fgV&K*WrVBm+BNiEIZU>P)QifL7=o z;HxncT#Kj(G7pb=5`tC^5Vsyck;qF@eWCZe0Wd(33|aT@W@-8^6I%GDqyB=YZB1+!n81oq*+z74{J^ z!A)3DDD@i1Sq&f9&NwTk0Ys8Yu*#vf8=ywepjEPk-{Q5r@eK@!UUwQ`-UEW~JjQZW zl&?aO>z9Rp1>#^aCDin5AWmF0a|L05>nQU5D8O9w0&Owcih=~Gq#(h5>3F~VFqFMO z2>`yTx;rWGLB?#dGo*=6GO}w15HXqk}(c2+;mNM zvw3v^6dV9a(X9RyYaqqTvwP72dAKd1!ZySRL_Ws~yHcb*3;ge9bO^26XSKWM73#?= z&{Uvj2FF}M80f9cf==+{0l1!Jug*~~3Sv?R8Q;J;iBnxkpBx_}&g4Rmp# zY=%9iN)St(e2k@B13ojXL7WS1(YrGkH5Ae@_#A8m1fDH^HgH2LmjIno=+2B%7z#p0 zQ4eTEUcMR$So)vikCkVOf8okou7tSdtM{t63Z5*2cEFXV#J$(g)1xndYqP*-0~0jV zQ~s|rirbnnH~6;UH8!K6n`Ey(VYCcCnQz+N8BEu@ZOuzgd!z2;rzl@J0SaPEdOnYg z8JL5XLx8e+teRC>2EZVKgKf@)$12$B06YT)jM#uFju0$3h}aZ}HC~K@q*Y)qmoqU| zrI`Q&$4C<-2o8DK3SnW|0}>gp5U~dJUjI~k#EyRss9im<#BsXjD?kRuox(!YyN(uH0DxZAX3JpqIjb3OsYFyL1YgiMBX z0=UHK3pF)tp#j@3e89>25~r8Q`L`49qQH*NXCTFwGywHFpz{gXvXfZ=#G-*UmIBAmY~k` zewQ$}Upo|F+=D}|SRR2K~H?vur z%8LMg6y#mm5t1^hIS6)%quBq(`9FAB3rdn6<2Vbg<)Fs^)Vg0At;8BHV9TaiWPSs; zGItPKRcv==CLjVwxR;m?+1P$^1AXi0e~%px5h0>ih};b`7}WiS9YP!LW*$vS7$ zV9sMeOYtQGAZ(5J8N5FE(&hsrz5388X|n(0QDJN3S-L2uAcJWKB;#@|dwd0>MF((+ zQ|tjzjLo}{7lohUlS%mBJK)96m_akH5^$)lIx2qiSJ)W=&yqaTRvSv5T|cC@PqVIE&wV?ELJK65BI>Ar+0rJUhMCsV1YJR16iWE z)Fp9MQ(aPxEpAgZ2)Yc?Lk(s%|G)F5t0$V|68wddXVBV>PcxIicVgC0|DVLUbO+ZH zLHr0va45W<%A`0@p9m1XaSo_ooz5Z*7_s>efK+n;hSQ1SCwm0-a}pPi4OgrGv*1;BcsNi9_7$SN=aZJVIz)zM8oL z9;}-^@RJWqO&?)egRy#WNml__F!?(0lZ?k2AlQdudH0|tVp4{8)}Hhh13&_JaSL_E zo;Ho6e~gv{4vB>SL*gz9RxE}pz7)pM5W)!ZOVAx5)8~->=-obSq2wVXVFHj=p8}XU zM%o~?53c2t0;S6*khV*S`}009&PzG=37B&IKhrS@aD4;l@nTm?F2NXu&cnz5B6Gwp z$@N(#5*8LSc?KQ z^k^5<1tAyig5V?!I!^#490dc&f&3W=59tJi_$&Px)Ek7ajlxanr%R~xt1|T>kl?fv zh5cQKx=acoB8K=9%;FOOu_qR4i=k|emimp`-4O%rm?AHu-j&@Su{{LZn{ey}x_HlUw}}mOHShi^0Tm@WgvLz5YmhVGnnt zM-LE3HGjGQa7~39&dG3K5)H)xU!8*HAKWg45s-Lx2t_Pzbn#rV9up$Q3@CQSW{)P7 z0FGBZkHcCRj-LMtQ~|^|`K2710T|#TAOt9+S41lifdz}IhTfw}hCDtIw#F(sp9nW3 z?||A+5WOXcB+)tWI1;B9^UpAF-UXmg77O#g_CNhNaj}~22^5F#gQ6~ZOc6IaRe<7~ zo#}xJI7f8#-V%V&BcPxQBDe)8{5Lq+OR-=&L<}B={gSh|hJZpeoXhS8P8a@1jqG?> z@H%YpZOPIJ=-=?dP1iG^Utk^k`&Zme{s7xUK>REm=Q6K<#6g!;h(yMu|K}tD~9>w6D*e^j)#kw;4vR?HSB|M zGfFwK{1-sI0qA97Bbu-fl3SO|h65KNPT^?-+09r;*p(u+z{DtAU6a-ta0cScIR~7V z2w;3~Kt1<9v4b~vGi;^-f%B*^R-br)s2)(esxux9Wx9K5EnfhWf8kax-;c1uSG3Nz z3Vk!SyMR@*_;1j5ylusT!?ed&i(OE{MTbBFE2JLVi<`-p2l@)9w{VpBY7w{}19|#? zY3{$}IY5cO3J}MMHnsn0!e|^R76GCUS@wEjG+K8-f`$M$j;K_NlHp(jFNI#DqZgWM z${M$d>+9~K0@&rYmz*{WA;eAF>>aq)B;lP`6A-=5^mctD?*UD-@%^KG4o<<)XP#z&$GTc+R7C>( z260KMaY9#|F4@l%j^97li9QSKX$e*AEQ0+MevME?etJN`bg$XH;wjJ1^G;*iJx{&w zwXP(d9jx5+h`MfHf|YQaYF{&>jir;Izt^PL1zpHj5oSa~;SSGQ{e9z`e~KLiDLQ@k z(=M@lwf$$3yDIT64$c#huA;*Z=)vA%xgSvV1~vG$bQuMGs)BPhZP#*z&_tMj zIQu{HQ6vKpzSRB><6V&2Qb-|=x-VC*i2P$d9Z>obD=fYRJi7wb%rV&SSfL26bsJC* z!ow^QU*fay9d=Kn<)Urua63LF()y*eE8 zGoz|x#sDe`oUA3*(ZG={KI5zV3&7hqT-L-OsIj7swXDDNDtdQ2?`Ld`@u@iiCCbprFwA;-#k&p_&*b6N-g)5Bosz-62T1G5jlot4jbzFyOS&3!hX6q%L zk9Pqwq8~^8VhbZ7LEVBUkN}w0$4fc;3I>ro`IvCFcoU|zxrBi{#yWal%838AFanCE zuog5eNMY)NuYl`Yf^17nRSO0r<9<`&KLuC>FaB}Wp22WsLqVDUIav;@+q)Zi9M~jr zp6ufjkX=2XJcX62FF%8v(1f=VB9AdQIPnff^)_D9@+92g?=I&H861do~ zRe)CCG|oByd`k`8KZezo4-jQ28|fE6AxTJ{CR7VCU)=B6*f%=cG#fPsg=h%4CFt6h z#jEV~<~_YfWX>A|f;zIBmKfUpsn^It&S=RoF~mRDiOvjta+kKkk~*08_-D`!(so(! zO-dBE|J0wXt~Oe!XVZ2)q{890PIOOnC7wQE2B`qGLYUo_nTW zs)S=!7&u?Bv4snM(0_@%%}PXJRA4Zjc%{}R%F>touj zDaE;fcF~>(9O=RW#vuwd*T+Xtnrp$TMez;KBGU}pB&JGUhv>osape&UtJ&pVfy-|kw!-wb1YWb;vasyB?cGT!L;c{si#M%oK7^|KO_@A-8fWvC7E%0OD%G3FHVuocL7{z!v7wQ5Df+KqJz&08Y#P zlc~66i%b5ZgX7#LnIzFMPA1~W-=-OICV72~7Qh8Ybbu=bE6yv0+Ufv>Ct}wsih+DI zbP2|*aA`MiY;BuHWD5o?M5wm_#VgVEY7$^NLI4G4Ai|Vqf&U~5b3aX3NL^h_>HAQ( zq4Gl@sXw(_bU7$)ruN(r`HxD3r&tP~iWUg%rNP8IhCXEZn^;u}`*o(3FVcW!zcK z70?3YL&*0o07L#9wrpz%h@fg#;HWA%f=0^%`~{L{;Gq29z|isu`wbGDDn2v8PMWj* z(!dzd0W6XzZ##uAHMk`fHm_J8*w)CQc$WLwtn~wXmuL!K6itDBQBMaFk@$mB z@zcwR*n%G)*@adka@Of%U+%T(_`{aNUbolZso>Dx21^*5E~8%DJ12R$06!*Mir8y2 z=ge1c*(Bg2GSAeOiS5`f5bEPHOi|KHDk!jGa2G-asayu@6ej;)gY|Fe{|nanPab}v zGPd4c8tE=^`4IV(Dx6i?z`w+h|4SJ6=yR!z8K<9O&k>??i<+;zX~!p0-~IoQsU!b? z&D5o@GIfn~z`eFOYiAd9c7ZOqmwbK*DcAuKrws&PaMtZINY`Tru0F!W$p|cEzz1tw zO8hVK*;xfA7D0*@06}}e#%>A_=D-EaJDW(IKRCL#r1h|eNCli1;ocS6E z!X+%WD^O&Oe_SChtKJj*_`d|`Gq96#MFyPWhQMW#gj1g<>3?w1BVs=vB5n7I+Zex7_aZ8p6pDPb`fIXsH`xqp71LvqiBNHVjdv9O#7!NM>g1IK| zl2i6GE{^l{iJWzX^*#2_64>1MEsp(mu2z>>?BO@p0K}fLgAPCVjhMexgr9my@iFh& zCyVdoUU&fOlKW(=EePsEhE&otNYaVs*!{!gnrdc8ok+@=TM`-Jrn7@T-|D=A?~6g1 z{VJVM%k-e{@03j&MWrXjE~b;*ql)HbzGEJar%X6mRt<~!?2|1tM&>UbZVjZQ@X4{- z8)kRX728Vf4Y8d_S$yuRXPfJbamg@-4sA2XVB)v^`2{vde=?r>?Ud2bG)8t4pUZgh z^H-;Fo*V5JN37_+(06_;QY*QwfsA5?uFqXOkBZj%T37taKl?7%TG`p#RY}d*rtxL- zzs8YxvEeD&DVyM$-BPhG*w%zmSuPpBr)T5ak)Bw`fqq%GjfY~MyAdFn+O{7iuFlx$ z8ZioC4ais8YGA86f~`K7=eT&EEnUOc(!m+CQp@|hA9Z4Ky$S24tV8)IW!(x8VnQCQ z@FBXMrS@4OH}g@BKbW_+R_gY(Pxj?jrOtha&GqWn`=-8!8Xj@Hw<(+ZD*RW9!RTq~ z6Pk3ud`(tW|HC_FsrJd;v;!=Yc^tFlW95*$f1(AvQZ%E=7*omq{?bXWNeH%KD3dSh z0_sZEc9$2nJ2s03K5K!HYE}1okL<9e(x+Zw-GzO-sf-s-&b@*keg&Us4q79O&;4zk z#Ac!OAIHB9@`>!ukcu)A0gj{pjoRw+yHpucpV*ciqaan*ZEvUmzNt+Vh}|U zs|P>)oHgV+1BPXlh>{fIDBeBltA5Q8!+UN#JG=NK$(!^~nF|3?eS$k{3F)5Dv(7q7 zf(v2U$A>)*{ohpYQu4}`AYp8^P{^Xuv4sAVVu|H&+Uw?K{P|9Df^)B7zl$pVaJyQ% zy9bprFI?Ze%GSIX(mVc&LiOqIeWxLpU-0)eOM`k<>x!(0?Xx?A7~Eu>Zg|$pGvVJ5 zZl;(OIxZ2p@6vE^R=x1(WgDepC0^MWO=_xiWjPv#Pw4m(8` z9kiJHQ*C6E%S?{n`n+;zN)pn1<0DlTwe5~ec17~otQ^6#HL^;xgl1+s+QF2CoM)lhVWA zR4Xk{=U0gZg-br5|Me|IEb9-FxmNF?`w+- zpJUCO5r@A$A?;$rU5uQ~a@hl^(Uyzz+<8zmv(h z+xd}44r>uEFfY>P1f@RMO1u8-t}{xaKYXSz@e`XHp8it4=-TTEe&@$aA%Q(256hwVL z{B~_Z7{3l_E;0?NY_VEh%GxTKzOEFCw4k`PuM(XW9|;kqsPi(CF42c4|^EwA!>&K)~E zC&F|eE6r?}_pDAy1YyYR>O;ICUSV&NfcpzeZ2WHx`9*U#yzPl7!mnrhZ1Ik^tVF`q zl<(x+(Ce(W+`41p`R}gt=A!`KkgED*x>NBOM?Kji) z6O@aC2V8!{H~5ndD#B|!yTuweG@e+bK4E)pFV}lmKbhV#)IQH;RiBo~aSWvr-BD6` z_S}0d>B-|5`i9wEkf!SZNY-Pxg?F&Jw2I+vUfyXp#k`TYyYpV+mX1Y~=(k4=-u!dJ zaqn$JKk_}~G@002%R+nAJq^qvw3Bw<#Wboxqm=s%QvJWO*A-Rw&P&X$3#qOMEGL9$ z&~fOd((U~zHC1sIE8fm++`P&9gWz|!@!9lUKoR?T%`E2B)cyyLIH|CEO2ygUV5W)H z+rOehchPsdd_U;lMg6@88wo!(h`D&hCjY*Lab;kB3UQh;X7#dcAj@YyKLPybbAQyW zw|}D^nHmBo-QTQwM;v!Q~x)W#m_aikKliLKVrdIca(rAO` zlXmG)?-#x*cjn{&P@y!IxOi{%cd87yHZaX~D9n{2dF~?+ohXEoI7WmS1RA8vwSRG? z5n5VkOkVS>>JKMva@hO(#A5!!PIb{)W>SB_{*A3p`Ye=5DJR4f>bzl_X=YX^p}3K7 zkfweN{&PHC|1Jz6F?%;bX{4MzQdoA=>b^)XvqP$&M%n`l#A8C~MYof-IeGmU7I%=* z-<;W{`q2r2&n(AsOx_osBT0Ff*kY3fDFg2jkGbEyti1NSt{r(F+vR{>pR%$*g#CGx z@3C{#CZBf|jg+aZg#U`av`gYXS|^a=b^QXefrvLR!VwQCBGahq&z$r&$8Mp>LaJzE z;5ZrncL;v5>_Xc1clvO6UWy`c5i%nq<+>{u4A7U@R%wvr0l{_--yAt>msq@?`d2*_ z89b0L_&~hd>02jAB9}v)>fX4CI6sz4yjk7zcInmKMkbkcDZ$%zjyWx1@)S+0<|1yC zW^bBjk?$RP7Kj)qmB=2;4wj=~Br@pbxLaJ7*T@n&)gORl!LW>d(Gv2QEoRCg<%i+M z?9~n7qC*eE!{$bnZs#pVWBRR|B!@DCWx_qya|992%WgVrzet-hNfHz$_O4+-J(Ii{ z%Eo-v7(b9xI_Lodu$MG`I<^ddbGp?jZT4o$H0`*4U1ZcL)H%-MdE~6IE5rOvHs|98-(aOGkW5a}oA&yOBjUakCawVl6D}2zyyP4&!6| z)?|`GQrWF=R~JgWFXJ=2Jljh}ruR zboNwUjy={M=jvuJMRzZ{CrnZ(rMhH9=00MF<{-5?Ekk$^i8D=yfLClLa|6yZ*1_u% zWzAu+7qUxai*JR((`Ogv>U^M#qtPd(dvb32KPo#dAHPGcT>`&%X zf8vWW$;T%p%&p`0lcn|vwW{47kAnEuN${>U=Y8jRv`A57)mWy}H!k)>dGMGtDotSK zyNBCCyAL(!ZOIQl`N6aal5SHusIKAgsnKz^u2mme44=iUhxNyX3+i5V!}TxMPXm9n zcwP{Sz%8Vo6gtHjo?;6s0zNc!GS#VMoM^pnQd6U;Z?rZM2= ztq*44l#f{FkqpaMa|B9#0;oy!sAWCi5% zA3QE*@j3`4P$KNo4t53FKgv^roCje_UmMhhjJ%(e>dRm`W=iu-^VnNI8K)<36Y$wL z*Pl;4tn?u}s-Dbk6&oWI?~W*c4`XTwu~+_f7*dU;)V3=TC`s(x-$URLchuh?@-;T` zEL!rnmm}SK@kfGOeQuKV{RfSU1j+3Xg|~`$jFegQDEh8R=G+8rE=p1!3{%d77pAF4 zw6E<-kZb32#J)jh@EIO>oF}0+g=h_>gM{#M?e6sP17hVTH*2`LzwVc^Zwp^k7My$G2@Vmrv= zAnqHqetl>9$!u|?`};*nlhdYer9lsL)6*>FRmmA)y9j|)0h$q0F#hk+6y>DDYbm|vF0d!#sK6B;?IfwW+oqr2B%p!`6 zSKiNg(@Afa*_2~Jvcnd0$nEf0$PGB3bTSm$B4dfOIyF67zK^uxtdfIvORTSR`zNr3 z5($9t{dXsw{>+y2^<1kYSz<5_-XxZF{L}HmI1{!bU2D36X#omB5cJYi&!bDDk2Cb%JPXNYUt-wZ{_w%cDVE-C9#-}g8upj^#jYIDY5FI zsfKomrQAi^$3u0LOFS0VCkc*2@dtc&YV#~cSLC9Of;`yKPS@2ub$XjV1?T_riDhxv z&78u5d14OC&8nU|@W;5|jeY##I8?+?*DaSQo7{DvP50e zlpK}>xz)1=w*R>L`__S6IsQU;L%(Vga{c;?H8f-0yUEt*rWEmmpq5`AHgjoHd=60< z)a?)0XUz9oN9EmZ_wap+1J)nv)Ctov3dFYD^)ieVbPZ{Fe_v_%_v++pVS2w^q|er^ zCua_cv|(EJQr5r9k;d`99hziW4dfmeNxdhCQet|Ozh%AYlj!~KBU_C2(#QMjyz2+H z95G+xs=lCR1J-+JhRLhZ_}}ZAB3Cp8^QQUC4!=__HPKq$Qq>k@#h;7UC}Y8_S2vHm z8G2touTS-{c#P^1dGV(@rKW)e`rZ!=T71Xrh@{66$(2Dqg)+d0+SYp^5fWOo?bkgIAx<`DdRkNgb zFMns1x2;5ttRM)UjRu_fpf=O@Dw^IPVb(DN!o_@etL zBRu23rOF?+-yPoR)UOO~m|kUDNcF=uwd93Y$8Tj^b$)#2$}~yhtPk!FS@;)XJqEX4 z9BwR~I)ZuR!(tbqjopT_y&x2(`9$mx5+AmhlQ$20*@phorrC20;)S|q>Xxhg@Q}Bj z`C?<-y5au%-9)Fq{=&|xmzP16A)8$60m3I%gQZgXT>YS)5mvA-)k1joJet^;*2%ru z_oN^H-106|MNoAB&Sc3V6*O#@HV2GTmktex@3uctMwDlz0d}f;HeMl;=Jcy`k$kBd z7p#-td)G6;vX6fw+U99G5!}z@T-ZGS+&$l~o_}1UTO}g1wcYUgujlH-?CsdbqtjE- zGw|+ThhwdcX0z@hGPAv20z&VkBFu|CcAr!9HE6X#H^PnQ-pqaPzFc6MKhW6@!hC~BSp8y!%{?7A@3)rkv+1band=H!TY9Ta^z_P z1T{;2#qB9W-?aK2I{Wu1S6+y&DM@KwtHFF8L!5f#=?-H*^(XkABbVW?5F=5ajm*Fs z#|KTSvb$kDRuNfO!af_9T~@cr#rOK`@%NkI&u~gdqM}M^#RE-KN2%*KMi(dR~9^dir+%n zBh0J86qiY?*hYbBkNLyH=YZUJ*O)Tx&})A_)<+bnCdrf5R~(`hM!Xq~s(yg#*lD)1 zvfEy?_j6F+Nmms+#JckE_hv1MteLs`DP= ze)If$V7gr~r@cnTI7j<#O>uH#*K^OK-AM4wtg5@v+2LPzmXBqMp$C-p^*mWx%?&6b zKOQDht^+}=x*i&iK5EgL+-5g;daj?N8=RuJNO`Xs8RjTEoDctovnY99y8g1FQofVi z-KFWAzX@4ptD5ynhgOgwbMTDx%I~SB{!g&)dW5;Jm#qztJ_%$NP>=EM)wE;(CeRCP zX23cx!c5XX+(3-d{1n;nW23AO^p?+>XG}nzO4YV)Z8lF59zI&PaPmE_-=u6WCdbM? zTAD-|Ks4)aaU_?EBUQTUXa~JU@VgDpY1=o|KNwj$&xg|-I%8sVVn(HcG}qMYlWFC< zxZne|`c-Xto7is_G!y&>^JgVhgNRf);FMT(S7x=4!)lr0+~#CuEb24pN8)|c`IdCO z4?FgIoo;2nOWfmfkF4ChYRuf|C(FCu+fxR69VQ0r1Z1ZVbA(^$#`CgId_S)bI1p)i z&})-6ccKbIs+26_H$AAm6CNg&+gR4{`IFyH*2qDBcoUBsond{WsdinRI5Slt z_%Af;!!6ne_4w~a8fQO#AZPd7`!RZP56&7SN5rFf@;vf8EM{0)V-^!(cm);nEQq2Se0RER72dsDp8NJxvS&tlC1nJZlvOL5 z*8-DYn;u!Z#2Q0+{I(Ac9=uuqW*?H@9u955A>q9h~k)5kx zO>fd`L}h5Ee(G-ew<<$-50g?aB|w1L8+@!PkC@&waUQjR6&-?~ktWm4ldd!M?@R_1 zW@x~DANq_eBRW}TTx(@E8}^!t)=53s$(YfT8_XEZI>+-w3feZ!F0`*(bM99fG*Qxd zKTX4XW+*A5ocP{ZZl9s%XAd=aZpI&KjN9*Uu0UiM&Eg(3Ph@ zqW((R7l#_Y6%{?M>KaUd4!9F%ccfSfA{sDFI`%6X^7~$YTT~X-Pd^IY=o=6lSn@2A z=ej@8>szi-z7yN_*+g7vL}IrySPAsWYGCCtxA`}e7_IQ5q`P#+ZGUQgrF!I1eT63W zzh6kQyf1T{Pb1$#6A|5%ySaF?`D1fln`?E- zxZAxMLBCxmr^`u85U{75@87u?3?_eczNVLQvFRi0EIMn)x15{Mx5eacJN{yIWh??x zrAm>_>Qt{rvTw^HEdD0+SzCoyVm)&2BvYhyV9wAo!FlZlki^c)H#h;C5XH3(&CKa{Gv799)PzU_e>Nllis(jSgCH=xnk_uTF5zgf-|szsN6 z1M<2p-YuaQ6&JRRUKd}ZW4v*{8@3Y)h#tt7m@JQHh6D`5-cd8owl5v|F3EnbV9;~O zYhY>kxdvHO{+i+R)i64w`YCl{-SwwGug6+2#Hqf2*=I$hs~a-Qm9yy+1-*8@mmXno zBeFn)8X-EbnetArjJS`j!E(gq{x(a zg%ltm zwXIq^OhWx+J3~^e=$PB-X0)2OotM0d$+Nsj!#Vnfb=m!&lU+pHDJgCJvKP~vADiCK zWnSpMtHk7O9#jq=tUk?@_h5ic3s&XZK>|m2VaBHxr0P7K*!UBK<;g9p#X|PnnGj!f zwZ#WtQAY>V6C6&8j}MkFvPyIy!nMm8`EN{Lj1aBTB<9yen945>cF$M^r!wzid9Ypc z6W>ChXQnQkZ!RJ_t2*D6LfGb=PD&RNp$_5xHl7Ej7rn#5MzKU8at-rZGQP)2xjLnZ z3DB9{IDcg)+lx7Zu>(1baFnAx_up`N7n$&?>w88G(_KzjgFRXNXZt=9UaTHq2@*1dH>cuF>6b$+Zt4}eVBA}G+Nn3_q^_D62%m`5M5}m&^dcH(HpXa z+;+>JVS0Gc)B0x32$VGI)TZ}5K97*bI>pH|bJf*l6un`|w3j$R!PI=NbV&uOdb7zK zB1J~mGdV%TZ)G3D4FlOIu{#fF>K&P|mw%@~) zH>+BAbYMOy(?&W|rq#Xlv_22ZjpoXv3$C5f}|fDn_QGzSk-LrPHsGV&k4RLc2W+Gd)M?B3LGr zNBV6bvE1@Q>tP#ZlE<92i9Xl!;;35bU$qU+&r5Tk(446TO@9B|grc0Ay=NW9y2u2i^5ycY#-q)R~4NyueRYr(`V? literal 0 HcmV?d00001 From 2e886c4d505a85be5841581dcba8fddcca59e6e2 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 27 Jul 2022 18:29:18 +0200 Subject: [PATCH 04/29] Updated the instructions for theh CI/CD Workshop lab 1 --- content/amazon-ec2-spot-cicd-workshop/lab1.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/lab1.md b/content/amazon-ec2-spot-cicd-workshop/lab1.md index 8b0df14e..2799f675 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab1.md +++ b/content/amazon-ec2-spot-cicd-workshop/lab1.md @@ -65,7 +65,7 @@ EoF Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. ```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json ``` ## Sign-in to Jenkins @@ -101,8 +101,9 @@ When configuring the plugin, think about how you could force build processes to 10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); 11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; 12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; -13. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); -14. Finally, click on the **Save** button. +13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); +14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); +15. Finally, click on the **Save** button. Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. {{% /expand%}} From 7cf828b13b3e35fc00a75e44daa882fd0a19661d Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 28 Jul 2022 14:56:32 +0200 Subject: [PATCH 05/29] Fixed problems with the CF for the CI/CD Workshop --- .../container/plugins.txt | 2 +- .../jobs/seed.tar.gz | Bin 17678 -> 7995 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt index cb383d61..604af92b 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -9,7 +9,7 @@ git:4.11.3 github-branch-source:1677.v731f745ea_0cf gradle:1.39.4 ldap:2.10 -mailer:438.v02c7f0a_12fa_4 +mailer:435.v79ef3972b_5c7 matrix-auth:3.1.5 pam-auth:1.8 pipeline-github-lib:38.v445716ea_edda_ diff --git a/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz b/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz index 784b2bdf9feb3e40e57eb35e7b0c18efb7cf3122..8a4669b495539a271e0609ba43e77c0a5735f6f9 100644 GIT binary patch literal 7995 zcmaJ`c|276`&TNr+;YoRNle|8_Hjds+*7F}dqPnrw-8xUWX)-pgi4DgCS=Jr6;ZZR z$(DW3I>b_pzzkYwt@mZhG`+2|L&w~>5o7mq0z5j}dk4Ec-wVV6g z&Qptgo%ZrrewqIIpclqRcEq@CWp`&}TWO61DOQ;4UCcS_y+_5QRxvlqaas2t&n@3n zn01|fyfyQ1v*}}lH@dbvGb?r=rytV74ldcQSf_YL(wnEt^`;+{R2=*K(X%>8>Vr33 zptCP?!YOY&%kY$~=dKI;(N`OmmxMb<9@&%C5Y zqUtj0Oe9TUOg1(ofMZtu+;624!=>i}(E(IF$?gRG9+*^&t3C6_SUdey+!utbxV*=6 zRNbJ&C7EIwSr98%$^T(!->o2^B4_!xUuUL{p3HG!zGlcJrdTuUc7FA+>u7Cu>G;~3 z`McQ%lLxm4O@zorYyUnCXV2P`x?0Ao;E9$DWZN7+|3z*LA&89Um+s5U%oQ0Wz*%-6E^`Ec!P?vp<95q^bwJN)pGeC4@zlL44TzXC=PDQlP zAA9myeuUd}P}l8=9~Gli%0J|ksftj~afIlt{y1sQJ3U@rfopqs!c6N7UfTY>`#U!^ zz&9yR!WTO&pj&sodZb-gCBQbe4fLvgZfd#mmASH)hppro)U|I6f{oURZD?3a@7+y4 zI_&#WmLU=Lx+jTF08ufp?th zd^5W#W<}l6?kYtpp>)w$U(0wotfF|YCDhE$teT?CZ{Qo5rk4aSu6U!LZDUQgx>9^UDb5X5x#m+f;;cV+g&+DEp^tCA=NhslNMwdR&jd1a1Ehp zDVO2VoNL-YEr|Eek-m8kALl2GpCSmUu3UEWV~X_*r{^W=zfz+|`K#Lk>|UW37ayw| z%DgmnqErjzR@1-pt%rW-arfo#O&VpN@aH^g;Sw_;$W+@$s)}?*{+0ew`*C5b1hft zo9`$nN^sMHFpiqu`1#4L5i%+D+aq`dUmWZsF1J8Pgmy#p6)?kE$m|wsN zEZ_58PUcQTY3bXyi$IW9H~0LCTYELeE^4?o{s|6t`M)+_ds9qI%s32$p@W6Ld)|(A z5qx~**|%M=`dllHuFC}mD>0=#F@3lWM-zwNfzJM0gJ?6;GDvQc6YQA!Y!66NCp3xa zY17P;M@rGGY;I|o0JM^t3=)B9`7yKSX^X!f2O)+r;Px|E{wJkDn&%1RJ`FG^n8(vNXnU9eUFTIF>>*L?0yM#A4^KDGbE?u(s z6%3DckA+!yR_f51=&J%aYD>>SJUp{K=>Zha@zs9pV&aBTbso)Wwcnf09=D@mshd>> z@h@Hy2b#_XA&VccfPbxk%^PGU_R=GE#=!bHjQwQ>g#>ap4*BElV)K5yW*4#ehQ*w^ zNi@ZIEus5CKtF1l2E~oy2@{Oe72jxQvhB3N^hb0PFa#g`^~!=kaiq9lf#q^A$=81i z`G(|-Kl@nNlW}=Z!&Ow@Q%l=7<`ETjsL<~kv2G6E$nh?R9`5s8m9U?pJMNf*eRHSi z9eu^v*JezWP-|#RY)7SWr1sU!CF{yBlAeFK`;8CeIci_Aa4UGi9P?6AXI^3RsyTAw zgdc0f4(&!ft$Lcx4z@bw?K@vP9N-t2US-F%Qsq>72z*_boHjo#dG)UvmvgV>wLX)j zR8?BEdrGW@wtJXFdQAO@#=Z-aZNsnRdDihv`55>7=Wg{20r=j%TCP4rdbTW8d)+UzEK@0?Av>0Yv=BZiWjH_^C8wkE01iCx}U8Sfp>`90Sv z$>yunKj(JNq$L%_Coem%;ijx+rK@~!ORLkN%#fY+Y56jRCy5Lfg^5KEU0+J~SrgN6DW~f^_t_eEiKr^x`MMAT#@S47w}vQZ$Y8w}Cglbc#m6ueE;pPC_WuVN z_|{9M|9!$nImX9thlWg?g%LxoLAK**EbCx#M9dBY|LV^a%Mj0Q&4Tm`%6;u}SAvYb z8g{St=;V6Q4l&~cbxvU(%b8<7J|jN9#QPOD#ao7pt$ObzL~k2ibYo3 zh1%V_dy<^lax)H!=PQ*i|I9!U`Hd zMSVTrXfF#3Co;{3kVRiO%vVhpZ*+ToJ;mFa%mDYMPm_r1UeGo0z;nE`UR@=J?E!63 zN<13I`mNyL6umlb8hZ}rMlT89-uV?V;*vv%G!D{p{&zRWA-J(%2zkG$4xMWWx2EVe zYe>p^OWoxq<5t-1=<@dHFICo}@|~`G%xIFRx{5)lDK^jBn&} zcZwd~j2!`2r_IESP(*u=h0HfKVvRL0!KdI*FS?@#8x3j}=S-p|Dq^_K9MM?^!5oC( zL>Q@fh|eI4dfBdf^lGk#cDU&HrB`g8hDa`?v3ql3;qX-=EOHm2bi{Q%XRm{hWlB&} z(swNQxy)G(Z8fJi_lmsfa*)Q=T|mw5Ian;RtDt2GETRS28IUSEK!lPPeHjsq){W6T z#vla8difN-D`zj1S2+oTT=3Kmh@OCYRUX{Gy7xiU9-P{Yr}_|2Wpk_Tds@&Z34aj| zSBa;Qf8o>=h2%((FU3lF1xPKVO7LkecHcysafGgkc<3Zr1{pDXaGmeqwhx9Y?~cLn z&M{~Z1>qpy=$MwV8fb=~24)@|phQxeXtFztwhAJP8;7~HU_QE6fcY$+fQyzrpZyow zotCNmU>IL^U1nw4he8?i)52$3mkdw#^Q|v54DqZ2}<#kHtNd_Omen@wWUc+$v%7*G^FBDJ(ha&uKHf&fKgal_|H zWH#IpL_8Ng%%P{pgZi=~9J-j){rOXL7ZNGrQ}o7T&8S|ZIEO=_n)l68Mbt7}rx$jo zf_v2=BJ}3JxKfGu%_eU!$|OnASsuq`Nw;Z=c~QhB_9TBmq)jZj^mFJ-JPal?sC6x> zD{$S>0qbIq5U#EEVk3j#g)`!Wx*{ghn86ORuJLCOZBsRpo<+0vKn{Np>?hGkm-A&{ zqTCN^Qzz{=`p7xW0qSBLna%!7LhRgYk#S~ z=hM>2fkLW^yLRTiuMpHTV?=^EA<-BlQzS^Nluj`IHZ7f`li{r}Fcb!5L&`IV&VnZi z?PYNEek#n9yLzk&?55~}gGiw0())3xWD)U2lkZASB;U3C=furHXq9Rc$z1B)aB!{= zGw%l(3F+_HY2c*4f?j(Utii)(hSQlN%>5t~hAImf8slgppLRxrkH*hwfukx2ajO+2 z-$x}F?aZnLxrOhDr%^K#)Jh_?T0oB%(w(1^kMm`U{+sxHQrd0}!a322QuwYBKBwo9 zq_lx7zC^G}gPUK`Z7;w{njd=oD6aHD?Cc!8bnq9hHnXtxBiH!+8fCTr)N%+}%B7J= ztsL7?7g^Oze$8Pa7RN${uqJVm6X)}d9^n=aLfS02!pq>k@mk24QV}CXF*3FX z7roGn#6yH@kBF&ijJN#~xJLu<+fSy-&nJP<9S1G(1)7?LKOCPnO+$S=DP*ByBx{#X z!!j$WM*sJ8lqQ0hOvB2O@i$riwTJjdtD6eR&H2KcmFuw|); zZhx*nMPEFt8GVUA*KbF8SUm}mJ2)1{R@dL=1MLSu>VU|&t;coxL^miCTm4hSHr1ki zq9D>r1*qz=GI%h9+On}#r2ft&6W_59%*WtyIs8qk7s+`YB23Jl$T5nfoBMpU)59mD zm$GGAvozy}^Q*f!#x`0&6?_KqX_WLWVUe4F@mN%jA#9YiF1B*-eyZT-33{;41eZqs zp&pGO!?&D|CY;U|l6*u`kLGsR2Ifi3}}iTvlD!Xf%IR-&{{bOfxUvIM&z zV30EBBw&!Av||WqXR)JyL)0V8p_riTWdGFC2ZXel)Uir1?4_AT+!j(r@1i0HpGL7S zy|fl5-(1|Dbg2t?Xdo9fJgc8&u+)vbj89F6GCaMF3j%kZ9#``<{3BO<*R_ueoxaa& z_O!IGUij?8^2>88G`nj86r|!p=|4cKN5eU$=i^1j)p_LP^oRrP3e z{+N1Id_F_dFla^hleW{s(wch;+q&KJh?!Aq#80iwF_&mv=q+(^Y_0B+vEpwt>8j>} zC(7!eR86Fh2(Snt=D4a1YOe}I8!LlRH?FH06RJKVnBoBt)Hn`quKchy4!S{ao=6)TQN1RS`Ijd|VVw0@XCRxD;vO>85rnjjn5iB_1cKxZ zko%P?+f1bz>rpqMT;!5pns^)*zD84%;31c3+yjU92NKf%ikTviq(b`k*%Xf>tVml- z2Niv}8Duuaj3MwfYPANET1n5R_~$TUvOqblL_l9a|DZ}5bSV-2B>rdU?h$(Avx?GN z^r|#in2D0MDUHx;N1N(v2~*=3up(ryl9FrClArQLTQ+QyoJL6^3X6Ipq!drW z?is|co1iY-O}KuEH)M_;3VyZb_*(+?j;>({JhZsz4rkJiigy112FFFtE|7>K@mN^~ z^1mvAdxV@$!8?Vk6}?Ck+R9}fT}-CkzI`*2D%a1*;LKyuv<)i*iJV%HoDJC@@abMR zSru_mavc}aBHt2`+@?FI&7ljnqKqRyJIek**NLS07N2&1&aq;ltH^ZR!|A_hbv=%q z)iNegABZ-E@ZmZ^)2fAwrn5_L9zZik5&F^@+9J}3uh|lQcZOMsBai8GYenW+#3nia zO|DWERg5A_LRvN1kB>iNAAi5XpP_6Sfu*A`@^S<&cW`PLIi24^S_1uYOR5hJ65U&4 zN{n7uwUvE1*WhqV+BHLlUu?_W$hpmyPzr9Oq%_Iqg$G~GtQ(OC?)1M9fOg4IeVw_3 zKW2C}ufLX^*xow0t;RL)i^ar7|NYP2A$VZJ^bM(R?TY24^n|c=kp}By;&?^Z9$GHn z6w+dT8_yWek_mu^XZ({N)mkj!O#C}>>*>R^?Q)Uj;a=Gc`^(oTDH*WYlQQN_6{=Xc zJrWp>Cy#C{)uwJ&Sii?(e}8_%=;rimteYyT2{-Zxx2QNLR%`#r zX??Ji=)D^TN)p?8l`vU;Pe6I#Ge2UDe2d`{58({{%+~95R&|_K{=4JH9!Z6MyZ$V1 zV`!Yb(gph!L@MJS|96v4_JSYWOo?_u4iO5KopRHi$A#dhVp>A03T`A_mRFL>+UmRCqBZ=``!wXW>rePf&6r%$bR z9deiH>XQ<#5NhX#f7%jI9XOaBe`l9R?w%cnrxNJdhrir+eN;0%eUgr{yhaU!=x_I3 zRG@`t%6PXf7CvERtJpiBD-(hp7tR7`*ptXlvSn+5OeX2h9Y*QCh_o3229YW{chpf+n@w)0l2@3Oso|*K#iu6Qs2wO^U9;DLm)!Y{6mh zD;XjQtLon_EX9+eNM9r>7~JR4XdoLILSVD<*v5CgHtQv=epwVz3H;{TO9tF~2PkRa zNQvKeW`5RHA_yn@D!+*&HJ9)Dz~^}(k?HmLS4ZPm16mgD|YGd>$az^yT8Qdn_xImqJbj<7f? z7VH-#exZdA(w3X!gsUZyoxbj8y`^7_Dc4|INz0H9s%jY!3+7U!CDAX2MV7F68hty= zWga6N5Oe1qbg`Sl){tX8*jVJ!LG;)j*f(zqYpiU7xvXm%GfYLox#B9QFC?i>MFu|Y zfj5oiuwlZrEs<3C`6_&)a8}JtavX0)s_BhW2PBEApn(zO8fR~`&~*eH&%Fa9kuU$b zc|EU1zahp27U#o0Z8N|c3tXD~{r9D+?_g`nXNxoCG# zZ);o})-z<*JfvkNIPau}9JX7vV66MY=uB*$oYCU!qm0C=X9oX7M0%h9&dS=o+C9gx z=oQ7U)`@+>C_u+#{gZjpluj2Pp{c62re%YjN+g<#AeJ_g^`z^`u6(wp zThZy%CXeXPei;q@TjYZks!D1;>fET7w7wwW!j#g8MZDD7u5~!X&lgv}Cmg+QvMI6o zS#Xym7R2LMj^S>MuqPIDQ=l-Eze}`k{?v#IMo*sy{KVk1(!`Pa} z7tnPH9aA&1tyxc&058i!p~8I3BeU%Z89c{H#1;(`LS)uf(jd?gUr~>y+$E618%YcG-d{hNv_AwY8w0T- z3~nCA*0Zkp@TtR~Tl%X@agC5}cvtFjtbdc|SXgIwN9NOA7PkyS^?{z1$f(inZ?ApDwR(iD|Z&>-!>yMsynxCB?o2x+MKBUptt+0+KNt!g0||H+p?mLw$rj% z7hCrURIP>T%MId5EcU|Jw1T9{#bfdCr15ZGEBsE$qfV^89Ew^NTbUHej3;A|@>MNJ?00PXNrj+jLHp(v8N#@?3giKq(r!>yR!;n+h|Q zIX7`0Mwa1=#Zi5 zcY%au({}Xs4|+dP74o2&d8o{J!?L@y?6$4n?-wJjEbJXkvdK#A0M^RF5(Nz-!ds~h zFYQsqqZ43V^k|YeQ<*a==7DtyhF1kC$QG_eyL?N z+nlgm$a_W+OXx$|zEHdP_7|bH?mEb^flBVvkCGACqkUFES`|z284Jua!l)E+`-ys^ zlZ1ZWI)c#Fxwo=Lp<|zUlR|-YA$rmVHRwrw=S{e|sr{p=a3xVT_iiDbfjdX*7P0Eb zZGRi2`<}6I9>;6`evsuAv2nkC#yu=l!H{L++D%1vahGe{ZyqQzE0~K;iCI-zFAN9S z9_-NFBfAxrSWsyK24lEh86b6f`GI?xpn3B5P66Yq!`2gOwwXJ~Y?v zh&M9jkx;b1EMY*3IqfGPWdCVgR1MZ_pWX30d;9J&Z?N;T zz;fs_=!c4FO?YHY?k2?b3Z5YyxP~Q;&s4{^nEl&&Hml_~4Er|rliHRSTw}@hF zS`95aqL}RWX*KhrPTZDf4QDmuqo^ocSRXdqoOV4}4r>0xRYYKzI`-nTE7PtI^hFI` zZO>ab2vb2VeA0fm_#w6w%HIx(YX39{+~S1u->A`(1AJD8g~!KbtY_+h{7487jT#>O z1ma4;+yUYUID~wAC2<0EIR4lHw;F;*L4rCSOMd~vh5%zMQ01`)gcd~LP`-O0q5WU) zviFi4VYE13$^nQ_CI9P?vZFhhLKB(W$YLgJ>i|O+B#8K!Jzq}@*yf?Mjs*D8%2vZ> z0Q==?jh+d5xAJU~5qAo~FC74*5`gH<+vau(G443wuQ%H`)pTob&&RbeAVU2JI3a4f zlo#gsL`lv?WcLHOSMq(?lj1po-z@hgB2{iQi`r2LklMCGk4iuJ`j@bRz6&3>CvYvP z6`?&9`4$6SSxU=`R8E;SKK+bQ)x+LDBgujjd%eiF9Q;TH&NqBZFGvPs2+etiy`Gd5 zsIc*FcPuuG>D@F9vt*pP~|Aj_VE@! z3znz8+v;!Y*sP4$z}8iQ^zlTEN?eKK!!?+wR-$-C__c1Uzhu{YuuNK-u@al`C#c}; zRSBYC@g=FF;Nv#}`A)s|aii4)Cw}~PWuXjhRXbYbE2@^hVFY>zD-C9SFAn-KM+5t1 z0r#Z{_>|UqU9cxnnBSA29gTK~eC1#-1$2W%k2EeJ>g+^Vj_$}n%jd;7%~7kRFSWn) zcpe$uBda*DuP3L0h;Y|)9+Kt>17zZH5S@zGf?}iNbY&biD&~|v-0ymPRh=w8ob!>> zcNPBFBd^Cjj}2+#26CKg3rJ6hC*JAk<+d=~jU`57N=Nn2m`g_)AEn>*{kqfmdfS&6F-v89z|#VQ$;B;}O5+8mgC6JqXY9k>ZzXDis1 ziQAXYqn`wb10GOKVQDx$FVGhnci$JwsZ!f4wwYi^vPkx*%M$F zv_TCJc*m889_UiI{^DWxDwqqX>Hct!Cl#0~0fgV&K*WrVBm+BNiEIZU>P)QifL7=o z;HxncT#Kj(G7pb=5`tC^5Vsyck;qF@eWCZe0Wd(33|aT@W@-8^6I%GDqyB=YZB1+!n81oq*+z74{J^ z!A)3DDD@i1Sq&f9&NwTk0Ys8Yu*#vf8=ywepjEPk-{Q5r@eK@!UUwQ`-UEW~JjQZW zl&?aO>z9Rp1>#^aCDin5AWmF0a|L05>nQU5D8O9w0&Owcih=~Gq#(h5>3F~VFqFMO z2>`yTx;rWGLB?#dGo*=6GO}w15HXqk}(c2+;mNM zvw3v^6dV9a(X9RyYaqqTvwP72dAKd1!ZySRL_Ws~yHcb*3;ge9bO^26XSKWM73#?= z&{Uvj2FF}M80f9cf==+{0l1!Jug*~~3Sv?R8Q;J;iBnxkpBx_}&g4Rmp# zY=%9iN)St(e2k@B13ojXL7WS1(YrGkH5Ae@_#A8m1fDH^HgH2LmjIno=+2B%7z#p0 zQ4eTEUcMR$So)vikCkVOf8okou7tSdtM{t63Z5*2cEFXV#J$(g)1xndYqP*-0~0jV zQ~s|rirbnnH~6;UH8!K6n`Ey(VYCcCnQz+N8BEu@ZOuzgd!z2;rzl@J0SaPEdOnYg z8JL5XLx8e+teRC>2EZVKgKf@)$12$B06YT)jM#uFju0$3h}aZ}HC~K@q*Y)qmoqU| zrI`Q&$4C<-2o8DK3SnW|0}>gp5U~dJUjI~k#EyRss9im<#BsXjD?kRuox(!YyN(uH0DxZAX3JpqIjb3OsYFyL1YgiMBX z0=UHK3pF)tp#j@3e89>25~r8Q`L`49qQH*NXCTFwGywHFpz{gXvXfZ=#G-*UmIBAmY~k` zewQ$}Upo|F+=D}|SRR2K~H?vur z%8LMg6y#mm5t1^hIS6)%quBq(`9FAB3rdn6<2Vbg<)Fs^)Vg0At;8BHV9TaiWPSs; zGItPKRcv==CLjVwxR;m?+1P$^1AXi0e~%px5h0>ih};b`7}WiS9YP!LW*$vS7$ zV9sMeOYtQGAZ(5J8N5FE(&hsrz5388X|n(0QDJN3S-L2uAcJWKB;#@|dwd0>MF((+ zQ|tjzjLo}{7lohUlS%mBJK)96m_akH5^$)lIx2qiSJ)W=&yqaTRvSv5T|cC@PqVIE&wV?ELJK65BI>Ar+0rJUhMCsV1YJR16iWE z)Fp9MQ(aPxEpAgZ2)Yc?Lk(s%|G)F5t0$V|68wddXVBV>PcxIicVgC0|DVLUbO+ZH zLHr0va45W<%A`0@p9m1XaSo_ooz5Z*7_s>efK+n;hSQ1SCwm0-a}pPi4OgrGv*1;BcsNi9_7$SN=aZJVIz)zM8oL z9;}-^@RJWqO&?)egRy#WNml__F!?(0lZ?k2AlQdudH0|tVp4{8)}Hhh13&_JaSL_E zo;Ho6e~gv{4vB>SL*gz9RxE}pz7)pM5W)!ZOVAx5)8~->=-obSq2wVXVFHj=p8}XU zM%o~?53c2t0;S6*khV*S`}009&PzG=37B&IKhrS@aD4;l@nTm?F2NXu&cnz5B6Gwp z$@N(#5*8LSc?KQ z^k^5<1tAyig5V?!I!^#490dc&f&3W=59tJi_$&Px)Ek7ajlxanr%R~xt1|T>kl?fv zh5cQKx=acoB8K=9%;FOOu_qR4i=k|emimp`-4O%rm?AHu-j&@Su{{LZn{ey}x_HlUw}}mOHShi^0Tm@WgvLz5YmhVGnnt zM-LE3HGjGQa7~39&dG3K5)H)xU!8*HAKWg45s-Lx2t_Pzbn#rV9up$Q3@CQSW{)P7 z0FGBZkHcCRj-LMtQ~|^|`K2710T|#TAOt9+S41lifdz}IhTfw}hCDtIw#F(sp9nW3 z?||A+5WOXcB+)tWI1;B9^UpAF-UXmg77O#g_CNhNaj}~22^5F#gQ6~ZOc6IaRe<7~ zo#}xJI7f8#-V%V&BcPxQBDe)8{5Lq+OR-=&L<}B={gSh|hJZpeoXhS8P8a@1jqG?> z@H%YpZOPIJ=-=?dP1iG^Utk^k`&Zme{s7xUK>REm=Q6K<#6g!;h(yMu|K}tD~9>w6D*e^j)#kw;4vR?HSB|M zGfFwK{1-sI0qA97Bbu-fl3SO|h65KNPT^?-+09r;*p(u+z{DtAU6a-ta0cScIR~7V z2w;3~Kt1<9v4b~vGi;^-f%B*^R-br)s2)(esxux9Wx9K5EnfhWf8kax-;c1uSG3Nz z3Vk!SyMR@*_;1j5ylusT!?ed&i(OE{MTbBFE2JLVi<`-p2l@)9w{VpBY7w{}19|#? zY3{$}IY5cO3J}MMHnsn0!e|^R76GCUS@wEjG+K8-f`$M$j;K_NlHp(jFNI#DqZgWM z${M$d>+9~K0@&rYmz*{WA;eAF>>aq)B;lP`6A-=5^mctD?*UD-@%^KG4o<<)XP#z&$GTc+R7C>( z260KMaY9#|F4@l%j^97li9QSKX$e*AEQ0+MevME?etJN`bg$XH;wjJ1^G;*iJx{&w zwXP(d9jx5+h`MfHf|YQaYF{&>jir;Izt^PL1zpHj5oSa~;SSGQ{e9z`e~KLiDLQ@k z(=M@lwf$$3yDIT64$c#huA;*Z=)vA%xgSvV1~vG$bQuMGs)BPhZP#*z&_tMj zIQu{HQ6vKpzSRB><6V&2Qb-|=x-VC*i2P$d9Z>obD=fYRJi7wb%rV&SSfL26bsJC* z!ow^QU*fay9d=Kn<)Urua63LF()y*eE8 zGoz|x#sDe`oUA3*(ZG={KI5zV3&7hqT-L-OsIj7swXDDNDtdQ2?`Ld`@u@iiCCbprFwA;-#k&p_&*b6N-g)5Bosz-62T1G5jlot4jbzFyOS&3!hX6q%L zk9Pqwq8~^8VhbZ7LEVBUkN}w0$4fc;3I>ro`IvCFcoU|zxrBi{#yWal%838AFanCE zuog5eNMY)NuYl`Yf^17nRSO0r<9<`&KLuC>FaB}Wp22WsLqVDUIav;@+q)Zi9M~jr zp6ufjkX=2XJcX62FF%8v(1f=VB9AdQIPnff^)_D9@+92g?=I&H861do~ zRe)CCG|oByd`k`8KZezo4-jQ28|fE6AxTJ{CR7VCU)=B6*f%=cG#fPsg=h%4CFt6h z#jEV~<~_YfWX>A|f;zIBmKfUpsn^It&S=RoF~mRDiOvjta+kKkk~*08_-D`!(so(! zO-dBE|J0wXt~Oe!XVZ2)q{890PIOOnC7wQE2B`qGLYUo_nTW zs)S=!7&u?Bv4snM(0_@%%}PXJRA4Zjc%{}R%F>touj zDaE;fcF~>(9O=RW#vuwd*T+Xtnrp$TMez;KBGU}pB&JGUhv>osape&UtJ&pVfy-|kw!-wb1YWb;vasyB?cGT!L;c{si#M%oK7^|KO_@A-8fWvC7E%0OD%G3FHVuocL7{z!v7wQ5Df+KqJz&08Y#P zlc~66i%b5ZgX7#LnIzFMPA1~W-=-OICV72~7Qh8Ybbu=bE6yv0+Ufv>Ct}wsih+DI zbP2|*aA`MiY;BuHWD5o?M5wm_#VgVEY7$^NLI4G4Ai|Vqf&U~5b3aX3NL^h_>HAQ( zq4Gl@sXw(_bU7$)ruN(r`HxD3r&tP~iWUg%rNP8IhCXEZn^;u}`*o(3FVcW!zcK z70?3YL&*0o07L#9wrpz%h@fg#;HWA%f=0^%`~{L{;Gq29z|isu`wbGDDn2v8PMWj* z(!dzd0W6XzZ##uAHMk`fHm_J8*w)CQc$WLwtn~wXmuL!K6itDBQBMaFk@$mB z@zcwR*n%G)*@adka@Of%U+%T(_`{aNUbolZso>Dx21^*5E~8%DJ12R$06!*Mir8y2 z=ge1c*(Bg2GSAeOiS5`f5bEPHOi|KHDk!jGa2G-asayu@6ej;)gY|Fe{|nanPab}v zGPd4c8tE=^`4IV(Dx6i?z`w+h|4SJ6=yR!z8K<9O&k>??i<+;zX~!p0-~IoQsU!b? z&D5o@GIfn~z`eFOYiAd9c7ZOqmwbK*DcAuKrws&PaMtZINY`Tru0F!W$p|cEzz1tw zO8hVK*;xfA7D0*@06}}e#%>A_=D-EaJDW(IKRCL#r1h|eNCli1;ocS6E z!X+%WD^O&Oe_SChtKJj*_`d|`Gq96#MFyPWhQMW#gj1g<>3?w1BVs=vB5n7I+Zex7_aZ8p6pDPb`fIXsH`xqp71LvqiBNHVjdv9O#7!NM>g1IK| zl2i6GE{^l{iJWzX^*#2_64>1MEsp(mu2z>>?BO@p0K}fLgAPCVjhMexgr9my@iFh& zCyVdoUU&fOlKW(=EePsEhE&otNYaVs*!{!gnrdc8ok+@=TM`-Jrn7@T-|D=A?~6g1 z{VJVM%k-e{@03j&MWrXjE~b;*ql)HbzGEJar%X6mRt<~!?2|1tM&>UbZVjZQ@X4{- z8)kRX728Vf4Y8d_S$yuRXPfJbamg@-4sA2XVB)v^`2{vde=?r>?Ud2bG)8t4pUZgh z^H-;Fo*V5JN37_+(06_;QY*QwfsA5?uFqXOkBZj%T37taKl?7%TG`p#RY}d*rtxL- zzs8YxvEeD&DVyM$-BPhG*w%zmSuPpBr)T5ak)Bw`fqq%GjfY~MyAdFn+O{7iuFlx$ z8ZioC4ais8YGA86f~`K7=eT&EEnUOc(!m+CQp@|hA9Z4Ky$S24tV8)IW!(x8VnQCQ z@FBXMrS@4OH}g@BKbW_+R_gY(Pxj?jrOtha&GqWn`=-8!8Xj@Hw<(+ZD*RW9!RTq~ z6Pk3ud`(tW|HC_FsrJd;v;!=Yc^tFlW95*$f1(AvQZ%E=7*omq{?bXWNeH%KD3dSh z0_sZEc9$2nJ2s03K5K!HYE}1okL<9e(x+Zw-GzO-sf-s-&b@*keg&Us4q79O&;4zk z#Ac!OAIHB9@`>!ukcu)A0gj{pjoRw+yHpucpV*ciqaan*ZEvUmzNt+Vh}|U zs|P>)oHgV+1BPXlh>{fIDBeBltA5Q8!+UN#JG=NK$(!^~nF|3?eS$k{3F)5Dv(7q7 zf(v2U$A>)*{ohpYQu4}`AYp8^P{^Xuv4sAVVu|H&+Uw?K{P|9Df^)B7zl$pVaJyQ% zy9bprFI?Ze%GSIX(mVc&LiOqIeWxLpU-0)eOM`k<>x!(0?Xx?A7~Eu>Zg|$pGvVJ5 zZl;(OIxZ2p@6vE^R=x1(WgDepC0^MWO=_xiWjPv#Pw4m(8` z9kiJHQ*C6E%S?{n`n+;zN)pn1<0DlTwe5~ec17~otQ^6#HL^;xgl1+s+QF2CoM)lhVWA zR4Xk{=U0gZg-br5|Me|IEb9-FxmNF?`w+- zpJUCO5r@A$A?;$rU5uQ~a@hl^(Uyzz+<8zmv(h z+xd}44r>uEFfY>P1f@RMO1u8-t}{xaKYXSz@e`XHp8it4=-TTEe&@$aA%Q(256hwVL z{B~_Z7{3l_E;0?NY_VEh%GxTKzOEFCw4k`PuM(XW9|;kqsPi(CF42c4|^EwA!>&K)~E zC&F|eE6r?}_pDAy1YyYR>O;ICUSV&NfcpzeZ2WHx`9*U#yzPl7!mnrhZ1Ik^tVF`q zl<(x+(Ce(W+`41p`R}gt=A!`KkgED*x>NBOM?Kji) z6O@aC2V8!{H~5ndD#B|!yTuweG@e+bK4E)pFV}lmKbhV#)IQH;RiBo~aSWvr-BD6` z_S}0d>B-|5`i9wEkf!SZNY-Pxg?F&Jw2I+vUfyXp#k`TYyYpV+mX1Y~=(k4=-u!dJ zaqn$JKk_}~G@002%R+nAJq^qvw3Bw<#Wboxqm=s%QvJWO*A-Rw&P&X$3#qOMEGL9$ z&~fOd((U~zHC1sIE8fm++`P&9gWz|!@!9lUKoR?T%`E2B)cyyLIH|CEO2ygUV5W)H z+rOehchPsdd_U;lMg6@88wo!(h`D&hCjY*Lab;kB3UQh;X7#dcAj@YyKLPybbAQyW zw|}D^nHmBo-QTQwM;v!Q~x)W#m_aikKliLKVrdIca(rAO` zlXmG)?-#x*cjn{&P@y!IxOi{%cd87yHZaX~D9n{2dF~?+ohXEoI7WmS1RA8vwSRG? z5n5VkOkVS>>JKMva@hO(#A5!!PIb{)W>SB_{*A3p`Ye=5DJR4f>bzl_X=YX^p}3K7 zkfweN{&PHC|1Jz6F?%;bX{4MzQdoA=>b^)XvqP$&M%n`l#A8C~MYof-IeGmU7I%=* z-<;W{`q2r2&n(AsOx_osBT0Ff*kY3fDFg2jkGbEyti1NSt{r(F+vR{>pR%$*g#CGx z@3C{#CZBf|jg+aZg#U`av`gYXS|^a=b^QXefrvLR!VwQCBGahq&z$r&$8Mp>LaJzE z;5ZrncL;v5>_Xc1clvO6UWy`c5i%nq<+>{u4A7U@R%wvr0l{_--yAt>msq@?`d2*_ z89b0L_&~hd>02jAB9}v)>fX4CI6sz4yjk7zcInmKMkbkcDZ$%zjyWx1@)S+0<|1yC zW^bBjk?$RP7Kj)qmB=2;4wj=~Br@pbxLaJ7*T@n&)gORl!LW>d(Gv2QEoRCg<%i+M z?9~n7qC*eE!{$bnZs#pVWBRR|B!@DCWx_qya|992%WgVrzet-hNfHz$_O4+-J(Ii{ z%Eo-v7(b9xI_Lodu$MG`I<^ddbGp?jZT4o$H0`*4U1ZcL)H%-MdE~6IE5rOvHs|98-(aOGkW5a}oA&yOBjUakCawVl6D}2zyyP4&!6| z)?|`GQrWF=R~JgWFXJ=2Jljh}ruR zboNwUjy={M=jvuJMRzZ{CrnZ(rMhH9=00MF<{-5?Ekk$^i8D=yfLClLa|6yZ*1_u% zWzAu+7qUxai*JR((`Ogv>U^M#qtPd(dvb32KPo#dAHPGcT>`&%X zf8vWW$;T%p%&p`0lcn|vwW{47kAnEuN${>U=Y8jRv`A57)mWy}H!k)>dGMGtDotSK zyNBCCyAL(!ZOIQl`N6aal5SHusIKAgsnKz^u2mme44=iUhxNyX3+i5V!}TxMPXm9n zcwP{Sz%8Vo6gtHjo?;6s0zNc!GS#VMoM^pnQd6U;Z?rZM2= ztq*44l#f{FkqpaMa|B9#0;oy!sAWCi5% zA3QE*@j3`4P$KNo4t53FKgv^roCje_UmMhhjJ%(e>dRm`W=iu-^VnNI8K)<36Y$wL z*Pl;4tn?u}s-Dbk6&oWI?~W*c4`XTwu~+_f7*dU;)V3=TC`s(x-$URLchuh?@-;T` zEL!rnmm}SK@kfGOeQuKV{RfSU1j+3Xg|~`$jFegQDEh8R=G+8rE=p1!3{%d77pAF4 zw6E<-kZb32#J)jh@EIO>oF}0+g=h_>gM{#M?e6sP17hVTH*2`LzwVc^Zwp^k7My$G2@Vmrv= zAnqHqetl>9$!u|?`};*nlhdYer9lsL)6*>FRmmA)y9j|)0h$q0F#hk+6y>DDYbm|vF0d!#sK6B;?IfwW+oqr2B%p!`6 zSKiNg(@Afa*_2~Jvcnd0$nEf0$PGB3bTSm$B4dfOIyF67zK^uxtdfIvORTSR`zNr3 z5($9t{dXsw{>+y2^<1kYSz<5_-XxZF{L}HmI1{!bU2D36X#omB5cJYi&!bDDk2Cb%JPXNYUt-wZ{_w%cDVE-C9#-}g8upj^#jYIDY5FI zsfKomrQAi^$3u0LOFS0VCkc*2@dtc&YV#~cSLC9Of;`yKPS@2ub$XjV1?T_riDhxv z&78u5d14OC&8nU|@W;5|jeY##I8?+?*DaSQo7{DvP50e zlpK}>xz)1=w*R>L`__S6IsQU;L%(Vga{c;?H8f-0yUEt*rWEmmpq5`AHgjoHd=60< z)a?)0XUz9oN9EmZ_wap+1J)nv)Ctov3dFYD^)ieVbPZ{Fe_v_%_v++pVS2w^q|er^ zCua_cv|(EJQr5r9k;d`99hziW4dfmeNxdhCQet|Ozh%AYlj!~KBU_C2(#QMjyz2+H z95G+xs=lCR1J-+JhRLhZ_}}ZAB3Cp8^QQUC4!=__HPKq$Qq>k@#h;7UC}Y8_S2vHm z8G2touTS-{c#P^1dGV(@rKW)e`rZ!=T71Xrh@{66$(2Dqg)+d0+SYp^5fWOo?bkgIAx<`DdRkNgb zFMns1x2;5ttRM)UjRu_fpf=O@Dw^IPVb(DN!o_@etL zBRu23rOF?+-yPoR)UOO~m|kUDNcF=uwd93Y$8Tj^b$)#2$}~yhtPk!FS@;)XJqEX4 z9BwR~I)ZuR!(tbqjopT_y&x2(`9$mx5+AmhlQ$20*@phorrC20;)S|q>Xxhg@Q}Bj z`C?<-y5au%-9)Fq{=&|xmzP16A)8$60m3I%gQZgXT>YS)5mvA-)k1joJet^;*2%ru z_oN^H-106|MNoAB&Sc3V6*O#@HV2GTmktex@3uctMwDlz0d}f;HeMl;=Jcy`k$kBd z7p#-td)G6;vX6fw+U99G5!}z@T-ZGS+&$l~o_}1UTO}g1wcYUgujlH-?CsdbqtjE- zGw|+ThhwdcX0z@hGPAv20z&VkBFu|CcAr!9HE6X#H^PnQ-pqaPzFc6MKhW6@!hC~BSp8y!%{?7A@3)rkv+1band=H!TY9Ta^z_P z1T{;2#qB9W-?aK2I{Wu1S6+y&DM@KwtHFF8L!5f#=?-H*^(XkABbVW?5F=5ajm*Fs z#|KTSvb$kDRuNfO!af_9T~@cr#rOK`@%NkI&u~gdqM}M^#RE-KN2%*KMi(dR~9^dir+%n zBh0J86qiY?*hYbBkNLyH=YZUJ*O)Tx&})A_)<+bnCdrf5R~(`hM!Xq~s(yg#*lD)1 zvfEy?_j6F+Nmms+#JckE_hv1MteLs`DP= ze)If$V7gr~r@cnTI7j<#O>uH#*K^OK-AM4wtg5@v+2LPzmXBqMp$C-p^*mWx%?&6b zKOQDht^+}=x*i&iK5EgL+-5g;daj?N8=RuJNO`Xs8RjTEoDctovnY99y8g1FQofVi z-KFWAzX@4ptD5ynhgOgwbMTDx%I~SB{!g&)dW5;Jm#qztJ_%$NP>=EM)wE;(CeRCP zX23cx!c5XX+(3-d{1n;nW23AO^p?+>XG}nzO4YV)Z8lF59zI&PaPmE_-=u6WCdbM? zTAD-|Ks4)aaU_?EBUQTUXa~JU@VgDpY1=o|KNwj$&xg|-I%8sVVn(HcG}qMYlWFC< zxZne|`c-Xto7is_G!y&>^JgVhgNRf);FMT(S7x=4!)lr0+~#CuEb24pN8)|c`IdCO z4?FgIoo;2nOWfmfkF4ChYRuf|C(FCu+fxR69VQ0r1Z1ZVbA(^$#`CgId_S)bI1p)i z&})-6ccKbIs+26_H$AAm6CNg&+gR4{`IFyH*2qDBcoUBsond{WsdinRI5Slt z_%Af;!!6ne_4w~a8fQO#AZPd7`!RZP56&7SN5rFf@;vf8EM{0)V-^!(cm);nEQq2Se0RER72dsDp8NJxvS&tlC1nJZlvOL5 z*8-DYn;u!Z#2Q0+{I(Ac9=uuqW*?H@9u955A>q9h~k)5kx zO>fd`L}h5Ee(G-ew<<$-50g?aB|w1L8+@!PkC@&waUQjR6&-?~ktWm4ldd!M?@R_1 zW@x~DANq_eBRW}TTx(@E8}^!t)=53s$(YfT8_XEZI>+-w3feZ!F0`*(bM99fG*Qxd zKTX4XW+*A5ocP{ZZl9s%XAd=aZpI&KjN9*Uu0UiM&Eg(3Ph@ zqW((R7l#_Y6%{?M>KaUd4!9F%ccfSfA{sDFI`%6X^7~$YTT~X-Pd^IY=o=6lSn@2A z=ej@8>szi-z7yN_*+g7vL}IrySPAsWYGCCtxA`}e7_IQ5q`P#+ZGUQgrF!I1eT63W zzh6kQyf1T{Pb1$#6A|5%ySaF?`D1fln`?E- zxZAxMLBCxmr^`u85U{75@87u?3?_eczNVLQvFRi0EIMn)x15{Mx5eacJN{yIWh??x zrAm>_>Qt{rvTw^HEdD0+SzCoyVm)&2BvYhyV9wAo!FlZlki^c)H#h;C5XH3(&CKa{Gv799)PzU_e>Nllis(jSgCH=xnk_uTF5zgf-|szsN6 z1M<2p-YuaQ6&JRRUKd}ZW4v*{8@3Y)h#tt7m@JQHh6D`5-cd8owl5v|F3EnbV9;~O zYhY>kxdvHO{+i+R)i64w`YCl{-SwwGug6+2#Hqf2*=I$hs~a-Qm9yy+1-*8@mmXno zBeFn)8X-EbnetArjJS`j!E(gq{x(a zg%ltm zwXIq^OhWx+J3~^e=$PB-X0)2OotM0d$+Nsj!#Vnfb=m!&lU+pHDJgCJvKP~vADiCK zWnSpMtHk7O9#jq=tUk?@_h5ic3s&XZK>|m2VaBHxr0P7K*!UBK<;g9p#X|PnnGj!f zwZ#WtQAY>V6C6&8j}MkFvPyIy!nMm8`EN{Lj1aBTB<9yen945>cF$M^r!wzid9Ypc z6W>ChXQnQkZ!RJ_t2*D6LfGb=PD&RNp$_5xHl7Ej7rn#5MzKU8at-rZGQP)2xjLnZ z3DB9{IDcg)+lx7Zu>(1baFnAx_up`N7n$&?>w88G(_KzjgFRXNXZt=9UaTHq2@*1dH>cuF>6b$+Zt4}eVBA}G+Nn3_q^_D62%m`5M5}m&^dcH(HpXa z+;+>JVS0Gc)B0x32$VGI)TZ}5K97*bI>pH|bJf*l6un`|w3j$R!PI=NbV&uOdb7zK zB1J~mGdV%TZ)G3D4FlOIu{#fF>K&P|mw%@~) zH>+BAbYMOy(?&W|rq#Xlv_22ZjpoXv3$C5f}|fDn_QGzSk-LrPHsGV&k4RLc2W+Gd)M?B3LGr zNBV6bvE1@Q>tP#ZlE<92i9Xl!;;35bU$qU+&r5Tk(446TO@9B|grc0Ay=NW9y2u2i^5ycY#-q)R~4NyueRYr(`V? From 2b973f8995016757261958061c0b7bd8ad0c4cd0 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 28 Jul 2022 15:14:57 +0200 Subject: [PATCH 06/29] Fixed problems with the CF for the CI/CD Workshop --- content/amazon-ec2-spot-cicd-workshop/lab1.md | 4 ++-- .../container/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/lab1.md b/content/amazon-ec2-spot-cicd-workshop/lab1.md index 2799f675..c727e3d1 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab1.md +++ b/content/amazon-ec2-spot-cicd-workshop/lab1.md @@ -69,13 +69,13 @@ aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenki ``` ## Sign-in to Jenkins -The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **spotcicdworkshop** as the Username and the password that you supplied to the CloudFormation template as the password. +The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. {{%expand "Click to reveal detailed instructions" %}} 1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); 2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; 3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; 4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: - 1. Enter in **spotcicdworkshop** as the Username; + 1. Enter in **admin** as the Username; 2. Enter in the password that you supplied to the CloudFormation template as the Password. {{% /expand%}} diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml index 1417dcd9..8a5f9157 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml @@ -7,7 +7,7 @@ services: user: root build: . environment: - JENKINS_ADMIN_ID: spotcicdworkshop + JENKINS_ADMIN_ID: admin JENKINS_ADMIN_PASSWORD: JenkinsAdminPassword volumes: - /jenkins_home:/var/jenkins_home From d51ffd886100737ef68634de129ca1e6075834bc Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 28 Jul 2022 15:32:39 +0200 Subject: [PATCH 07/29] Fixed problems with the CF for the CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index bf7b9e02..ffa875c0 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -848,6 +848,7 @@ Resources: curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose # Download files from GitHub + mkdir -p /jenkins_home/jobs/ mkdir -p /usr/share/jenkins cd /usr/share/jenkins wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml From fce5f5bea1912cae5c76271a052575e881341391 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 28 Jul 2022 16:03:57 +0200 Subject: [PATCH 08/29] Fixed problems with the CF for the CI/CD Workshop --- workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt index 604af92b..3e0c706a 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -5,7 +5,7 @@ cloudbees-folder:6.729.v2b_9d1a_74d673 configuration-as-code:1512.vb_79d418d5fc8 credentials-binding:523.vd859a_4b_122e6 email-ext:2.90 -git:4.11.3 +git:4.11.4 github-branch-source:1677.v731f745ea_0cf gradle:1.39.4 ldap:2.10 From 16f4d6096df7b62ec4c96fdec8519c8e7ad02c4e Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 1 Aug 2022 12:28:55 +0200 Subject: [PATCH 09/29] Re-structure labs for CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop/_index.md | 2 +- content/amazon-ec2-spot-cicd-workshop/clea.md | 2 +- .../jenkins-asg/_index.md | 9 + .../{lab3.md => jenkins-asg/lab1.md} | 135 ++++++++- .../{lab2/_index.md => jenkins-asg/lab2.md} | 2 +- .../jenkins-ecs/_index.md | 10 + .../{ => jenkins-ecs}/lab4.md | 4 +- content/amazon-ec2-spot-cicd-workshop/lab1.md | 130 --------- .../lab2/clfn.md | 262 ------------------ .../lab2/lamb.md | 110 -------- content/amazon-ec2-spot-cicd-workshop/prep.md | 3 +- 11 files changed, 154 insertions(+), 515 deletions(-) create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md rename content/amazon-ec2-spot-cicd-workshop/{lab3.md => jenkins-asg/lab1.md} (50%) rename content/amazon-ec2-spot-cicd-workshop/{lab2/_index.md => jenkins-asg/lab2.md} (99%) create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/_index.md rename content/amazon-ec2-spot-cicd-workshop/{ => jenkins-ecs}/lab4.md (99%) delete mode 100644 content/amazon-ec2-spot-cicd-workshop/lab1.md delete mode 100644 content/amazon-ec2-spot-cicd-workshop/lab2/clfn.md delete mode 100644 content/amazon-ec2-spot-cicd-workshop/lab2/lamb.md diff --git a/content/amazon-ec2-spot-cicd-workshop/_index.md b/content/amazon-ec2-spot-cicd-workshop/_index.md index e3b47bc5..716737c5 100644 --- a/content/amazon-ec2-spot-cicd-workshop/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/_index.md @@ -2,7 +2,7 @@ title: "CI/CD and Test Workloads with EC2 Spot Instances" menuTitle: "CI/CD and Test Workloads" date: 2019-02-19T02:02:35 -weight: 80 +weight: 1 pre: "8. " --- diff --git a/content/amazon-ec2-spot-cicd-workshop/clea.md b/content/amazon-ec2-spot-cicd-workshop/clea.md index 0f41dc5c..e772baf6 100644 --- a/content/amazon-ec2-spot-cicd-workshop/clea.md +++ b/content/amazon-ec2-spot-cicd-workshop/clea.md @@ -1,6 +1,6 @@ +++ title = "Workshop Cleanup" -weight = 60 +weight = 1000 +++ Congratulations, you have completed this workshop! Your next challenge is to remove all of the resources that were provisioned in your account so as to ensure that no additional cost can be incurred. Please note that the steps below should be implemented in order - some later steps have dependencies on earlier ones! diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md new file mode 100644 index 00000000..91ca6650 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md @@ -0,0 +1,9 @@ ++++ +title = "Jenkins with Amazon EC2 Auto Scaling Groups" +weight = 100 ++++ +By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: +* When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and +* The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. + +To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/lab3.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md similarity index 50% rename from content/amazon-ec2-spot-cicd-workshop/lab3.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md index 1bab0f1a..e0828eda 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab3.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md @@ -1,7 +1,133 @@ +++ -title = "Lab 3: Externalise state data to add resiliency to Jenkins" -weight = 40 +title = "Jenkins with Amazon EC2 Auto Scaling Groups" +weight = 110 +++ +By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: +* When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and +* The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. + +To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. + +## Setting Up environment variables + +```bash +export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); +export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); +export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); +export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); +export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); +export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +``` + +## Provision an Auto Scaling Group for your build agents +Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. + +First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: + +```bash +cat < ~/asg-policy.json +{ + "LaunchTemplate":{ + "LaunchTemplateSpecification":{ + "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", + "Version":"1" + }, + "Overrides":[ + { + "InstanceType":"t2.large" + }, + { + "InstanceType":"t3.large" + }, + { + "InstanceType":"m4.large" + }, + { + "InstanceType":"m5.large" + }, + { + "InstanceType":"c5.large" + }, + { + "InstanceType":"c4.large" + } + ] + }, + "InstancesDistribution":{ + "OnDemandBaseCapacity": 0, + "OnDemandPercentageAboveBaseCapacity": 0, + "SpotAllocationStrategy":"capacity-optimized" + } +} +EoF +``` + +Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. + +```bash +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +``` + +## Sign-in to Jenkins +The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. +{{%expand "Click to reveal detailed instructions" %}} +1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); +2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; +3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; +4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: + 1. Enter in **admin** as the Username; + 2. Enter in the password that you supplied to the CloudFormation template as the Password. +{{% /expand%}} + +## Configure the EC2 Fleet Jenkins plugin +The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). + +When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. + +{{%expand "Click to reveal detailed instructions" %}} +1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; +2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; +3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; +4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; +6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; +7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: + 1. Change the Kind to **SSH Username with private key**; + 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; + 3. At the Username field, enter **ec2-user**; + 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the BEGIN RSA PRIVATE KEY and END RSA PRIVATE KEY fields; + 5. Click on the **Add** button. +8. Select the ec2-user option from the Credentials dropdown; +9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; +10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); +11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; +12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; +13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); +14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); +15. Finally, click on the **Save** button. + +Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. +{{% /expand%}} + +## Configure a build job to use the new Spot instance(s) +As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. + +{{%expand "Click to reveal detailed instructions" %}} +1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: + 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; + 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); + 3. Click on the **Save** button towards the bottom of the screen. +{{% /expand%}} + +## Test Spot Builds and Scale-out +Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. + +1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; +2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; +3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; +4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). + +----- + You're now using Spot instances for your code builds and for the environments that are built out for testing – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by a Spot Fleet. ## OBTAIN THE RELEVANT INFORMATION FOR CLOUDFORMATION FOR THIS LAB @@ -83,7 +209,4 @@ Once you've verified that your Spot Fleet is self-healing, you no longer have an {{%expand "Click to reveal detailed instructions" %}} 1. Remain at the EC2 Instances screen and search for the EC2 instance with the Name tag of **Jenkins Master (On-demand)**. Right-click on the one instance that should come up and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. -{{% /expand%}} - -## PROCEED TO LAB 4 -Once your on-demand Jenkins instance has been terminated, you may proceed with [Lab 4](/amazon-ec2-spot-cicd-workshop/lab4.html). +{{% /expand%}} \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/lab2/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md similarity index 99% rename from content/amazon-ec2-spot-cicd-workshop/lab2/_index.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md index f84e1da7..9d092eae 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab2/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md @@ -1,6 +1,6 @@ +++ title = "Lab 2: Deploy testing environments using Spot & Launch Templates" -weight = 30 +weight = 120 +++ Now that you are carrying out your software builds on Spot instances, the next step is to build out a CI/CD pipeline to have your software deployed to a test environment also running on Spot instances. Pipelines within Jenkins are defined using another plugin, and you'll define pipeline steps to both deploy and terminate your testing environment. The pipeline steps that you define in this lab will call a Lambda function, which in turn deploys a CloudFormation template to brings up your testing environment, and deletes the template to tear it down. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/_index.md new file mode 100644 index 00000000..e4215a89 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/_index.md @@ -0,0 +1,10 @@ ++++ +title = "Jenkins on ECS" +weight = 200 ++++ +You’ve now got a scalable solution using nothing but Spot instances for your CICD systems, your build agents and your test environments – however, you still have some inefficiencies with this setup: + +* Your Jenkins master utilizes a relatively low percentage of the CPU resources on the instance types that Jenkins is running on; and +* You still have at least one Jenkins build agent running at all times; + +These inefficiencies can be addressed by moving your solution to a container environment that continues to utilize Spot instances. This lab will see you configure the ECS cluster resources that were created by the initial CloudFormation template and migrate your Jenkins master and agents to this cluster. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/lab4.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md similarity index 99% rename from content/amazon-ec2-spot-cicd-workshop/lab4.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md index 32bccd8e..cc28c1ea 100644 --- a/content/amazon-ec2-spot-cicd-workshop/lab4.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md @@ -1,6 +1,6 @@ +++ -title = "Lab 4: Using containers backed by Spot instance in Auto Scaling Groups" -weight = 50 +title = "Jenkins on ECS" +weight = 210 +++ You’ve now got a scalable solution using nothing but Spot instances for your CICD systems, your build agents and your test environments – however, you still have some inefficiencies with this setup: diff --git a/content/amazon-ec2-spot-cicd-workshop/lab1.md b/content/amazon-ec2-spot-cicd-workshop/lab1.md deleted file mode 100644 index c727e3d1..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/lab1.md +++ /dev/null @@ -1,130 +0,0 @@ -+++ -title = "Lab 1: Reduce the cost of builds using Amazon EC2 Spot Fleet" -weight = 20 -+++ -By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: -* When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and -* The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. - -To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. - -## Setting Up environment variables - -```bash -export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); -export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); -export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); -export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); -export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); -export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); -``` - -## Provision an Auto Scaling Group for your build agents -Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. - -First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: - -```bash -cat < ~/asg-policy.json -{ - "LaunchTemplate":{ - "LaunchTemplateSpecification":{ - "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", - "Version":"1" - }, - "Overrides":[ - { - "InstanceType":"t2.large" - }, - { - "InstanceType":"t3.large" - }, - { - "InstanceType":"m4.large" - }, - { - "InstanceType":"m5.large" - }, - { - "InstanceType":"c5.large" - }, - { - "InstanceType":"c4.large" - } - ] - }, - "InstancesDistribution":{ - "OnDemandBaseCapacity": 0, - "OnDemandPercentageAboveBaseCapacity": 0, - "SpotAllocationStrategy":"capacity-optimized" - } -} -EoF -``` - -Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. - -```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json -``` - -## Sign-in to Jenkins -The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); -2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; -3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; -4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: - 1. Enter in **admin** as the Username; - 2. Enter in the password that you supplied to the CloudFormation template as the Password. -{{% /expand%}} - -## Configure the EC2 Fleet Jenkins plugin -The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). - -When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. - -{{%expand "Click to reveal detailed instructions" %}} -1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; -2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; -3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; -4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; -6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; -7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: - 1. Change the Kind to **SSH Username with private key**; - 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; - 3. At the Username field, enter **ec2-user**; - 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the BEGIN RSA PRIVATE KEY and END RSA PRIVATE KEY fields; - 5. Click on the **Add** button. -8. Select the ec2-user option from the Credentials dropdown; -9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; -10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); -11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; -12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; -13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); -14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); -15. Finally, click on the **Save** button. - -Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. -{{% /expand%}} - -## Configure a build job to use the new Spot instance(s) -As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: - 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; - 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); - 3. Click on the **Save** button towards the bottom of the screen. -{{% /expand%}} - -## Test Spot Builds and Scale-out -Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. - -1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; -2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; -3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; -4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). - -## Proceed to lab 2 -Once you've verified that builds are succeeding and that your ASG is capable of scaling out to handle queued build jobs, you may proceed with [Lab 2](/amazon-ec2-spot-cicd-workshop/lab2.html). diff --git a/content/amazon-ec2-spot-cicd-workshop/lab2/clfn.md b/content/amazon-ec2-spot-cicd-workshop/lab2/clfn.md deleted file mode 100644 index ff908867..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/lab2/clfn.md +++ /dev/null @@ -1,262 +0,0 @@ -+++ -title = "Code snippet: The Test Environment CloudFormation template" -weight = 10 -+++ -Below is a sanitized version of the CloudFormation template that will be used to launch your test environments using an Amazon EC2 Spot Fleet provisioned from an EC2 Launch Template: -```yaml -AWSTemplateFormatVersion: '2010-09-09' - -Description: A CloudFormation template that will deploy a test environment as a part of the Amazon EC2 Spot CI/CD Workshop. This template is provided as-is under a modified MIT license - please see https://github.com/aws-samples/amazon-ec2-spot-cicd-workshop/blob/master/LICENSE - -Parameters: - KeyPair: - Description: The Key Pair created in Step 3 of the Preparation Lab - Type: AWS::EC2::KeyPair::KeyName - - CurrentIP: - Description: The IP address supplied by the workshop participant when provisioning the Master CloudFormation template for this workshop. - Type: String - - AMILookupLambdaFunctionARN: - Description: The ARN correspoding to the AMI Lookup Lambda Function created by the Master CloudFormation template. - Type: String - - DeploymentArtifactsS3Bucket: - Description: The S3 bucket created by the Master CloudFormation template to host deployment artifacts. - Type: String - - VPC: - Description: The VPC created by the Master CloudFormation template, where the test environment will be provisioned. - Type: String - - SubnetA: - Description: Subnet A created by the Master CloudFormation template, one of three which will be used by the test environment. - Type: String - - SubnetB: - Description: Subnet B created by the Master CloudFormation template, one of three which will be used by the test environment. - Type: String - - SubnetC: - Description: Subnet C created by the Master CloudFormation template, one of three which will be used by the test environment. - Type: String - - -Resources: - - IAMRoleGameOfLife: - Type: AWS::IAM::Role - # DependsOn: None - # DependedOn: InstanceProfileGameOfLife - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: Test - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: s3:GetObject - Resource: !Sub "arn:aws:s3:::${DeploymentArtifactsS3Bucket}/*" - - InstanceProfileGameOfLife: - Type: AWS::IAM::InstanceProfile - DependsOn: IAMRoleGameOfLife - # DependedOn: GameOfLifeLaunchTemplate - Properties: - Path: '/' - Roles: - - !Ref IAMRoleGameOfLife - - AMILookupCustomResource: # A custom resource that provides the latest Amazon Linux AMI via AMILookupCustomResource.Id - Type: Custom::AMILookup - # DependsOn: - # DependedOn: - Properties: - Architecture: HVM64 - Region: !Ref AWS::Region - ServiceToken: !Ref AMILookupLambdaFunctionARN - - SecurityGroupGameOfLifeALB: # A Security Group that allows ingress access for HTTP on ALBs and used to access the Game of Life test environment - Type: AWS::EC2::SecurityGroup - # DependsOn: None - # DependedOn: None - Properties: - GroupName: Spot CICD Workshop Game of Life ALB Security Group - GroupDescription: A Security Group that allows ingress access for HTTP on ALBs and used to access the Game of Life test environment - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 80 - ToPort: 80 - CidrIp: 0.0.0.0/0 - VpcId: !Ref VPC - - SecurityGroupGameOfLifeEC2: # A Security Group that allows ingress access for SSH and the default port that the Game of Life test application will run on - Type: AWS::EC2::SecurityGroup - DependsOn: SecurityGroupGameOfLifeALB - # DependedOn: JenkinsOnDemandEC2Instance - Properties: - GroupName: Spot CICD Workshop Game of Life EC2 Security Group - GroupDescription: A Security Group that allows ingress access for SSH and the default port that a Jenkins Master will run on - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: !Ref CurrentIP - - IpProtocol: tcp - FromPort: 8080 - ToPort: 8080 - CidrIp: !Ref CurrentIP - - IpProtocol: tcp - FromPort: 8080 - ToPort: 8080 - SourceSecurityGroupId: !Ref SecurityGroupGameOfLifeALB - VpcId: !Ref VPC - - GameOfLifeALB: # This is the Application Load Balancer that resides in front of the Game of Life test environment and is responsible for port-mapping requests from TCP:80 to TCP:8080 - Type: AWS::ElasticLoadBalancingV2::LoadBalancer - DependsOn: SecurityGroupGameOfLifeALB - # DependedOn: GameOfLifeALBListener - Properties: - Name: GameOfLifeALB - Scheme: internet-facing - SecurityGroups: - - !Ref SecurityGroupGameOfLifeALB - Subnets: - - !Ref SubnetA - - !Ref SubnetB - - !Ref SubnetC - - GameOfLifeALBTargetGroup: # This is the Target Group used by the GameOfLifeALB load balancer - Type: AWS::ElasticLoadBalancingV2::TargetGroup - # DependsOn: None - # DependedOn: GameOfLifeALBListener, GameOfLifeALBListenerRule - Properties: - HealthCheckIntervalSeconds: 15 - HealthCheckPath: /gameoflife/ - HealthCheckPort: 8080 - HealthCheckProtocol: HTTP - HealthCheckTimeoutSeconds: 5 - HealthyThresholdCount: 2 - Matcher: - HttpCode: 200 - Name: GameOfLifeALBTargetGroup - Port: 8080 - Protocol: HTTP - UnhealthyThresholdCount: 4 - VpcId: !Ref VPC - - GameOfLifeALBListener: # This is the ALB Listener used to access the Game of Life test environment - Type: AWS::ElasticLoadBalancingV2::Listener - DependsOn: - - GameOfLifeALB - - GameOfLifeALBTargetGroup - # DepenededOn: None - Properties: - DefaultActions: - - Type: forward - TargetGroupArn: !Ref GameOfLifeALBTargetGroup - LoadBalancerArn: !Ref GameOfLifeALB - Port: 80 - Protocol: HTTP - - GameOfLifeALBListenerRule: # The ALB Listener rule that forwards all traffic destined for the Game of Life test environment to the appropriate Target Group - Type: AWS::ElasticLoadBalancingV2::ListenerRule - DependsOn: - - GameOfLifeALBListener - - GameOfLifeALBTargetGroup - # DependedOn: None - Properties: - Actions: - - Type: forward - TargetGroupArn: !Ref GameOfLifeALBTargetGroup - Conditions: - - Field: path-pattern - Values: - - "/*" - ListenerArn: !Ref GameOfLifeALBListener - Priority: 1 - - GameOfLifeLaunchTemplate: # The Launch Template that will be used to deploy the test environment - Type: AWS::EC2::LaunchTemplate - DependsOn: SecurityGroupGameOfLifeEC2 - # DependedOn: None - Properties: - LaunchTemplateName: GameOfLifeLaunchTemplate - LaunchTemplateData: - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - DeleteOnTermination: 'true' - VolumeSize: 8 - VolumeType: gp2 - IamInstanceProfile: - Name: !Ref InstanceProfileGameOfLife - ImageId: !GetAtt AMILookupCustomResource.Id - InstanceType: t3.micro - KeyName: !Ref KeyPair - SecurityGroupIds: - - !Ref SecurityGroupGameOfLifeEC2 - TagSpecifications: - - ResourceType: instance - Tags: - - Key: Name - Value: Game of Life Test Instance - UserData: - Fn::Base64: !Sub | - #!/bin/bash - # Install all pending updates to the system - yum -y update - # Install Java 8 & Apache Tomcat 8 - yum -y install java-1.8.0-openjdk tomcat8 - # Download the Game of Life build artifact to Tomcat's webapps directory - aws s3 cp s3://${DeploymentArtifactsS3Bucket}/gameoflife-web/target/gameoflife.war /usr/share/tomcat8/webapps/gameoflife.war - # Start the Tomcat8 Service - service tomcat8 start - - GameOfLifeSpotFleet: # The Spot Fleet definition that will launch the EC2 instances that will comprise the test environment - Type: AWS::EC2::SpotFleet - DependsOn: GameOfLifeLaunchTemplate - # DependedOn: None - Properties: - SpotFleetRequestConfigData: - AllocationStrategy: diversified - IamFleetRole: !Sub arn:aws:iam::${AWS::AccountId}:role/aws-ec2-spot-fleet-tagging-role - LaunchTemplateConfigs: - - LaunchTemplateSpecification: - LaunchTemplateName: GameOfLifeLaunchTemplate - Version: !GetAtt GameOfLifeLaunchTemplate.LatestVersionNumber - Overrides: - - SubnetId: !Ref SubnetA - - LaunchTemplateSpecification: - LaunchTemplateName: GameOfLifeLaunchTemplate - Version: !GetAtt GameOfLifeLaunchTemplate.LatestVersionNumber - Overrides: - - SubnetId: !Ref SubnetB - - LaunchTemplateSpecification: - LaunchTemplateName: GameOfLifeLaunchTemplate - Version: !GetAtt GameOfLifeLaunchTemplate.LatestVersionNumber - Overrides: - - SubnetId: !Ref SubnetC - LoadBalancersConfig: - TargetGroupsConfig: - TargetGroups: - - Arn: !Ref GameOfLifeALBTargetGroup - ReplaceUnhealthyInstances: true - TargetCapacity: 2 - Type: maintain - -Outputs: - GameOfLifeDNSName: - Description: The DNS name of the Application Load Balancer that is used to gain access to the Game of Life testing environment. - Value: !GetAtt GameOfLifeALB.DNSName -``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/lab2/lamb.md b/content/amazon-ec2-spot-cicd-workshop/lab2/lamb.md deleted file mode 100644 index 3416ad00..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/lab2/lamb.md +++ /dev/null @@ -1,110 +0,0 @@ -+++ -title = "Code snippet: The SpotCICDWorkshop_ManageTestEnvironment Lambda function" -weight = 20 -+++ -Below is a sanitized version of the SpotCICDWorkshop_ManageTestEnvironment Lambda function that was deployed to your account by the CloudFormation template that you deployed during the Workshop Preparation: -```javascript -'use strict'; -var AWS = require('aws-sdk'); -var actions = { - deploy: function (cfn, ddb, request_payload) { - return new Promise(function (resolve, reject) { - var cfn_params = { - StackName: request_payload.stackName, - Capabilities: [ 'CAPABILITY_IAM' ], - Parameters: [ - { - ParameterKey: 'KeyPair', - ParameterValue: '${SpotCICDWorkshop.KeyPair}' - }, { - ParameterKey: 'CurrentIP', - ParameterValue: '${SpotCICDWorkshop.Current.Ip}' - }, { - ParameterKey: 'AMILookupLambdaFunctionARN', - ParameterValue: '${SpotCICDWorkshop.AMILookupLambdaFunction.Arn}' - }, { - ParameterKey: 'DeploymentArtifactsS3Bucket', - ParameterValue: '${SpotCICDWorkshop.DeploymentArtifactsS3Bucket}' - }, { - ParameterKey: 'VPC', - ParameterValue: '${SpotCICDWorkshop.VPC}' - }, { - ParameterKey: 'SubnetA', - ParameterValue: '${SpotCICDWorkshop.SubnetPublicA}' - }, { - ParameterKey: 'SubnetB', - ParameterValue: '${SpotCICDWorkshop.SubnetPublicB}' - }, { - ParameterKey: 'SubnetC', - ParameterValue: '${SpotCICDWorkshop.SubnetPublicC}' - } - ], - RoleARN: '${SpotCICDWorkshop.IAMRoleTestEnvironmentCloudFormation.Arn}', - TemplateURL: 'https://s3-us-west-2.amazonaws.com/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop_game-of-life.yaml' - }; - cfn.createStack(cfn_params, function(err, cfn_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'StackId: ', cfn_data.StackId); - return new Promise(function (resolve, reject) { - var ddb_params = { - Item: { - 'JobBaseName': { S: request_payload.jobBaseName }, - 'BuildID': { N: request_payload.buildId }, - 'CloudFormationStackID': { S: cfn_data.StackId } - }, - ReturnConsumedCapacity: 'TOTAL', - TableName: '${SpotCICDWorkshop.DynamoDBTestEnvironmentTable}' - }; - ddb.putItem(ddb_params, function(err, ddb_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'Consumed Capacity Units: ', ddb_data.ConsumedCapacity.CapacityUnits); - return resolve(); - }); - }); - }); - }); - }, - terminate: function(cfn, ddb, request_payload) { - return new Promise(function (resolve, reject) { - var ddb_params = { - Key: { - 'JobBaseName': { S: request_payload.jobBaseName }, - 'BuildID': { N: request_payload.buildId } - }, - TableName: '${SpotCICDWorkshop.DynamoDBTestEnvironmentTable}' - }; - ddb.getItem(ddb_params, function(err, ddb_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'CloudFormationStackId: ', ddb_data.Item.CloudFormationStackID.S); - return new Promise(function (resolve, reject) { - var cfn_params = { - StackName: request_payload.stackName, - RoleARN: '${SpotCICDWorkshop.IAMRoleTestEnvironmentCloudFormation.Arn}' - }; - cfn.deleteStack(cfn_params, function(err, cfn_data) { - if (err) { return reject(err); } - return resolve(); - }); - }); - }); - }); - } -}; -exports.handler = function (event, context, callback) { - var p = actions[event.action]; - if (!p) { - return callback('Unknown action'); - } - var msgAction = event.action.toUpperCase() + ' '; - var cfn = new AWS.CloudFormation(); - var ddb = new AWS.DynamoDB(); - console.log('[INFO]', 'Attempting', msgAction); - return p(cfn, ddb, event).then(function (data) { - return callback(null, data); - }).catch(function (err) { - console.log('[ERROR]', JSON.stringify(err)); - return callback(err); - }); -}; - -``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/prep.md b/content/amazon-ec2-spot-cicd-workshop/prep.md index f43b660c..c464378e 100644 --- a/content/amazon-ec2-spot-cicd-workshop/prep.md +++ b/content/amazon-ec2-spot-cicd-workshop/prep.md @@ -40,5 +40,4 @@ Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supp The stack should take around five minutes to deploy. -## PROCEED TO LAB 1 -Once the CloudFormation is in the process of being deployed, you've completed all of the preparation required to start the workshop, you may proceed with [Lab 1](/amazon-ec2-spot-cicd-workshop/lab1.html). +Once the CloudFormation is in the process of being deployed, you've completed all of the preparation required to start the workshop, you may proceed. From dfed6edca778dfa17622e2698beedaf36938d8d0 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 1 Aug 2022 12:32:40 +0200 Subject: [PATCH 10/29] Re-structure labs for CI/CD Workshop --- content/amazon-ec2-spot-cicd-workshop/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/_index.md b/content/amazon-ec2-spot-cicd-workshop/_index.md index 716737c5..e3b47bc5 100644 --- a/content/amazon-ec2-spot-cicd-workshop/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/_index.md @@ -2,7 +2,7 @@ title: "CI/CD and Test Workloads with EC2 Spot Instances" menuTitle: "CI/CD and Test Workloads" date: 2019-02-19T02:02:35 -weight: 1 +weight: 80 pre: "8. " --- From 0b782dea3198a5cb6cd5ec71163858bbd7a441df Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 1 Aug 2022 12:45:57 +0200 Subject: [PATCH 11/29] Split instructions for Jenkins on ASG for CI/CD Workshop --- .../jenkins-asg/_index.md | 13 +- .../jenkins-asg/asg.md | 51 +++++ .../jenkins-asg/configure.md | 40 ++++ .../jenkins-asg/lab1.md | 212 ------------------ .../jenkins-asg/test-build.md | 18 ++ .../jenkins-asg/test-persistence.md | 76 +++++++ content/amazon-ec2-spot-cicd-workshop/prep.md | 6 +- 7 files changed, 199 insertions(+), 217 deletions(-) create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md delete mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md index 91ca6650..97cfd256 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md @@ -6,4 +6,15 @@ By default, all job builds will be executed on the same instance that Jenkins is * When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and * The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. -To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. \ No newline at end of file +To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. + +## Setting Up environment variables + +```bash +export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); +export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); +export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); +export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); +export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); +export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md new file mode 100644 index 00000000..9fb154c4 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -0,0 +1,51 @@ ++++ +title = "Provision an Auto Scaling Group" +weight = 110 ++++ +Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. + +First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: + +```bash +cat < ~/asg-policy.json +{ + "LaunchTemplate":{ + "LaunchTemplateSpecification":{ + "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", + "Version":"1" + }, + "Overrides":[ + { + "InstanceType":"t2.large" + }, + { + "InstanceType":"t3.large" + }, + { + "InstanceType":"m4.large" + }, + { + "InstanceType":"m5.large" + }, + { + "InstanceType":"c5.large" + }, + { + "InstanceType":"c4.large" + } + ] + }, + "InstancesDistribution":{ + "OnDemandBaseCapacity": 0, + "OnDemandPercentageAboveBaseCapacity": 0, + "SpotAllocationStrategy":"capacity-optimized" + } +} +EoF +``` + +Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. + +```bash +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md new file mode 100644 index 00000000..ee5a3475 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md @@ -0,0 +1,40 @@ ++++ +title = "Configure the EC2 Fleet Jenkins Plugin" +weight = 115 ++++ +## Sign-in to Jenkins +The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. + +1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); +2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; +3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; +4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: + 1. Enter in **admin** as the Username; + 2. Enter in the password that you supplied to the CloudFormation template as the Password. + +## Configure the EC2 Fleet Jenkins plugin +The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). + +When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. + +1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; +2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; +3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; +4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; +6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; +7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: + 1. Change the Kind to **SSH Username with private key**; + 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; + 3. At the Username field, enter **ec2-user**; + 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the BEGIN RSA PRIVATE KEY and END RSA PRIVATE KEY fields; + 5. Click on the **Add** button. +8. Select the ec2-user option from the Credentials dropdown; +9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; +10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); +11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; +12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; +13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); +14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); +15. Finally, click on the **Save** button. + +Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md deleted file mode 100644 index e0828eda..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab1.md +++ /dev/null @@ -1,212 +0,0 @@ -+++ -title = "Jenkins with Amazon EC2 Auto Scaling Groups" -weight = 110 -+++ -By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: -* When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and -* The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. - -To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. - -## Setting Up environment variables - -```bash -export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); -export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); -export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); -export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); -export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); -export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); -``` - -## Provision an Auto Scaling Group for your build agents -Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. - -First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: - -```bash -cat < ~/asg-policy.json -{ - "LaunchTemplate":{ - "LaunchTemplateSpecification":{ - "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", - "Version":"1" - }, - "Overrides":[ - { - "InstanceType":"t2.large" - }, - { - "InstanceType":"t3.large" - }, - { - "InstanceType":"m4.large" - }, - { - "InstanceType":"m5.large" - }, - { - "InstanceType":"c5.large" - }, - { - "InstanceType":"c4.large" - } - ] - }, - "InstancesDistribution":{ - "OnDemandBaseCapacity": 0, - "OnDemandPercentageAboveBaseCapacity": 0, - "SpotAllocationStrategy":"capacity-optimized" - } -} -EoF -``` - -Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. - -```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json -``` - -## Sign-in to Jenkins -The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); -2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; -3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; -4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: - 1. Enter in **admin** as the Username; - 2. Enter in the password that you supplied to the CloudFormation template as the Password. -{{% /expand%}} - -## Configure the EC2 Fleet Jenkins plugin -The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). - -When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. - -{{%expand "Click to reveal detailed instructions" %}} -1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; -2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; -3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; -4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; -6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; -7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: - 1. Change the Kind to **SSH Username with private key**; - 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; - 3. At the Username field, enter **ec2-user**; - 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the BEGIN RSA PRIVATE KEY and END RSA PRIVATE KEY fields; - 5. Click on the **Add** button. -8. Select the ec2-user option from the Credentials dropdown; -9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; -10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); -11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; -12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; -13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); -14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); -15. Finally, click on the **Save** button. - -Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. -{{% /expand%}} - -## Configure a build job to use the new Spot instance(s) -As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: - 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; - 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); - 3. Click on the **Save** button towards the bottom of the screen. -{{% /expand%}} - -## Test Spot Builds and Scale-out -Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. - -1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; -2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; -3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; -4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). - ------ - -You're now using Spot instances for your code builds and for the environments that are built out for testing – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by a Spot Fleet. - -## OBTAIN THE RELEVANT INFORMATION FOR CLOUDFORMATION FOR THIS LAB -As with the previous labs, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab (in order to allow us to focus on the aspects of the workshop that directly apply to EC2 Spot). You will need to determine and make a note of what the **EFS Filesystem ID** is. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#)); -2. Click on the checkbox associated with the **SpotCICDWorkshop** stack; -3. From the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console, make note of values associated with the **EFSFileSystemID** key. -{{% /expand%}} - -## COPY THE CONTENTS OF JENKINS_HOME TO YOUR EFS FILE SYSTEM -In order to copy the contents of the JENKINS_HOME directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. - -Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/var/lib/jenkins** (this is the JENKINS_HOME directory) across to the root of your EFS file system. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); -2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; -3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity; -4. Mount the EFS file system that was created by the CloudFormation template at the /mnt mountpoint by entering the following command, replacing %FILE-SYSTEM-ID% with the EFSFileSystemID noted above: - - ```bash -sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ -$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ -.%FILE-SYSTEM-ID%.efs.eu-west-1.amazonaws.com:/ /mnt -``` -5. Stop the Jenkins service by entering the following command: - - ```bash -sudo service jenkins stop -``` - -6. The content of JENKINS_HOME is stored under /var/lib/jenkins – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): - - ```bash -sudo chown jenkins:jenkins /mnt -sudo cp -rpv /var/lib/jenkins/* /mnt -``` -{{% /expand%}} - -## PROVISION AN EC2 SPOT FLEET FOR YOUR NEW JENKINS HOST -The Spot Fleet that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents - though this time you'll be a bit more aggressive with the bid price to ensure that you see an overall saving over what you would have spent on an on-demand t3.medium instance. Additionally, you'll configure this Spot Fleet so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. - -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Click on the **Request Spot Instances** button; -3. At the first screen of the Spot instance launch wizard: - 1. Under the Tell us your application or task need heading, switch to the **Load balancing workloads** option; - 2. In the Configure your instances section, select the **JenkinsMasterLaunchTemplate** template from the Launch template dropdown. Change the Network to be the **Spot CICD Workshop VPC**. After making this selection, enable the check boxes for all three Availability Zones and then select the **Amazon EC2 Spot CICD Workshop Public Subnet** associated with each availability zone as the subnet to launch instances in; - 3. At the Tell us how much capacity you need section, keep the Total target capacity at **1** instance and the Optional On-Demand portion set to **0**, and then tick the **Maintain target capacity** checkbox. Once selected, leave the Interruption behavior set to **Terminate**; - 4. Again, you'll override the recommendations made by the console, so clear the tick from **Apply recommendations** checkbox. Click on the **Remove** links associated with the all of the instance types initially defined to remove them from the fleet configuration. Then click on the **Select instance types** button and add the **m3.large**, **m4.large**, **t2.medium** and **t3.medium** instance types to the fleet definition. Once the checkboxes for the required instance types have been ticked, click on the **Select** button. Once you have the four desired instance types listed in the fleet request, select the **Lowest Price** Fleet allocation strategy (since we’re interested in keeping cost to an absolute minimum for this use case, and it makes little sense to diversify a single instance across any number of instance pools); - 5. At the Additional request details section, remove the tick from the **Apply defaults** checkbox. As you want to keep the cost of running your Jenkins server below that for which you're currently paying, select the **Set your max price (per instance/hour)** option, and set the price to be the on-demand price of a t3.medium instance in the Ireland region, which is **$0.0456**. In order to ensure that the server can receive HTTP requests from the Application Load Balancer you've been using, tick the checkbox labelled **Receive traffic from one or more load balancers** and from the Target groups dropdown, select **JenkinsMasterEC2TargetGroup**. - 6. Review the Your fleet request as a glance section - it should indicate that your Fleet strength is strong as a result of being able to draw instances from 12 instance pools, and your Estimated price should indicate that you're expecting to make a 73% saving compared to the cost of equivalent on-demand resources; - 7. Lastly, click on the **Launch** button. - -## VERIFY THAT YOUR EC2 SPOT INSTANCE IS ATTACHED TO YOUR ALB TARGET GROUP -After a few moments, your Spot instance will start up and should attach itself the the Target Group being used by your Application Load Balancer. Determine if this registration has completed successfully and when it has done so, access Jenkins through your Load Balancer and fire off a build of Apache PDFBox to ensure that everything is still working as expected. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Target Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#TargetGroups)); -2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the fleet that you just launched. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete after placing your Spot Fleet request. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; -3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. -{{% /expand%}} - -## TEST THE SELF-HEALING ARCHITECTURE BY TERMINATING THE RUNNING EC2 SPOT INSTANCE -What happens when the market price for the capacity pool that your EC2 Spot instance is running in goes over your $0.0456 per instance-hour bid price? - -When this happens, your EC2 Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, Spot Fleet will observe that there are no running instances in the fleet and because the desired capacity is one, it will attempt to launch a replacement instance in a capacity pool that is still below the per instance-hour bid price that you set (this is why diversification across many capacity pool is a best practice). If such an instance is available, a replacement instance will be launched and bootstrapped in exactly the same manner as your original instance. - -Test this out by terminating the Spot instance with the **Jenkins Master (Spot)** name tag and verifying that a replacement instance comes up to take its place. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Instances** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances)); -2. Search for the EC2 instance with the Name tag of **Jenkins Master (Spot)**, right-click on it and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. Within moments of terminating this instance, your Spot Fleet registration should detect that the number of running instances is below the target that you’ve defined (that target being one instance) and therefore should launch a new, replacement spot instance. Refresh the list of EC2 instances every few seconds until you see the new instance be allocated (while you're waiting, it might also be interesting to see what's happening under the History tab of your Spot Fleet). Whilst your Jenkins service will be unavailable until this new spot instance launches and bootstraps, service should automatically be restored in a couple of minutes. -{{% /expand%}} - -## TERMINATE THE ON-DEMAND INSTANCE THAT WAS INITIALLY USED FOR YOUR JENKINS SERVER -Once you've verified that your Spot Fleet is self-healing, you no longer have any need for the On-demand instance. To prevent it from incurring unnecessary cost, it can be terminated. - -{{%expand "Click to reveal detailed instructions" %}} -1. Remain at the EC2 Instances screen and search for the EC2 instance with the Name tag of **Jenkins Master (On-demand)**. Right-click on the one instance that should come up and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. -{{% /expand%}} \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md new file mode 100644 index 00000000..df338b6e --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md @@ -0,0 +1,18 @@ ++++ +title = "Configure a Build Job to use Spot" +weight = 120 ++++ +As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. + +1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: + 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; + 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); + 3. Click on the **Save** button towards the bottom of the screen. + +## Test Spot Builds and Scale-out +Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. + +1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; +2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; +3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; +4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md new file mode 100644 index 00000000..80496fe4 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -0,0 +1,76 @@ ++++ +title = "Configure Persistence Storage when using Spot" +weight = 125 ++++ +You're now using Spot instances for your code builds and for the environments that are built out for testing – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by a Spot Fleet. + +## OBTAIN THE RELEVANT INFORMATION FOR CLOUDFORMATION FOR THIS LAB +As with the previous labs, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab (in order to allow us to focus on the aspects of the workshop that directly apply to EC2 Spot). You will need to determine and make a note of what the **EFS Filesystem ID** is. + +1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#)); +2. Click on the checkbox associated with the **SpotCICDWorkshop** stack; +3. From the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console, make note of values associated with the **EFSFileSystemID** key. + +## COPY THE CONTENTS OF JENKINS_HOME TO YOUR EFS FILE SYSTEM +In order to copy the contents of the JENKINS_HOME directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. + +Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/var/lib/jenkins** (this is the JENKINS_HOME directory) across to the root of your EFS file system. + +1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); +2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; +3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity; +4. Mount the EFS file system that was created by the CloudFormation template at the /mnt mountpoint by entering the following command, replacing %FILE-SYSTEM-ID% with the EFSFileSystemID noted above: + + ```bash +sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ +$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ +.%FILE-SYSTEM-ID%.efs.eu-west-1.amazonaws.com:/ /mnt +``` +5. Stop the Jenkins service by entering the following command: + + ```bash +sudo service jenkins stop +``` + +6. The content of JENKINS_HOME is stored under /var/lib/jenkins – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): + + ```bash +sudo chown jenkins:jenkins /mnt +sudo cp -rpv /var/lib/jenkins/* /mnt +``` + +## PROVISION AN EC2 SPOT FLEET FOR YOUR NEW JENKINS HOST +The Spot Fleet that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents - though this time you'll be a bit more aggressive with the bid price to ensure that you see an overall saving over what you would have spent on an on-demand t3.medium instance. Additionally, you'll configure this Spot Fleet so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. + +1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); +2. Click on the **Request Spot Instances** button; +3. At the first screen of the Spot instance launch wizard: + 1. Under the Tell us your application or task need heading, switch to the **Load balancing workloads** option; + 2. In the Configure your instances section, select the **JenkinsMasterLaunchTemplate** template from the Launch template dropdown. Change the Network to be the **Spot CICD Workshop VPC**. After making this selection, enable the check boxes for all three Availability Zones and then select the **Amazon EC2 Spot CICD Workshop Public Subnet** associated with each availability zone as the subnet to launch instances in; + 3. At the Tell us how much capacity you need section, keep the Total target capacity at **1** instance and the Optional On-Demand portion set to **0**, and then tick the **Maintain target capacity** checkbox. Once selected, leave the Interruption behavior set to **Terminate**; + 4. Again, you'll override the recommendations made by the console, so clear the tick from **Apply recommendations** checkbox. Click on the **Remove** links associated with the all of the instance types initially defined to remove them from the fleet configuration. Then click on the **Select instance types** button and add the **m3.large**, **m4.large**, **t2.medium** and **t3.medium** instance types to the fleet definition. Once the checkboxes for the required instance types have been ticked, click on the **Select** button. Once you have the four desired instance types listed in the fleet request, select the **Lowest Price** Fleet allocation strategy (since we’re interested in keeping cost to an absolute minimum for this use case, and it makes little sense to diversify a single instance across any number of instance pools); + 5. At the Additional request details section, remove the tick from the **Apply defaults** checkbox. As you want to keep the cost of running your Jenkins server below that for which you're currently paying, select the **Set your max price (per instance/hour)** option, and set the price to be the on-demand price of a t3.medium instance in the Ireland region, which is **$0.0456**. In order to ensure that the server can receive HTTP requests from the Application Load Balancer you've been using, tick the checkbox labelled **Receive traffic from one or more load balancers** and from the Target groups dropdown, select **JenkinsMasterEC2TargetGroup**. + 6. Review the Your fleet request as a glance section - it should indicate that your Fleet strength is strong as a result of being able to draw instances from 12 instance pools, and your Estimated price should indicate that you're expecting to make a 73% saving compared to the cost of equivalent on-demand resources; + 7. Lastly, click on the **Launch** button. + +## VERIFY THAT YOUR EC2 SPOT INSTANCE IS ATTACHED TO YOUR ALB TARGET GROUP +After a few moments, your Spot instance will start up and should attach itself the the Target Group being used by your Application Load Balancer. Determine if this registration has completed successfully and when it has done so, access Jenkins through your Load Balancer and fire off a build of Apache PDFBox to ensure that everything is still working as expected. + +1. Go to the **EC2** console and click on the **Target Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#TargetGroups)); +2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the fleet that you just launched. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete after placing your Spot Fleet request. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; +3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. + +## TEST THE SELF-HEALING ARCHITECTURE BY TERMINATING THE RUNNING EC2 SPOT INSTANCE +What happens when the market price for the capacity pool that your EC2 Spot instance is running in goes over your $0.0456 per instance-hour bid price? + +When this happens, your EC2 Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, Spot Fleet will observe that there are no running instances in the fleet and because the desired capacity is one, it will attempt to launch a replacement instance in a capacity pool that is still below the per instance-hour bid price that you set (this is why diversification across many capacity pool is a best practice). If such an instance is available, a replacement instance will be launched and bootstrapped in exactly the same manner as your original instance. + +Test this out by terminating the Spot instance with the **Jenkins Master (Spot)** name tag and verifying that a replacement instance comes up to take its place. + +1. Go to the **EC2** console and click on the **Instances** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances)); +2. Search for the EC2 instance with the Name tag of **Jenkins Master (Spot)**, right-click on it and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. Within moments of terminating this instance, your Spot Fleet registration should detect that the number of running instances is below the target that you’ve defined (that target being one instance) and therefore should launch a new, replacement spot instance. Refresh the list of EC2 instances every few seconds until you see the new instance be allocated (while you're waiting, it might also be interesting to see what's happening under the History tab of your Spot Fleet). Whilst your Jenkins service will be unavailable until this new spot instance launches and bootstraps, service should automatically be restored in a couple of minutes. + +## TERMINATE THE ON-DEMAND INSTANCE THAT WAS INITIALLY USED FOR YOUR JENKINS SERVER +Once you've verified that your Spot Fleet is self-healing, you no longer have any need for the On-demand instance. To prevent it from incurring unnecessary cost, it can be terminated. + +1. Remain at the EC2 Instances screen and search for the EC2 instance with the Name tag of **Jenkins Master (On-demand)**. Right-click on the one instance that should come up and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/prep.md b/content/amazon-ec2-spot-cicd-workshop/prep.md index c464378e..fea829f9 100644 --- a/content/amazon-ec2-spot-cicd-workshop/prep.md +++ b/content/amazon-ec2-spot-cicd-workshop/prep.md @@ -6,7 +6,7 @@ Before we start presenting content on the different use cases for EC2 Spot insta ## CREATE A NEW EC2 KEY PAIR You will need to access the SSH interfaces of some Linux EC2 instances created in this workshop. To do so in a secure manner, please create a new EC2 key pair with a name of **Spot CICD Workshop Key Pair** in the **EU (Ireland)** region (all activities for this workshop will be carried out in this region). -{{%expand "Click to reveal detailed instructions" %}} + 1. Log in to your AWS Account; 2. Switch to the **EU (Ireland)** region; 3. Provision a new EC2 Key Pair: @@ -14,7 +14,6 @@ You will need to access the SSH interfaces of some Linux EC2 instances created i 2. Click on the **Create Key Pair** button; 3. Enter **Spot CICD Workshop Key Pair** as the Key pair name and click on the **Create** button; 4. Your web browser should download a .pem file – keep this file as it will be required to access the EC2 instances that you create in this workshop. If you're using a Windows system, convert the .pem file to a puTTY .ppk file. If you're not sure how to do this, instructions are available [here](https://aws.amazon.com/premiumsupport/knowledge-center/convert-pem-file-into-ppk/). -{{% /expand%}} ## LAUNCH THE CLOUDFORMATION TEMPLATE So that you can concentrate on the aspects of this workshop that directly relate to Amazon EC2 Spot instances, there is a CloudFormation template that will deploy the base AWS infrastructure needed for all of the labs within the workshop - saving you from having to create things like VPCs, Security Groups, IAM policies and so forth. @@ -23,7 +22,7 @@ Download and deploy the CloudFormation template: [amazon-ec2-spot-cicd-workshop.yaml](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml) Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supply appropriate parameters when prompted. -{{%expand "Click to reveal detailed instructions" %}} + 1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); 2. Click on the **Create Stack** button towards the top of the console; 3. At the Select Template screen, select the **Upload a template file** radio button and choose the CloudFormation template you downloaded before, then click on the **Next** button; @@ -36,7 +35,6 @@ Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supp 7. Finally at the Review screen, verify your settings, mark the **I acknowledge that AWS CloudFormation might create IAM resources with custom names** checkbox and then click on the **Create** button. Wait for the stack to complete provisioning, which should take a couple of minutes. [^1]: It's good security practice to ensure that the web and SSH services being used in this workshop are not accessible to everyone on the Internet. In most cases, limiting access to the /24 CIDR block that you IP address is in provides a reasonable level of access control - but this may still be too restrictive in some corporate IT environments. If you have trouble accessing resources, additional instructions within this lab guide will guide you through what settings need to be manually changed. -{{% /expand%}} The stack should take around five minutes to deploy. From 92fc055c952e34fab3961c7ed21c56a33bcfd0a8 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 1 Aug 2022 18:02:36 +0200 Subject: [PATCH 12/29] Re-structure labs for CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop/_index.md | 10 +++---- .../jenkins-asg/_index.md | 7 ++++- .../jenkins-asg/asg.md | 27 ++++++++++++++----- .../{configure.md => ec2-fleet-jenkins.md} | 20 ++++++++------ .../jenkins-asg/{lab2.md => pipeline.md} | 9 ++----- .../jenkins-asg/test-build.md | 12 +++++---- .../jenkins-asg/test-persistence.md | 20 +++++++------- 7 files changed, 62 insertions(+), 43 deletions(-) rename content/amazon-ec2-spot-cicd-workshop/jenkins-asg/{configure.md => ec2-fleet-jenkins.md} (67%) rename content/amazon-ec2-spot-cicd-workshop/jenkins-asg/{lab2.md => pipeline.md} (97%) diff --git a/content/amazon-ec2-spot-cicd-workshop/_index.md b/content/amazon-ec2-spot-cicd-workshop/_index.md index e3b47bc5..902201a3 100644 --- a/content/amazon-ec2-spot-cicd-workshop/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/_index.md @@ -7,16 +7,14 @@ pre: "8. " --- ## Overview -During this workshop, you'll get hands-on with Amazon EC2 Spot and discover architectural best practices through the lens of DevOps and CI/CD. You'll deploy Jenkins build agents and test environments on Spot instances at a fraction of the cost of on-demand instances. You'll also implement mechanisms to ensure that your CI/CD tooling recovers from spot market events by decoupling application state from your compute resources. Finally, you'll migrate your CI/CD environment to a containered environment to eke out maximum performance and cost efficiency. In addition to covering the ins and outs of Spot, we'll share some of the Spot-based mechanisms used by customers to reduce the cost of their test and production workloads. +During this workshop, you'll get hands-on with Amazon EC2 Spot and discover architectural best practices through the lens of DevOps and CI/CD. You'll deploy Jenkins build agents and test environments on Spot instances at a fraction of the cost of on-demand instances. You'll also implement mechanisms to ensure that your CI/CD tooling recovers from spot market events by decoupling application state from your compute resources. Finally, you'll migrate your CI/CD environment to a containerized environment to eke out maximum performance and cost efficiency. In addition to covering the ins and outs of Spot, we'll share some of the Spot-based mechanisms used by customers to reduce the cost of their test and production workloads. ## Workshop Details This workshop will be broken down into a series of labs that flow on from each other (that is, you must complete each lab in order before proceeding with the next). The lab exercises that will be covered are: * Workshop preparation: Deploy pre-requisite resources through Amazon CloudFormation; -* Lab 1: Reduce the cost of builds using Amazon EC2 Spot Fleet; -* Lab 2: Deploy testing environments using Amazon EC2 Spot, Amazon CloudFormation & Amazon EC2 Launch Templates; -* Lab 3: Externalize state data to add resiliency and reduce cost for your CI/CD tooling; -* Lab 4: Using containers backed by Auto Scaling Groups comprised of both on-demand and Spot instances; +* Jenkins with Amazon EC2 Auto Scaling Groups using Spot; +* Jenkins on ECS using Spot; * Workshop clean up. As a reminder, you should have a laptop device (Windows/OSX/Linux are supported - tablets are not appropriate) with the current version of Google Chrome or Mozilla Firefox installed. You should also have a clean AWS account, with **AdministratorAccess** policy-level access. @@ -26,7 +24,7 @@ This workshop should take between two and three hours to complete, depending on #### Additional considerations when running this workshop in a corporate IT environment If you are running this workshop from a corporate IT environment, contact your Systems Administrator to ensure that you will be able to establish outbound Secure Shell (SSH) connections to an Internet host: -* If you cannot establish SSH connections to Internet hosts (and do not have a suitable workaround), you will not be able to complete Labs 3 & 4; +* If you cannot establish SSH connections to Internet hosts (and do not have a suitable workaround), you will not be able to complete some labs; * If you can establish SSH connections to Internet hosts, obtain from your Systems Administrator the source IP address CIDR block that connections will be established from. If you access the Internet through a transparent proxy server running in your corporate IT environment and this proxy server uses a different source address than where SSH connections come from, additional configuration of AWS Security Groups will need to be carried out. The lab guide will indicate the configuration steps required when appropriate. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md index 97cfd256..b43bc322 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md @@ -9,6 +9,7 @@ By default, all job builds will be executed on the same instance that Jenkins is To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. ## Setting Up environment variables +Let's get started by setting up the following environment variables you'll use in the workshop, the the following commands: ```bash export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); @@ -17,4 +18,8 @@ export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); -``` \ No newline at end of file +``` + +If for some reason the environment variables are cleared, you can run the previous commands again without any problem. + +You may now proceed with the next step. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md index 9fb154c4..88f1d300 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -1,10 +1,17 @@ +++ -title = "Provision an Auto Scaling Group" +title = "Provision an EC2 Auto Scaling Group" weight = 110 +++ -Before configuring the EC2 Fleet Jenkins Plugin, create an Auto Scaling Group (ASG) that will be used by the plugin to perform your application builds. As this is a batch processing use case, remember the best practices for this type of workload - leverage per-second billing (catered for through the use of an Amazon Linux AMI defined in the Launch Template); determine job completion and retry failed jobs (the former is handled by the Jenkins EC2 Fleet plugin); and be instance flexible. +An Auto Scaling group contains a collection of Amazon EC2 Instances that are treated as a logical grouping for the purposes of automatic scaling and management. An Auto Scaling group also enables you to use Amazon EC2 Auto Scaling features such as health check replacements and scaling policies. Both maintaining the number of instances in an Auto Scaling group and automatic scaling are the core functionality of the Amazon EC2 Auto Scaling service. -First, you are going to create the configuration file that will be used to launch the EC2 Fleet. Run the following commands: +Amazon EC2 Auto Scaling helps you ensure that you have the correct number of Amazon EC2 Instances available to handle the load for your application. You can specify the minimum and maximum number of instances, and Amazon EC2 Auto Scaling ensures that your group never goes below or above this size. + +When adopting EC2 Spot Instances, our recommendation is use Auto Scaling groups since it offers a very rich API with benefits like scale-in protection, health checks, lifecycle hooks, rebalance recommendation integration, warm pools and predictive scaling, and many more functionalities that we list below. + +## Launching an Auto Scaling group +For this workshop, you need to create an Auto Scaling group that will be used by the Jenkins plugin to perform your application builds. To apply Spot best practices we will launch a mixed instance Auto Scaling group using Spot. This first step does create a *json* file. The file describes a mixed-instance-policy section with a set of overrides that drive **diversification of Spot Instance pools**. The configuration of the Auto Scaling group does refer to the Launch Template that was created in the previous steps with Cloudformation. + +Run the following command: ```bash cat < ~/asg-policy.json @@ -44,8 +51,16 @@ cat < ~/asg-policy.json EoF ``` -Copy and paste this command to create the EC2 Fleet and export its identifier to an environment variable to later monitor the status of the fleet. +In the override section, choose as many instances that qualify for your application as possible, in this case we selected a group of 6 instance types that meet the “.large” criteria. + +With Auto Scaling groups you can define what is the balance between Spot vs On-Demand Instances that makes sense for your workload. OnDemandBaseCapacity allows you to set an initial capacity of On-Demand Instances to use. After that, any new procured capacity will be a mix of Spot and On-Demand Instances as defined by the OnDemandPercentageAboveBaseCapacity. This time, we configured the Auto Scaling group to launch only Spot instances. + +Additionally, the configuration above, sets the `SpotAllocationStrategy` to `capacity-optimized`. The `capacity-optimized` allocation strategy **allocates instances from the Spot Instance pools with the optimal capacity for the number of instances that are launching**, making use of real-time capacity data and optimizing the selection of used Spot Instances. You can read about the benefits of using `capcity-optimized` in the blog post [Capacity-Optimized Spot Instance allocation in action at Mobileye and Skyscanner](https://aws.amazon.com/blogs/aws/capacity-optimized-spot-instance-allocation-in-action-at-mobileye-and-skyscanner/). + +Let’s create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (to avoid having instances when there's no need), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. ```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 1 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json -``` \ No newline at end of file +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +``` + +You may now proceed with the next step. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md similarity index 67% rename from content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md index ee5a3475..f4635da9 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/configure.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md @@ -2,6 +2,10 @@ title = "Configure the EC2 Fleet Jenkins Plugin" weight = 115 +++ +The [EC2 Fleet Plugin](https://plugins.jenkins.io/ec2-fleet/) launches EC2 Spot or On Demand instances as worker nodes for Jenkins CI server, automatically scaling the capacity with the load. The EC2 FLeet plugin will request EC2 instances when excess jobs are detected. You can configure the plugin to use an Auto Scaling Group to launch instances instead of directly launching them by itself. This gives theh plugin all the benefits from Auto Scaling groups like allocation strategies, configure multiple instance types and availability zones, etc. Moreover, the EC2 Fleet plugin can automatically resubmit failed jobs caused by a Spot interruption. + +To start using this plugin, you need to configure it in Jenkins, so let's do it. + ## Sign-in to Jenkins The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. @@ -19,22 +23,22 @@ When configuring the plugin, think about how you could force build processes to 1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; 2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; -3. You don't need to configure any AWS Credentials as the plugin will use the IAM Role attached to the instance; -4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of EC2 Fleet requests made in the selected region; -6. Select the Request Id of the EC2 Fleet that you created earlier from the Spot Fleet dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; +3. **You don't need to configure any AWS Credentials** as the plugin will use the IAM Role attached to the instance; +4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of Auto Scaling groups in the selected region; +6. Select the Auto Scaling group that you created earlier (`Auto Scaling Group - EC2SpotJenkinsASG`) from the **EC2 Fleet** dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; 7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: 1. Change the Kind to **SSH Username with private key**; 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; 3. At the Username field, enter **ec2-user**; - 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the BEGIN RSA PRIVATE KEY and END RSA PRIVATE KEY fields; + 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the *BEGIN RSA PRIVATE KEY* and *END RSA PRIVATE KEY* fields; 5. Click on the **Add** button. -8. Select the ec2-user option from the Credentials dropdown; +8. Select the **ec2-user** option from the Credentials dropdown; 9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; 10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); 11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; -12. Set the Max Idle Minutes Before Scaledown to **5**. There's no need to keep a build agent running for too much longer than it's required; +12. Set the **Max Idle Minutes Before Scaledown** to **5**. There's no need to keep a build agent running for too much longer than it's required; 13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); -14. Change the Maximum Cluster Size from **1** to **2** (so that you can test fleet scale-out); +14. Change the Maximum Cluster Size from **1** to **5** (so that you can test fleet scale-out); 15. Finally, click on the **Save** button. -Within sixty-seconds, the Jenkins Slave Agent should have been installed on to the Spot instance that was launched by your EC2 fleet; you should see an EC2 instance ID appear underneath the Build Executor Status section on the left side of the Jenkins user interface. Underneath that, you should see that there is a single Build Executor on this host, which is in an idle state. \ No newline at end of file +For now, no instances are going to be launched as there are no pending jobs to run. So, let's configure an existing Jenkins job to use Spot instances. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md similarity index 97% rename from content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md index 9d092eae..432c49e4 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/lab2.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md @@ -1,6 +1,6 @@ +++ -title = "Lab 2: Deploy testing environments using Spot & Launch Templates" -weight = 120 +title = "Configure a Pipeline Job using Spot" +weight = 130 +++ Now that you are carrying out your software builds on Spot instances, the next step is to build out a CI/CD pipeline to have your software deployed to a test environment also running on Spot instances. Pipelines within Jenkins are defined using another plugin, and you'll define pipeline steps to both deploy and terminate your testing environment. The pipeline steps that you define in this lab will call a Lambda function, which in turn deploys a CloudFormation template to brings up your testing environment, and deletes the template to tear it down. @@ -12,12 +12,10 @@ If you view the CloudFormation template that is used to deploy the test environm ## INCREASE THE NUMBER OF BUILD EXECUTORS ON YOUR JENKINS BUILD AGENTS When executing a pipeline job in Jenkins, the pipeline consumes one of the build execution slots. Given that the the software builds themselves are executed using a build agent on a Spot instance, you should configure the Jenkins master server to also have a build executor - for pipeline execution to run on. -{{%expand "Click to reveal detailed instructions" %}} 1. Back at the Jenkins home page, click on the **Manage Jenkins** link on the left side menu again, and then the **Configure System** link; 2. Scroll down to the bottom of the page and under the **Spot Fleet Configuration** section, increase the **Number of Executors** to 3; 3. Click on the **Save** button at the bottom of the screen; 4. In the left column under **Build Executor Status**, you should see the EC2 Instance ID for the instances that make up your build fleet. Click on the Instance ID and then click on the **Delete Agent** link on the left hand side of the screen. When prompted if you're sure about deleting the agent, click on the **Yes** button. Deleting the agent from the host will force Jenkins to re-install it, this time with multiple build executors. -{{% /expand%}} ## RECONFIGURE THE GAME OF LIFE PROJECT The S3 Publisher Plugin should be pre-configured for use (it will use the IAM policy attached to the EC2 Instance Profile used by the Jenkins Master or the build agents to grant access to the S3 bucket where your deployment artifacts will be uploaded to). First up, you'll need to reconfigure the **Game of Life** project to publish it's artifacts to the S3 bucket that was created by the CloudFormation stack deployed at the beginning of this workshop. @@ -100,6 +98,3 @@ Once the deploy stage has been completed successfully: 3. Copy the value associated with the **GameOfLifeDNSName** key into the address bar of a new browser tab and append the **/gameoflife** suffix before hitting the Enter key. Once you've verified that that the Game of Life web application loads, return to Jenkins and click on the **Test** pipeline stage and click on **Proceed** to get the pipeline to process the Terminate stage. After this final stage has executed, go back to the CloudFormation console to verify that the GameOfLife stack is being deleted. - -## PROCEED TO LAB 3 -Once your test environment has been terminated, you may proceed with [Lab 3](/amazon-ec2-spot-cicd-workshop/lab3.html). diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md index df338b6e..ec069eb5 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md @@ -1,5 +1,5 @@ +++ -title = "Configure a Build Job to use Spot" +title = "Configure Build Jobs to use Spot" weight = 120 +++ As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. @@ -12,7 +12,9 @@ As alluded to in the previous section, you'll need to configure your build jobs ## Test Spot Builds and Scale-out Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. -1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, the first of which will be immediately assigned to the Spot instance to be worked on; -2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Spot Fleet has scaled out and build jobs are executing on both Spot instances; -3. After a couple of minutes (typically during the first **Apache Helix** build - around four minutes after you initiate the first build), the EC2 Fleet Status reported to the left of the screen will increment the **target** count to 2, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, a second build instance will appear in the **Build Executor Status**, though this build agent will initially appear to be offline. Once the instance has had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to it via SSH, and it will come online and process the next build job in the queue. Once you have concurrent builds being executed on two Spot instances, you can stop adding build jobs to the build queue; -4. After a period of around a five minutes after your builds have completed, one of the Spot instances should be terminated by the plugin - there's no need to wait for this to happen (take our word for it, but you can verify this later). \ No newline at end of file +1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, this will trigger the EC2 Fleet plugin to start launching build agents using Spot instances; +2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Auto Scaling group has scaled out and build jobs are executing on Spot instances; +3. After around four minutes, the **EC2 Fleet Status** reported to the left of the screen will increment the **target** count to 5, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, five build instances will appear in the **Build Executor Status**, though these build agents will initially appear to be offline. Once the instances have had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to them via SSH, and it will come online and process the next build jobs in the queue. Once you have concurrent builds being executed on the Spot instances, you can stop adding build jobs to the build queue; +4. After a period of around a five minutes, after your builds have completed, the Spot instances should be terminated by the plugin. + +**Note:** If the build agents are keep showing as `offline`, make sure that you've configured correctly the SSH key pair in the credentials section when configuring the EC2 Fleet plugin. You can click on the agent to open the logs and see why it's not coming online. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index 80496fe4..e784bf09 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -2,39 +2,39 @@ title = "Configure Persistence Storage when using Spot" weight = 125 +++ -You're now using Spot instances for your code builds and for the environments that are built out for testing – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by a Spot Fleet. +You're now using Spot instances for your code builds – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by an Auto Scaling group. -## OBTAIN THE RELEVANT INFORMATION FOR CLOUDFORMATION FOR THIS LAB -As with the previous labs, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab (in order to allow us to focus on the aspects of the workshop that directly apply to EC2 Spot). You will need to determine and make a note of what the **EFS Filesystem ID** is. +## Get the EFS Filesystem ID +As with the previous steps, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab. You will need to determine and make a note of what the **EFS Filesystem ID** is. 1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#)); 2. Click on the checkbox associated with the **SpotCICDWorkshop** stack; 3. From the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console, make note of values associated with the **EFSFileSystemID** key. ## COPY THE CONTENTS OF JENKINS_HOME TO YOUR EFS FILE SYSTEM -In order to copy the contents of the JENKINS_HOME directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. +In order to copy the contents of the `JENKINS_HOME` directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. -Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/var/lib/jenkins** (this is the JENKINS_HOME directory) across to the root of your EFS file system. +Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/var/lib/jenkins** (this is the `JENKINS_HOME` directory) across to the root of your EFS file system. 1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); 2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; 3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity; -4. Mount the EFS file system that was created by the CloudFormation template at the /mnt mountpoint by entering the following command, replacing %FILE-SYSTEM-ID% with the EFSFileSystemID noted above: +4. Mount the EFS file system that was created by the CloudFormation template at the /mnt mountpoint by entering the following command, replacing `%FILE-SYSTEM-ID%` with the **EFSFileSystemID** noted above: - ```bash +```bash sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ .%FILE-SYSTEM-ID%.efs.eu-west-1.amazonaws.com:/ /mnt ``` 5. Stop the Jenkins service by entering the following command: - ```bash +```bash sudo service jenkins stop ``` -6. The content of JENKINS_HOME is stored under /var/lib/jenkins – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): +6. The content of `JENKINS_HOME` is stored under /var/lib/jenkins – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): - ```bash +```bash sudo chown jenkins:jenkins /mnt sudo cp -rpv /var/lib/jenkins/* /mnt ``` From ce075f2ec1368a8ce0311d22fb018e8f8d2ef15d Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 1 Aug 2022 18:16:21 +0200 Subject: [PATCH 13/29] Updated the persistence lab for the CI/CD Workshop --- .../jenkins-asg/test-persistence.md | 12 +++--- .../amazon-ec2-spot-cicd-workshop.yaml | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index e784bf09..0e9b20a8 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -14,7 +14,7 @@ As with the previous steps, the CloudFormation stack deployed during your Worksh ## COPY THE CONTENTS OF JENKINS_HOME TO YOUR EFS FILE SYSTEM In order to copy the contents of the `JENKINS_HOME` directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. -Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/var/lib/jenkins** (this is the `JENKINS_HOME` directory) across to the root of your EFS file system. +Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/jenkins_home** (this is the `JENKINS_HOME` directory) across to the root of your EFS file system. 1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); 2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; @@ -26,17 +26,17 @@ sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retr $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ .%FILE-SYSTEM-ID%.efs.eu-west-1.amazonaws.com:/ /mnt ``` -5. Stop the Jenkins service by entering the following command: +5. Stop the Jenkins service by entering the following commands: ```bash -sudo service jenkins stop +cd /usr/share/jenkins +sudo docker-compose stop ``` -6. The content of `JENKINS_HOME` is stored under /var/lib/jenkins – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): +6. The content of `JENKINS_HOME` is stored under /jenkins_home – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): ```bash -sudo chown jenkins:jenkins /mnt -sudo cp -rpv /var/lib/jenkins/* /mnt +sudo cp -rpv /jenkins_home/* /mnt ``` ## PROVISION AN EC2 SPOT FLEET FOR YOUR NEW JENKINS HOST diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index ffa875c0..76734932 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -1011,24 +1011,29 @@ Resources: #!/bin/bash # Install all pending updates to the system yum -y update - # Configure YUM to be able to access official Jenkins RPM packages - wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo - # Import the Jenkins repository public key - rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key - # Configure YUM to be able to access contributed Maven RPM packages - wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo - # Update the release version in the Maven repository configuration for this mainline release of Amazon Linux - sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo - # Install the Java 8 SDK, Git, Jenkins and Maven - yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel git apache-maven - yum -y install jenkins-2.138.4-1.1 --nogpgcheck - # Set the default version of java to run out of the Java 8 SDK path (required by Jenkins) - update-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java - update-alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac - # Mount the Jenkins EFS volume at JENKINS_HOME - mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFSJenkinsHomeVolume}.efs.eu-west-1.amazonaws.com:/ /var/lib/jenkins - # Start the Jenkins service - service jenkins start + # Install Docker + amazon-linux-extras install docker + chkconfig docker on + service docker start + usermod -a -G docker ec2-user + # Install Docker Compose + curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + # Mount the Jenkins EFS volume + mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFSJenkinsHomeVolume}.efs.eu-west-1.amazonaws.com:/ /jenkins_home + # Download files from GitHub + mkdir -p /usr/share/jenkins + cd /usr/share/jenkins + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt + # Reset the password defined as the JenkinsAdminPassword + sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml + # Configure the Jenkins Location + curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml + # Start Jenkins + docker-compose up --build -d JenkinsSpotAgentLaunchTemplate: # This is a launch template that will be used to provision Jenkins build agents - showing how spot instances can be used to scale-out build jobs at low cost. Type: AWS::EC2::LaunchTemplate From 4be6b222f69759480f3935434501e4dbf3e4f15e Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 2 Aug 2022 15:36:02 +0200 Subject: [PATCH 14/29] Updated the persistence lab for the CI/CD Workshop --- .../jenkins-asg/_index.md | 7 +- .../jenkins-asg/asg.md | 4 +- .../jenkins-asg/test-persistence.md | 117 ++++++++++++------ .../amazon-ec2-spot-cicd-workshop.yaml | 13 ++ 4 files changed, 97 insertions(+), 44 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md index b43bc322..de814032 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md @@ -12,11 +12,8 @@ To address these behaviours, Jenkins provides the capability to execute builds o Let's get started by setting up the following environment variables you'll use in the workshop, the the following commands: ```bash -export VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values='Amazon EC2 Spot CICD Workshop VPC' | jq -r '.Vpcs[0].VpcId'); -export SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_ID}" --filters Name=tag:Type,Values='Private'); -export SUBNET_1=$((echo $SUBNETS) | jq -r '.Subnets[0].SubnetId'); -export SUBNET_2=$((echo $SUBNETS) | jq -r '.Subnets[1].SubnetId'); -export SUBNET_3=$((echo $SUBNETS) | jq -r '.Subnets[2].SubnetId'); +export PRIVATE_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPrivateSubnets'].OutputValue" --output text); +export PUBLIC_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPublicSubnets'].OutputValue" --output text); export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); ``` diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md index 88f1d300..ec50cba9 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -19,7 +19,7 @@ cat < ~/asg-policy.json "LaunchTemplate":{ "LaunchTemplateSpecification":{ "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", - "Version":"1" + "Version":"\$Latest" }, "Overrides":[ { @@ -60,7 +60,7 @@ Additionally, the configuration above, sets the `SpotAllocationStrategy` to `cap Let’s create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (to avoid having instances when there's no need), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. ```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${SUBNET_1},${SUBNET_2},${SUBNET_3}" --mixed-instances-policy file://asg-policy.json +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${PRIVATE_SUBNETS}" --mixed-instances-policy file://asg-policy.json ``` You may now proceed with the next step. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index 0e9b20a8..8b84ea93 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -5,72 +5,115 @@ weight = 125 You're now using Spot instances for your code builds – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by an Auto Scaling group. ## Get the EFS Filesystem ID -As with the previous steps, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab. You will need to determine and make a note of what the **EFS Filesystem ID** is. +As with the previous steps, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab. You will need to determine and make a note of what the **EFS Filesystem ID** is. To get it, run the following command get the ID from the CloudFormation stack output: -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#)); -2. Click on the checkbox associated with the **SpotCICDWorkshop** stack; -3. From the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console, make note of values associated with the **EFSFileSystemID** key. - -## COPY THE CONTENTS OF JENKINS_HOME TO YOUR EFS FILE SYSTEM -In order to copy the contents of the `JENKINS_HOME` directory to the EFS file system (for which the ID of which you determined in the previous step), you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. +```bash +aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='EFSFileSystemID'].OutputValue" --output text; +``` -Once the file system has been mounted, stop the Jenkins service and set the file system permission of the root of your filesystem so that the jenkins user and group are owners. Finally, copy the contents of **/jenkins_home** (this is the `JENKINS_HOME` directory) across to the root of your EFS file system. +## Copy the contents of Jenkins home to your EFS file system +In order to copy the contents of the `JENKINS_HOME` directory to the EFS file system, you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. To do so, SSH into the Jenkins server: 1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); 2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; 3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity; -4. Mount the EFS file system that was created by the CloudFormation template at the /mnt mountpoint by entering the following command, replacing `%FILE-SYSTEM-ID%` with the **EFSFileSystemID** noted above: - + +Then, run the following command and replace the `EFSFileSystemID` text with the ID you got in the previous section: + ```bash sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ -.%FILE-SYSTEM-ID%.efs.eu-west-1.amazonaws.com:/ /mnt +.EFSFileSystemID.efs.eu-west-1.amazonaws.com:/ /mnt ``` -5. Stop the Jenkins service by entering the following commands: + +Once the file system has been mounted, stop the Jenkins container. ```bash cd /usr/share/jenkins -sudo docker-compose stop +docker-compose stop ``` -6. The content of `JENKINS_HOME` is stored under /jenkins_home – copy this content (whilst preserving permissions) to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): +The content of `JENKINS_HOME` is stored under /jenkins_home – copy this content to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): ```bash sudo cp -rpv /jenkins_home/* /mnt ``` -## PROVISION AN EC2 SPOT FLEET FOR YOUR NEW JENKINS HOST -The Spot Fleet that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents - though this time you'll be a bit more aggressive with the bid price to ensure that you see an overall saving over what you would have spent on an on-demand t3.medium instance. Additionally, you'll configure this Spot Fleet so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. - -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Click on the **Request Spot Instances** button; -3. At the first screen of the Spot instance launch wizard: - 1. Under the Tell us your application or task need heading, switch to the **Load balancing workloads** option; - 2. In the Configure your instances section, select the **JenkinsMasterLaunchTemplate** template from the Launch template dropdown. Change the Network to be the **Spot CICD Workshop VPC**. After making this selection, enable the check boxes for all three Availability Zones and then select the **Amazon EC2 Spot CICD Workshop Public Subnet** associated with each availability zone as the subnet to launch instances in; - 3. At the Tell us how much capacity you need section, keep the Total target capacity at **1** instance and the Optional On-Demand portion set to **0**, and then tick the **Maintain target capacity** checkbox. Once selected, leave the Interruption behavior set to **Terminate**; - 4. Again, you'll override the recommendations made by the console, so clear the tick from **Apply recommendations** checkbox. Click on the **Remove** links associated with the all of the instance types initially defined to remove them from the fleet configuration. Then click on the **Select instance types** button and add the **m3.large**, **m4.large**, **t2.medium** and **t3.medium** instance types to the fleet definition. Once the checkboxes for the required instance types have been ticked, click on the **Select** button. Once you have the four desired instance types listed in the fleet request, select the **Lowest Price** Fleet allocation strategy (since we’re interested in keeping cost to an absolute minimum for this use case, and it makes little sense to diversify a single instance across any number of instance pools); - 5. At the Additional request details section, remove the tick from the **Apply defaults** checkbox. As you want to keep the cost of running your Jenkins server below that for which you're currently paying, select the **Set your max price (per instance/hour)** option, and set the price to be the on-demand price of a t3.medium instance in the Ireland region, which is **$0.0456**. In order to ensure that the server can receive HTTP requests from the Application Load Balancer you've been using, tick the checkbox labelled **Receive traffic from one or more load balancers** and from the Target groups dropdown, select **JenkinsMasterEC2TargetGroup**. - 6. Review the Your fleet request as a glance section - it should indicate that your Fleet strength is strong as a result of being able to draw instances from 12 instance pools, and your Estimated price should indicate that you're expecting to make a 73% saving compared to the cost of equivalent on-demand resources; - 7. Lastly, click on the **Launch** button. - -## VERIFY THAT YOUR EC2 SPOT INSTANCE IS ATTACHED TO YOUR ALB TARGET GROUP +## Provision an Auto Scaling group for the new Jenkins host +The Auto Scaling group that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents. Additionally, you'll configure this Auto Scaling group so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. + +Run the following commands to create the Auto Scaling group configuration file: + +```bash +export LAUNCH_TEMPLATE_HOST_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsMasterLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +cat < ~/asg-jenkins-host-policy.json +{ + "LaunchTemplate":{ + "LaunchTemplateSpecification":{ + "LaunchTemplateId":"${LAUNCH_TEMPLATE_HOST_ID}", + "Version":"\$Latest" + }, + "Overrides":[ + { + "InstanceType":"t2.large" + }, + { + "InstanceType":"t3.large" + }, + { + "InstanceType":"m4.large" + }, + { + "InstanceType":"m5.large" + }, + { + "InstanceType":"c5.large" + }, + { + "InstanceType":"c4.large" + } + ] + }, + "InstancesDistribution":{ + "OnDemandBaseCapacity": 0, + "OnDemandPercentageAboveBaseCapacity": 0, + "SpotAllocationStrategy":"capacity-optimized" + } +} +EoF +``` + +Then, run the following command to create the Auto Scaling group. Notice that you might need to re-create the environment variables you created before. + +```bash +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsHostASG --min-size 1 --max-size 1 --desired-capacity 1 --vpc-zone-identifier "${PUBLIC_SUBNETS}" --mixed-instances-policy file://asg-jenkins-host-policy.json; +``` + +To include the new instances into the Application Load Balancer, attach the Auto Scaling group to the Target Group by running the following command: + +```bash +export EC2_TARGET_GROUP=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsMasterEC2TargetGroup'].OutputValue" --output text); +aws autoscaling attach-load-balancer-target-groups \ + --auto-scaling-group-name EC2SpotJenkinsHostASG \ + --target-group-arns $EC2_TARGET_GROUP; +``` + +## Verify that the new Spot instance is running the Jenkins server After a few moments, your Spot instance will start up and should attach itself the the Target Group being used by your Application Load Balancer. Determine if this registration has completed successfully and when it has done so, access Jenkins through your Load Balancer and fire off a build of Apache PDFBox to ensure that everything is still working as expected. 1. Go to the **EC2** console and click on the **Target Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#TargetGroups)); -2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the fleet that you just launched. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete after placing your Spot Fleet request. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; +2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the Auto Scaling group that you just created. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; 3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. -## TEST THE SELF-HEALING ARCHITECTURE BY TERMINATING THE RUNNING EC2 SPOT INSTANCE -What happens when the market price for the capacity pool that your EC2 Spot instance is running in goes over your $0.0456 per instance-hour bid price? - -When this happens, your EC2 Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, Spot Fleet will observe that there are no running instances in the fleet and because the desired capacity is one, it will attempt to launch a replacement instance in a capacity pool that is still below the per instance-hour bid price that you set (this is why diversification across many capacity pool is a best practice). If such an instance is available, a replacement instance will be launched and bootstrapped in exactly the same manner as your original instance. +## Test theh self-healing architecture running with Spot instances +At some point in time, you might receive a Spot interruption notice when On-Demand needs the capacity back. When this happens, your Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, the Auto Scaling group will observe that there are no running instances in the group and because the desired capacity is one, it will attempt to launch a replacement instance the pool with more capacity available (this is why diversification across many capacity pool is a best practice). The new instance will be launched and bootstrapped in exactly the same manner as your original instance. Test this out by terminating the Spot instance with the **Jenkins Master (Spot)** name tag and verifying that a replacement instance comes up to take its place. 1. Go to the **EC2** console and click on the **Instances** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances)); -2. Search for the EC2 instance with the Name tag of **Jenkins Master (Spot)**, right-click on it and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. Within moments of terminating this instance, your Spot Fleet registration should detect that the number of running instances is below the target that you’ve defined (that target being one instance) and therefore should launch a new, replacement spot instance. Refresh the list of EC2 instances every few seconds until you see the new instance be allocated (while you're waiting, it might also be interesting to see what's happening under the History tab of your Spot Fleet). Whilst your Jenkins service will be unavailable until this new spot instance launches and bootstraps, service should automatically be restored in a couple of minutes. +2. Search for the EC2 instance with the Name tag of **Jenkins Master (Spot)**, right-click on it and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. Within moments of terminating this instance, your Auto Scaling group should detect that the number of running instances is below the target that you’ve defined (that target being one instance) and therefore should launch a new, replacement spot instance. Refresh the list of EC2 instances every few seconds until you see the new instance be allocated. Whilst your Jenkins service will be unavailable until this new spot instance launches and bootstraps, service should automatically be restored in a couple of minutes. -## TERMINATE THE ON-DEMAND INSTANCE THAT WAS INITIALLY USED FOR YOUR JENKINS SERVER -Once you've verified that your Spot Fleet is self-healing, you no longer have any need for the On-demand instance. To prevent it from incurring unnecessary cost, it can be terminated. +## Terminate the On-Demand instance for Jenkins +Once you've verified that your new Auto Scaling group using Spot instances is self-healing, you no longer have any need for the On-demand instance. To prevent it from incurring unnecessary cost, it can be terminated. 1. Remain at the EC2 Instances screen and search for the EC2 instance with the Name tag of **Jenkins Master (On-demand)**. Right-click on the one instance that should come up and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. \ No newline at end of file diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml index 76734932..63e71acc 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml @@ -1020,6 +1020,7 @@ Resources: curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose # Mount the Jenkins EFS volume + mkdir -p /jenkins_home mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFSJenkinsHomeVolume}.efs.eu-west-1.amazonaws.com:/ /jenkins_home # Download files from GitHub mkdir -p /usr/share/jenkins @@ -1324,7 +1325,19 @@ Outputs: Description: Security Group for Jenkins nodes. Use this value to configure Jenkins ECS Plugin Value: !Ref SecurityGroupJenkins + JenkinsVPCPrivateSubnets: + Description: The private subnets where Jenkins will be deployed. Use this value to configure Jenkins Spot agents. + Value: + !Join [ + ",", + [!Ref SubnetPrivateA, !Ref SubnetPrivateB, !Ref SubnetPrivateC], + ] + JenkinsVPCPublicSubnets: Description: The public subnets where Jenkins will be deployed. Use this value to configure Jenkins ECS plugin Value: !Join [",", [!Ref SubnetPublicA, !Ref SubnetPublicB, !Ref SubnetPublicC]] + + JenkinsMasterEC2TargetGroup: + Description: Target Group for Jenkins EC2 nodes. + Value: !Ref JenkinsMasterALBTargetGroupEC2 From 064bb507f8b88f04dcc126f547c6e16bb66668c8 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 3 Aug 2022 13:27:28 +0200 Subject: [PATCH 15/29] Re-structure labs for CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop/_index.md | 27 +- content/amazon-ec2-spot-cicd-workshop/clea.md | 83 --- .../conclusion.md | 4 + .../jenkins-asg/_index.md | 22 +- .../jenkins-asg/asg.md | 4 +- .../jenkins-asg/clean.md | 24 + .../jenkins-asg/pipeline.md | 100 --- .../{ => jenkins-asg}/prep.md | 21 +- .../jenkins-asg/test-build.md | 4 +- .../jenkins-asg/test-persistence.md | 2 +- .../jenkins-ecs/lab4.md | 2 +- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 651 ++++++++++++++++++ ...=> amazon-ec2-spot-cicd-workshop-ecs.yaml} | 0 13 files changed, 719 insertions(+), 225 deletions(-) delete mode 100644 content/amazon-ec2-spot-cicd-workshop/clea.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/conclusion.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md delete mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md rename content/amazon-ec2-spot-cicd-workshop/{ => jenkins-asg}/prep.md (78%) create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml rename workshops/amazon-ec2-spot-cicd-workshop/{amazon-ec2-spot-cicd-workshop.yaml => amazon-ec2-spot-cicd-workshop-ecs.yaml} (100%) diff --git a/content/amazon-ec2-spot-cicd-workshop/_index.md b/content/amazon-ec2-spot-cicd-workshop/_index.md index 902201a3..2c5bccd1 100644 --- a/content/amazon-ec2-spot-cicd-workshop/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/_index.md @@ -1,31 +1,22 @@ --- -title: "CI/CD and Test Workloads with EC2 Spot Instances" -menuTitle: "CI/CD and Test Workloads" +title: "CI/CD Workloads with EC2 Spot Instances" +menuTitle: "CI/CD Workloads" date: 2019-02-19T02:02:35 weight: 80 pre: "8. " --- ## Overview -During this workshop, you'll get hands-on with Amazon EC2 Spot and discover architectural best practices through the lens of DevOps and CI/CD. You'll deploy Jenkins build agents and test environments on Spot instances at a fraction of the cost of on-demand instances. You'll also implement mechanisms to ensure that your CI/CD tooling recovers from spot market events by decoupling application state from your compute resources. Finally, you'll migrate your CI/CD environment to a containerized environment to eke out maximum performance and cost efficiency. In addition to covering the ins and outs of Spot, we'll share some of the Spot-based mechanisms used by customers to reduce the cost of their test and production workloads. +In this workshop, you'll get hands-on with Spot instances and discover architectural best practices through the lens of DevOps and CI/CD. We'll dive dive deep on how to deploy tools like Jenkins and use Spot instances as build agents. You'll also implement mechanisms to ensure that your CI/CD tooling recovers from Spot interruptions by simulating failures and decoupling application state from your compute resources. Moreover, you'll migrate your CI/CD environment to a containerized environments using ECS to eke out maximum performance and cost efficiency. In addition to covering the best practices to use Spot, we'll share some of the Spot-based mechanisms used by customers to optimize their infrastructure resources. -## Workshop Details -This workshop will be broken down into a series of labs that flow on from each other (that is, you must complete each lab in order before proceeding with the next). The lab exercises that will be covered are: +## Workshop Labs +This workshop will be broken down into a series of labs using differenct CI/CD tools and AWS services, topics covered are: -* Workshop preparation: Deploy pre-requisite resources through Amazon CloudFormation; -* Jenkins with Amazon EC2 Auto Scaling Groups using Spot; -* Jenkins on ECS using Spot; -* Workshop clean up. +* [Jenkins with Auto Scaling groups](/amazon-ec2-spot-cicd-workshop/jenkins-asg.html) +* [Jenkins with ECS](/amazon-ec2-spot-cicd-workshop/jenkins-ecs.html) +{{% notice note %}} As a reminder, you should have a laptop device (Windows/OSX/Linux are supported - tablets are not appropriate) with the current version of Google Chrome or Mozilla Firefox installed. You should also have a clean AWS account, with **AdministratorAccess** policy-level access. +{{% /notice %}} This workshop should take between two and three hours to complete, depending on your proficiency with the AWS services being featured. - -#### Additional considerations when running this workshop in a corporate IT environment -If you are running this workshop from a corporate IT environment, contact your Systems Administrator to ensure that you will be able to establish outbound Secure Shell (SSH) connections to an Internet host: - -* If you cannot establish SSH connections to Internet hosts (and do not have a suitable workaround), you will not be able to complete some labs; -* If you can establish SSH connections to Internet hosts, obtain from your Systems Administrator the source IP address CIDR block that connections will be established from. - -If you access the Internet through a transparent proxy server running in your corporate IT environment and this proxy server uses a different source address than where SSH connections come from, additional configuration of AWS Security Groups will need to be carried out. The lab guide will indicate the configuration steps required when appropriate. - diff --git a/content/amazon-ec2-spot-cicd-workshop/clea.md b/content/amazon-ec2-spot-cicd-workshop/clea.md deleted file mode 100644 index e772baf6..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/clea.md +++ /dev/null @@ -1,83 +0,0 @@ -+++ -title = "Workshop Cleanup" -weight = 1000 -+++ -Congratulations, you have completed this workshop! Your next challenge is to remove all of the resources that were provisioned in your account so as to ensure that no additional cost can be incurred. Please note that the steps below should be implemented in order - some later steps have dependencies on earlier ones! - -## DELETE ALL OBJECTS WITHIN THE S3 BUCKET CREATED BY CLOUDFORMATION -When you attempt to delete the CloudFormation template that you deployed during the Workshop Preparation, the S3 bucket that it provisioned will not be able to be removed unless it is empty. Delete all objects in this bucket (note, you don't need to remove the bucket itself). - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **S3** console (or [click here](https://s3.console.aws.amazon.com/s3/home?region=us-east-1)); -2. Click on the name of the S3 bucket that you noted in Lab 2 (it should have a name of spotcicdworkshop-deploymentartifactss3bucket-random_chars); -3. Mark the checkbox next to the **gameoflife-web** directory, then click on the **Actions** dropdown and select **Delete**. At the confirmation screen, click on the **Delete** button. -{{% /expand%}} - -## MODIFY THE JENKINS MASTER ECS SERVICE SO THAT 0 TASKS ARE DESIRED -Before the EC2 instances that comprise your ECS cluster can be terminated, you'll need to ensure that no containers are running in the cluster. The containers hosting your build agents are likely already stopped as they only remain online for a few seconds after your build jobs have completed - but the container running Jenkins needs to be stopped. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **ECS** console (or [click here](https://eu-west-1.console.aws.amazon.com/ecs/home?region=eu-west-1#/clusters)) and click on the **SpotCICDWorkshopECSCluster** link; -2. Click on the checkbox of the service with a Service Name that begins with **SpotCICDWorkshop-ECSServiceJenkinsMaster**, then click on the **Update** button; -3. Change the Number of tasks from **1** back to **0**, then click on the **Next Step** button; -4. At the Configure network screen, click on the **Next Step** button. Repeat the same action at the Set Auto Scaling (optional) screen. Finally at the Review screen, click on the **Update Service** button. Once the service has been updated, click on the **View Service** button. -{{% /expand%}} - -## DELETE ALL AUTO SCALING GROUPS; -As was the case with EC2 instances created by Spot Fleet requests, you won't be able to terminate an EC2 instance that belongs to the Auto Scaling Group that you created without the Auto Scaling Group re-launching a new instance to replace the one that you terminated. Therefore, the Auto Scaling Group that you created in Lab 4 must be deleted. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Auto Scaling Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/autoscaling/home?region=eu-west-1#AutoScalingGroups)); -2. Mark the checkbox next to the Auto Scaling Group with the name of **Spot CICD Workshop ECS Auto Scaling Group** and from the **Actions** dropdown, click **Delete**. In the confirmation, select **Yes, Delete**. -{{% /expand%}} - -## CANCEL ALL REMAINING SPOT REQUESTS -The CloudFormation template will not be able to delete subnet and VPC resources if there are EC2 resources still running. Spot fleets are an interesting case though - if you terminate an EC2 instance belonging to a fleet, the fleet will attempt to re-launch it (as shown in Lab 3). Therefore you will need to ensure that all of the Spot Fleets that you created in this workshop have been cancelled. - - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Select the checkboxes corresponding to the Spot Fleet request that you created during this workshop that still remain (it should be noted that if you followed every step in the Lab guides, all of these Spot Fleets should already have been removed); -3. Click the Actions dropdown at the top of the screen and select the **Cancel spot request** option. At the confirmation dialogue, ensure that the **Terminate instances** checkbox is selected and click on the **Confirm** button. -{{% /expand%}} - -## ENSURE THAT ALL EC2 INSTANCES HAVE BEEN TERMINATED -Having removed all of the Spot Fleets and Auto Scaling groups from the VPC that was created by the CloudFormation template deployed during the Workshop Preparation, you should double-check that all EC2 instances used during the workshop have been terminated. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Instances** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); - * If there is a running instance with a Name of **Jenkins Master (On-Demand)**, mark the checkbox associated with this instance and from the **Actions** dropdown, select **Instance State** > **Terminate**. At the confirmation dialog, click on the **Yes, Terminate** button; - * If there are running instances with a name of **Jenkins Master (Spot)** or **Jenkins Build Agent**, you still have open Spot Fleet requests - repeat the **Cancel all remaining Spot Requests** section above; - * If there are running instances with a name of **Spot CICD Workshop ECS Instance**, you still have an active Auto Scaling Group - repeat the **Delete all Auto Scaling Groups** section above; - * If there are running instances with a name of **Game of Life Test Instance**, you still have the test environment that you created in Lab 2 running - it's fine for these instances to be running, though you will have to delete the CloudFormation stack associated with this test environment prior to deleting the main stack in the next section. -{{% /expand%}} - -## DELETE ALL CLOUDFORMATION STACKS -Once you've checked that all EC2 instances have been terminated and there are no objects present in the S3 bucket used for deployment artifacts, you can remove the CloudFormation template that you deployed during the Workshop Preparation. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); -2. If you see a Stack named **GameOfLife**, mark the checkbox associated with this stack and from the **Actions** dropdown, select the **Delete Stack** option. At the confirmation pop-up, click on the **Yes, Delete** button. Note, you must wait for this stack to be deleted before proceeding on to the next step. -3. Mark the checkbox associated with the stack named **SpotCICDWorkshop** and from the **Actions** dropdown, select the **Delete Stack** option. At the confirmation dialog, click on the **Yes, Delete** button. -{{% /expand%}} - -## DELETE ALL CLOUDWATCH LOG GROUPS RELATED TO THE SPOTCICDWORKSHOP -The Lambda functions used throughout the workshop (notably, to create and tear down the testing environment in Lab 2, and to look up the latest version of various AMIs throughout the workshop) wrote log entries to CloudWatch Logs. These should be removed. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudWatch** console and click on the **Logs** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/cloudwatch/home?region=eu-west-1#logs:)); -2. For each of the Log Groups that start with **/aws/lambda/SpotCICDWorkshop**: - 1. Select the radio box associated with the Log Group; - 2. From the **Actions** dropdown, select the **Delete log group** option. From the resulting dialog, click on the **Yes, Delete** button. -{{% /expand%}} - -## REMOVE THE AMAZON EC2 KEYPAIR CREATED DURING THE WORKSHOP PREPARATION -The final resource that needs to be removed was the first one that you created - the EC2 Key Pair that you created prior to launching the CloudFormation stack. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Key Pairs** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#KeyPairs)); -2. Mark the check box associated with the Key Pair named **Spot CICD Workshop Key Pair** and click on the **Delete** button. At the resulting pop-up, confirm this action by clicking on the **Yes** button. -{{% /expand%}} - -## THANK YOU -At this point, we would like to than you for attending this workshop. diff --git a/content/amazon-ec2-spot-cicd-workshop/conclusion.md b/content/amazon-ec2-spot-cicd-workshop/conclusion.md new file mode 100644 index 00000000..f4cd8c5c --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/conclusion.md @@ -0,0 +1,4 @@ ++++ +title = "Conclusion" +weight = 1000 ++++ diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md index de814032..eb40c184 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/_index.md @@ -1,22 +1,18 @@ +++ -title = "Jenkins with Amazon EC2 Auto Scaling Groups" +title = "Jenkins with Auto Scaling groups" weight = 100 +++ -By default, all job builds will be executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: +By default, all job builds are executed on the same instance that Jenkins is running on. This results in a couple of less-than-desirable behaviours: * When CPU-intensive builds are being executed, there may not be sufficient system resources to display the Jenkins server interface; and * The Jenkins server is often provisioned with more resources than the server interface requires in order to allow builds to execute. When builds are not being executed, these server resources are essentially going to waste. To address these behaviours, Jenkins provides the capability to execute builds on external hosts (called build agents). Further, AWS provides a Jenkins plugin to allow Jenkins to scale out a fleet of EC2 instances in order to execute build jobs on. This lab will focus on implementing EC2 Spot build agents, showcasing what a batch processing workload typically looks like when using Amazon EC2 Spot instances. -## Setting Up environment variables -Let's get started by setting up the following environment variables you'll use in the workshop, the the following commands: +## What we'll be doing? +The purpose of these labs is to help you understand how to configure build agents using Spot instances. We'll focus on configuring a Jenkins plugin so that build agents are launched only when you need them. At the end of these labs, you'll be able to: -```bash -export PRIVATE_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPrivateSubnets'].OutputValue" --output text); -export PUBLIC_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPublicSubnets'].OutputValue" --output text); -export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); -``` - -If for some reason the environment variables are cleared, you can run the previous commands again without any problem. - -You may now proceed with the next step. \ No newline at end of file +* Learn how to launch Spot instances following our best practices using Auto Scaling groups +* Configure properly the EC2 Fleet Jenkins plugin +* Configure Jenkins build jobs to use Spot instances as agents +* Simulate Spot interruption events to increase fault-tolerance +* Test how resilient and fault-toleran Jenkins could be using Spot instances \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md index ec50cba9..146efea7 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -61,6 +61,4 @@ Let’s create the Auto Scaling group. In this case the Auto Scaling group spans ```bash aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${PRIVATE_SUBNETS}" --mixed-instances-policy file://asg-policy.json -``` - -You may now proceed with the next step. \ No newline at end of file +``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md new file mode 100644 index 00000000..1c1da386 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md @@ -0,0 +1,24 @@ ++++ +title = "Cleanup" +weight = 150 ++++ +Congratulations, you have completed the Jenkins with Auto Scaling group lab! Your next challenge is to remove all of the resources that were provisioned in your account so as to ensure that no additional cost can be incurred. Please note that the commands below should be executed in order - some later steps have dependencies on earlier ones! + +## Delete all Auto Scaling groups + +```bash +aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG; +aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsHostASG; +``` + +## Delete theh CloudFormation stack + +```bash +aws cloudformation delete-stack --stack-name SpotCICDWorkshop; +``` + +## Remove theh EC2 key pair +The final resource that needs to be removed was the first one that you created - the EC2 Key Pair that you created prior to launching the CloudFormation stack. + +1. Go to the **EC2** console and click on the **Key Pairs** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#KeyPairs)); +2. Mark the check box associated with the Key Pair named **Spot CICD Workshop Key Pair** and click on the **Delete** button. At the resulting pop-up, confirm this action by clicking on the **Yes** button. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md deleted file mode 100644 index 432c49e4..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/pipeline.md +++ /dev/null @@ -1,100 +0,0 @@ -+++ -title = "Configure a Pipeline Job using Spot" -weight = 130 -+++ -Now that you are carrying out your software builds on Spot instances, the next step is to build out a CI/CD pipeline to have your software deployed to a test environment also running on Spot instances. Pipelines within Jenkins are defined using another plugin, and you'll define pipeline steps to both deploy and terminate your testing environment. The pipeline steps that you define in this lab will call a Lambda function, which in turn deploys a CloudFormation template to brings up your testing environment, and deletes the template to tear it down. - -If you view the CloudFormation template that is used to deploy the test environment, pay close attention to the following lines: - -* Lines 183-218 contains the resource definition for the Launch Template used within the Spot Fleet. Of particular note, the User Data defined within the Launch Template updates the system with all current patches, installs the Java 8 OpenJDK and Apache Tomcat 8, deploys the build artifact generated by Jenkins and starts up the Tomcat service; -* Lines 220-250 define the Spot Fleet that is used to build out the test environment. Specifically, we are following the best practices for stateless applications using Spot: the fleet uses a diversified allocation strategy to ensure that instances are distributed across multipe availablility zones and the fleet is set to replace any instances that are unhealthy; - -## INCREASE THE NUMBER OF BUILD EXECUTORS ON YOUR JENKINS BUILD AGENTS -When executing a pipeline job in Jenkins, the pipeline consumes one of the build execution slots. Given that the the software builds themselves are executed using a build agent on a Spot instance, you should configure the Jenkins master server to also have a build executor - for pipeline execution to run on. - -1. Back at the Jenkins home page, click on the **Manage Jenkins** link on the left side menu again, and then the **Configure System** link; -2. Scroll down to the bottom of the page and under the **Spot Fleet Configuration** section, increase the **Number of Executors** to 3; -3. Click on the **Save** button at the bottom of the screen; -4. In the left column under **Build Executor Status**, you should see the EC2 Instance ID for the instances that make up your build fleet. Click on the Instance ID and then click on the **Delete Agent** link on the left hand side of the screen. When prompted if you're sure about deleting the agent, click on the **Yes** button. Deleting the agent from the host will force Jenkins to re-install it, this time with multiple build executors. - -## RECONFIGURE THE GAME OF LIFE PROJECT -The S3 Publisher Plugin should be pre-configured for use (it will use the IAM policy attached to the EC2 Instance Profile used by the Jenkins Master or the build agents to grant access to the S3 bucket where your deployment artifacts will be uploaded to). First up, you'll need to reconfigure the **Game of Life** project to publish it's artifacts to the S3 bucket that was created by the CloudFormation stack deployed at the beginning of this workshop. - -1. Back at the Jenkins home page, go into the **Game of Life** project by clicking on its Name link. Next, click on the **Configure** link in the left side menu; -2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression; -3. Scroll down to the Post-build Actions section and to the bottom of that section, click on the **Add post-build action** button, then select the **Publish artifacts to S3 Bucket** option. A new configuration block will appear; -4. Within the new configuration block, select the **Spot CICD Workshop Artifacts** option from the S3 Profile dropdown. Click on the **Add** button next to the Files to upload label; -5. In the Source field, enter **\*\*/*.war** as the match condition for your build artifact - this will ensure that the WAR (Web Application Resource) file produce by our build process is uploaded; -6. In the Destination bucket field, enter in the name of the S3 bucket that was created by the CloudFormation template deployed in Lab 1 (the S3 bucket name can be obtained from the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console or by [clicking here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks?filter=active) - look for the value associated with the **DeploymentArtifactsS3Bucket** key); -7. In the Bucket Region dropdown, select the **eu-west-1** region; -8. Finally, mark both the **No upload on build failure** and **Publish from Slave** checkboxes - this will ensure that uploads only occur when a build is successful, and to ensure that our build agents are aware that they will also have the permissions to upload (given that they are using the same IAM role that our master server is using); -9. At the bottom of the page, click on the **Save** button. - -## INSPECT THE CI/CD PIPELINE AND LAMBDA FUNCTION THAT WILL MANIPULATE THE TEST ENVIRONMENT -Your Jenkins server has been preconfigured with a Game of Life Pipeline job that contains four stages as defined in the following Groovy script: - -```groovy -node { - stage('Build') { - // Build our project and upload artifact to S3 - build 'Game of Life' - } - stage('Deploy') { - // Invoke a Lambda function that will create our test environment by launching a Spot Fleet - invokeLambda([awsAccessKeyId: '', awsRegion: 'eu-west-1', awsSecretKey: '', functionName: 'SpotCICDWorkshop_ManageTestEnvironment', payload: '''{ - "action": "deploy", - "stackName": "GameOfLife", - "jobBaseName": "${JOB_BASE_NAME}", - "buildId": "${BUILD_ID}" - }''', synchronous: true, useInstanceCredentials: true]) - } - stage('Test') { - // A manual approval stage designed to halt the pipeline until someone indicates that the pipeline can proceed - input 'Run your tests' - } - stage('Terminate') { - // Invoke a Lambda function to tear down our test environment once testing has been completed - invokeLambda([awsAccessKeyId: '', awsRegion: 'eu-west-1', awsSecretKey: '', functionName: 'SpotCICDWorkshop_ManageTestEnvironment', payload: '''{ - "action": "terminate", - "stackName": "GameOfLife", - "jobBaseName": "${JOB_BASE_NAME}", - "buildId": "${BUILD_ID}" - }''', synchronous: true, useInstanceCredentials: true]) - } -} -``` - -* The **Build** stage should simply execute the Game of Life build job that you just reconfigured. As you'd expect from the configuration change that you made, the deployment artifact generated by this stage will be stored in an S3 bucket; -* The **Deploy** stage will need to invoke a Lambda function, which in turn will deploy a CloudFormation template. The template will create a Spot Fleet consisting of two instances using configuration defined in an EC2 Launch Template; and within this Template will be EC2 UserData that will deploy your build artifact and all of its dependencies. The template will also deploy an Application Load Balancer to front the Game of Life application, through which you will test the application; -* The **Test** stage will be a manual approval stage, designed to allow you the opportunity to test out the deployed artifact; and -* The **Terminate** stage will invoke a Lambda function that will destroy the test environment by deleting the CloudFormation stack. - -In the Groovy script, note that the same Lambda function (SpotCICDWorkshop_ManageTestEnvironment) is called for both the Deploy and Terminate stages, but a different “action” parameter value is passed on to the function. - -Next, take a look at the Lambda function to observe what it's going to do upon execution - you can view the function by going to the **Lambda** console and clicking on the **SpotCICDWorkshop\_ManageTestEnvironment** function (or by [clicking here](https://eu-west-1.console.aws.amazon.com/lambda/home?region=eu-west-1#/functions/SpotCICDWorkshop_ManageTestEnvironment)) - or you can click on the code snippet link to the left to view a version of the code that has not been customised to access resources within your account by CloudFormation. - -The **deploy** action within the function starts from line 4: - -* Lines 9-35 contain the CloudFormation parameters that are passed on to the CloudFormation template. These parameters were generated based on the outputs of the initial CloudFormation template that you deployed in the Workshop Preparation; -* Line 36 defines the IAM role that is assumed when deploying the template - again, this role was deployed as a part of the initial CloudFormation template defined in the Workshop Preparation; -* Line 37 contains the URL of the CloudFormation template that is going to be launched by the function: [https://s3-us-west-2.amazonaws.com/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop_game-of-life.yaml](https://s3-us-west-2.amazonaws.com/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop_game-of-life.yaml) -* Line 39 attempts to deploy the stack; and -* If the request to deploy the stack was successful, lines 43-56 store some metadata (including the CloudFormation Stack ID) into a DynamoDB table that was created by the initial CloudFormation template. - -The **terminate** action within the function starts from line 61: - -* Lines 63-70 queries the DynamoDB table to obtain the CloudFormation Stack ID added by the deploy action; and -* If the stack ID was able to be obtained from DynamoDB, it is used to tell CloudFormation to delete the stack with that ID on lines 74-81. - -Finally, lines 87-102 contain the boilerplate code used to load the relevant AWS SDK libraries and call the actions defined above. - -## DEPLOY, TEST, THEN TERMINATE A TEST ENVIRONMENT USING THE GAME OF LIFE PIPELINE -You should now be at the stage where your the Game of Life Pipeline can be tested. To commence this, go back to the Jenkins Home page and click on the **Game of Life Pipeline** project. Within the project, click on the **Build Now** link on the left. When the pipeline initiates, you'll get to see the execution status of each stage as the workflow progresses through the pipeline. - -Once the deploy stage has been completed successfully: - -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)) and wait for the deployment of the GameOfLife stack to switch to a **CREATE_COMPLETE** status; -2. When the stack has deployed, click on the checkbox next to the GameOfLife stack, then click on the Outputs tab at the bottom of the screen; -3. Copy the value associated with the **GameOfLifeDNSName** key into the address bar of a new browser tab and append the **/gameoflife** suffix before hitting the Enter key. - -Once you've verified that that the Game of Life web application loads, return to Jenkins and click on the **Test** pipeline stage and click on **Proceed** to get the pipeline to process the Terminate stage. After this final stage has executed, go back to the CloudFormation console to verify that the GameOfLife stack is being deleted. diff --git a/content/amazon-ec2-spot-cicd-workshop/prep.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md similarity index 78% rename from content/amazon-ec2-spot-cicd-workshop/prep.md rename to content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md index fea829f9..2a4fa3b1 100644 --- a/content/amazon-ec2-spot-cicd-workshop/prep.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md @@ -1,10 +1,10 @@ +++ -title = "Workshop Preparation" +title = "Setup with CloudFormation" weight = 10 +++ Before we start presenting content on the different use cases for EC2 Spot instances through the lens of CI/CD workloads, you'll need to prepare the AWS account that you've come to this workshop with. Specifically, this workshop involves working with a large number of AWS resources as well as a deployment of Jenkins that if manually configured, would leave you little time to discover how to use EC2 Spot instances in the most effective manner. To address that, you will deploy an Amazon CloudFormation template that does a lot of the heavy lifting of provisioning these resources for you. -## CREATE A NEW EC2 KEY PAIR +## Create a new EC2 Key Pair You will need to access the SSH interfaces of some Linux EC2 instances created in this workshop. To do so in a secure manner, please create a new EC2 key pair with a name of **Spot CICD Workshop Key Pair** in the **EU (Ireland)** region (all activities for this workshop will be carried out in this region). 1. Log in to your AWS Account; @@ -15,11 +15,11 @@ You will need to access the SSH interfaces of some Linux EC2 instances created i 3. Enter **Spot CICD Workshop Key Pair** as the Key pair name and click on the **Create** button; 4. Your web browser should download a .pem file – keep this file as it will be required to access the EC2 instances that you create in this workshop. If you're using a Windows system, convert the .pem file to a puTTY .ppk file. If you're not sure how to do this, instructions are available [here](https://aws.amazon.com/premiumsupport/knowledge-center/convert-pem-file-into-ppk/). -## LAUNCH THE CLOUDFORMATION TEMPLATE +## Launchh the CloudFormation template So that you can concentrate on the aspects of this workshop that directly relate to Amazon EC2 Spot instances, there is a CloudFormation template that will deploy the base AWS infrastructure needed for all of the labs within the workshop - saving you from having to create things like VPCs, Security Groups, IAM policies and so forth. Download and deploy the CloudFormation template: -[amazon-ec2-spot-cicd-workshop.yaml](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml) +[amazon-ec2-spot-cicd-workshop-asg.yaml](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml) Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supply appropriate parameters when prompted. @@ -38,4 +38,15 @@ Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supp The stack should take around five minutes to deploy. -Once the CloudFormation is in the process of being deployed, you've completed all of the preparation required to start the workshop, you may proceed. +## Setting Up environment variables +You need to set up the following environment variables that you'll use in the workshop, to do so, run the following commands: + +```bash +export PRIVATE_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPrivateSubnets'].OutputValue" --output text); +export PUBLIC_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPublicSubnets'].OutputValue" --output text); +export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +``` + +{{% notice note %}} +If for some reason the environment variables are cleared, you can run the previous commands again without any problem. +{{% /notice %}} diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md index ec069eb5..ccc9d646 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md @@ -17,4 +17,6 @@ Now it’s time to test out how Jenkins handles pushing builds to spot instances 3. After around four minutes, the **EC2 Fleet Status** reported to the left of the screen will increment the **target** count to 5, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, five build instances will appear in the **Build Executor Status**, though these build agents will initially appear to be offline. Once the instances have had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to them via SSH, and it will come online and process the next build jobs in the queue. Once you have concurrent builds being executed on the Spot instances, you can stop adding build jobs to the build queue; 4. After a period of around a five minutes, after your builds have completed, the Spot instances should be terminated by the plugin. -**Note:** If the build agents are keep showing as `offline`, make sure that you've configured correctly the SSH key pair in the credentials section when configuring the EC2 Fleet plugin. You can click on the agent to open the logs and see why it's not coming online. \ No newline at end of file +{{% notice note %}} +If the build agents are keep showing as `offline`, make sure that you've configured correctly the SSH key pair in the credentials section when configuring the EC2 Fleet plugin. You can click on the agent to open the logs and see why it's not coming online. +{{% /notice %}} \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index 8b84ea93..c3e2474c 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -105,7 +105,7 @@ After a few moments, your Spot instance will start up and should attach itself t 2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the Auto Scaling group that you just created. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; 3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. -## Test theh self-healing architecture running with Spot instances +## Test the self-healing architecture running with Spot instances At some point in time, you might receive a Spot interruption notice when On-Demand needs the capacity back. When this happens, your Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, the Auto Scaling group will observe that there are no running instances in the group and because the desired capacity is one, it will attempt to launch a replacement instance the pool with more capacity available (this is why diversification across many capacity pool is a best practice). The new instance will be launched and bootstrapped in exactly the same manner as your original instance. Test this out by terminating the Spot instance with the **Jenkins Master (Spot)** name tag and verifying that a replacement instance comes up to take its place. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md index cc28c1ea..f9cd4ed3 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md @@ -1,5 +1,5 @@ +++ -title = "Jenkins on ECS" +title = "Setup with CloudFormation" weight = 210 +++ You’ve now got a scalable solution using nothing but Spot instances for your CICD systems, your build agents and your test environments – however, you still have some inefficiencies with this setup: diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml new file mode 100644 index 00000000..58ec8ec6 --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -0,0 +1,651 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: A CloudFormation template that will deploy all AWS resources that are required to run the Amazon EC2 Spot CI/CD Workshop. This template is provided as-is under a modified MIT license - please see https://github.com/aws-samples/amazon-ec2-spot-cicd-workshop/blob/master/LICENSE + +Parameters: + KeyPair: + Description: The Key Pair created earlier in the Preparation Lab + Type: AWS::EC2::KeyPair::KeyName + + CurrentIP: + Description: Your current IP address (used to limit access to SSH services on EC2 instances) + Type: String + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\/([0-9]|[1-2][0-9]|3[0-2])$ + ConstraintDescription: must be specified in CIDR notation (e.g, 123.45.67.0/24) + Default: 0.0.0.0/0 + + JenkinsAdminPassword: + Description: The password that you would like to use for the admin account in your Jenkins server (must be at least 8 characters) + Type: String + ConstraintDescription: must be at least 8 characters in length + MinLength: 8 + NoEcho: true + + AmazonLinux2LatestAmiId: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 + +Resources: + IAMRoleJenkins: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Policies: + - PolicyName: JenkinsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - ec2:DescribeSpotFleetInstances + - ec2:ModifySpotFleetRequest + - ec2:CreateTags + - ec2:DescribeRegions + - ec2:DescribeInstances + - ec2:TerminateInstances + - ec2:DescribeInstanceStatus + - ec2:DescribeSpotFleetRequests + Resource: "*" + - Effect: Allow + Action: + - autoscaling:DescribeAutoScalingGroups + - autoscaling:UpdateAutoScalingGroup + Resource: "*" + - Effect: Allow + Action: + - iam:ListInstanceProfiles + - iam:ListRoles + - iam:PassRole + Resource: "*" + + InstanceProfileJenkins: + Type: AWS::IAM::InstanceProfile + DependsOn: IAMRoleJenkins + Properties: + Path: "/" + Roles: + - !Ref IAMRoleJenkins + + VPC: # This is the VPC that the CI/CD environment will be running in + Type: AWS::EC2::VPC + Properties: + CidrBlock: 192.168.0.0/21 + EnableDnsHostnames: "true" + EnableDnsSupport: "true" + InstanceTenancy: default + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop VPC + + SubnetPublicA: # The first of three subnets defined within the VPC... + Type: AWS::EC2::Subnet + DependsOn: VPC + # DependedOn: SubnetPublicARouteTableAssociation + Properties: + AvailabilityZone: !Select [0, !GetAZs ""] + CidrBlock: 192.168.0.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Public Subnet A + + SubnetPublicB: # ... and the second of three subnets... + Type: AWS::EC2::Subnet + DependsOn: VPC + # DependedOn: SubnetPublicBRouteTableAssociation + Properties: + AvailabilityZone: !Select [1, !GetAZs ""] + CidrBlock: 192.168.1.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Public Subnet B + + SubnetPublicC: # ... and the third of three subnets defined within the VPC + Type: AWS::EC2::Subnet + DependsOn: VPC + # DependedOn: SubnetPublicCRouteTableAssociation + Properties: + AvailabilityZone: !Select [2, !GetAZs ""] + CidrBlock: 192.168.2.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Public Subnet C + + SubnetPrivateA: # The first of three subnets defined within the VPC... + Type: AWS::EC2::Subnet + DependsOn: VPC + Properties: + AvailabilityZone: !Select [0, !GetAZs ""] + CidrBlock: 192.168.3.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Private Subnet A + - Key: Type + Value: Private + + SubnetPrivateB: # ... and the second of three subnets... + Type: AWS::EC2::Subnet + DependsOn: VPC + Properties: + AvailabilityZone: !Select [1, !GetAZs ""] + CidrBlock: 192.168.4.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Private Subnet B + - Key: Type + Value: Private + + SubnetPrivateC: # ... and the third of three subnets defined within the VPC + Type: AWS::EC2::Subnet + DependsOn: VPC + Properties: + AvailabilityZone: !Select [2, !GetAZs ""] + CidrBlock: 192.168.5.0/24 + MapPublicIpOnLaunch: "true" + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Private Subnet C + - Key: Type + Value: Private + + InternetGateway: # Create an Internet Gateway in order to allow EC2 instances to be accessible via the Internet + Type: AWS::EC2::InternetGateway + Properties: + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Internet Gateway + + InternetGatewayAttachment: # Once both the VPC and Internet Gateway have been created, attach the Internet Gateway to the VPC + Type: AWS::EC2::VPCGatewayAttachment + DependsOn: + - InternetGateway + - VPC + Properties: + InternetGatewayId: !Ref InternetGateway + VpcId: !Ref VPC + + EIPNATGateway: + Type: AWS::EC2::EIP + DependsOn: + - InternetGatewayAttachment + - VPC + Properties: + Domain: vpc + + NATGateway: + Type: AWS::EC2::NatGateway + DependsOn: + - SubnetPublicA + Properties: + AllocationId: !GetAtt EIPNATGateway.AllocationId + SubnetId: !Ref SubnetPublicA + + RouteTablePublic: # Create a route table which will be used within the VPC + Type: AWS::EC2::RouteTable + DependsOn: VPC + Properties: + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Public Route Table + + RouteTablePrivate: # Create a route table which will be used within the VPC + Type: AWS::EC2::RouteTable + DependsOn: VPC + Properties: + VpcId: !Ref VPC + Tags: + - Key: Name + Value: Amazon EC2 Spot CICD Workshop Private Route Table + + DefaultRoutePublic: # Add a default route to the route table, pointing to the Internet Gateway + Type: AWS::EC2::Route + DependsOn: + - RouteTablePublic + - InternetGatewayAttachment + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref RouteTablePublic + + DefaultRoutePrivate: # Add a default route to the route table, pointing to the NAT Gateway + Type: AWS::EC2::Route + DependsOn: + - RouteTablePrivate + - NATGateway + Properties: + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: !Ref NATGateway + RouteTableId: !Ref RouteTablePrivate + + SubnetPublicARouteTableAssociation: # Associate the route table with the first of the three subnets in the VPC... + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePublic + - SubnetPublicA + Properties: + RouteTableId: !Ref RouteTablePublic + SubnetId: !Ref SubnetPublicA + + SubnetPublicBRouteTableAssociation: # ... and the second of three subnets... + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePublic + - SubnetPublicB + Properties: + RouteTableId: !Ref RouteTablePublic + SubnetId: !Ref SubnetPublicB + + SubnetPublicCRouteTableAssociation: # ... and the third of the three subnets in the VPC + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePublic + - SubnetPublicC + Properties: + RouteTableId: !Ref RouteTablePublic + SubnetId: !Ref SubnetPublicC + + SubnetPrivateARouteTableAssociation: # Associate the route table with the first of the three subnets in the VPC... + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePrivate + - SubnetPrivateA + Properties: + RouteTableId: !Ref RouteTablePrivate + SubnetId: !Ref SubnetPrivateA + + SubnetPrivateBRouteTableAssociation: # ... and the second of three subnets... + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePrivate + - SubnetPrivateB + Properties: + RouteTableId: !Ref RouteTablePrivate + SubnetId: !Ref SubnetPrivateB + + SubnetPrivateCRouteTableAssociation: # ... and the third of the three subnets in the VPC + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: + - RouteTablePrivate + - SubnetPrivateC + Properties: + RouteTableId: !Ref RouteTablePrivate + SubnetId: !Ref SubnetPrivateC + + SecurityGroupJenkins: # A Security Group that allows ingress access for SSH and the default port that a Jenkins Master will run on + Type: AWS::EC2::SecurityGroup + DependsOn: VPC + Properties: + GroupName: Spot CICD Workshop Jenkins Security Group + GroupDescription: A Security Group that allows ingress access for SSH and the default port that a Jenkins Master will run on + SecurityGroupIngress: + - Description: SSH access from the IP address that the workshop participant is using. + IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref CurrentIP + - Description: SSH access from hosts within the Spot CICD Workshop VPC - required for Build Agent deployments on to EC2 Spot instances. + IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 192.168.0.0/21 + - Description: HTTP (8080) access from the IP address that the workshop participant is using. + IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + CidrIp: !Ref CurrentIP + - Description: HTTP (8080) access from hosts within the Spot CICD Workshop VPC, including the ALBs that reside in front of Jenkins, and Build Agents running on EC2 nodes. + IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + CidrIp: 192.168.0.0/21 + - Description: JNLP (5000) access from hosts within the Spot CICD Workshop VPC - required for Build Agent deployments on EC2 nodes to communicate back with the Jenkins Master container. + IpProtocol: tcp + FromPort: 5000 + ToPort: 5000 + CidrIp: 0.0.0.0/0 + - Description: JNLP (50000) access from hosts within the Spot CICD Workshop VPC - the legacy port that used to be required for Build Agent deployments on EC2 nodes to communicate back with the Jenkins Master container. + IpProtocol: tcp + FromPort: 50000 + ToPort: 50000 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VPC + + SecurityGroupJenkinsALB: # A Security Group that allows ingress access for HTTP on ALBs and used to access the Jenkins Master + Type: AWS::EC2::SecurityGroup + DependsOn: VPC + Properties: + GroupName: Spot CICD Workshop Jenkins ALB Security Group + GroupDescription: A Security Group that allows ingress access for HTTP on ALBs and used to access the Jenkins Master + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VPC + + SecurityGroupEFS: + Type: AWS::EC2::SecurityGroup + DependsOn: + - SecurityGroupJenkins + - VPC + Properties: + GroupName: Spot CICD Workshop EFS Security Group + GroupDescription: A Security Group that allows access to EFS volume targets from the Jenkins Securiy Group + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 2049 + ToPort: 2049 + SourceSecurityGroupId: !Ref SecurityGroupJenkins + VpcId: !Ref VPC + + JenkinsOnDemandEC2Instance: # This workshop starts from a baseline where we have a Jenkins server running on an on-demand EC2 instance. This resource launches and bootstraps this server + Type: AWS::EC2::Instance + DependsOn: + - InstanceProfileJenkins + - SecurityGroupJenkins + - SubnetPublicA + Properties: + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + DeleteOnTermination: "true" + VolumeSize: 50 + VolumeType: gp2 + IamInstanceProfile: !Ref InstanceProfileJenkins + ImageId: !Ref AmazonLinux2LatestAmiId + InstanceType: t3.medium + KeyName: !Ref KeyPair + SecurityGroupIds: + - !Ref SecurityGroupJenkins + SubnetId: !Ref SubnetPublicA + Tags: + - Key: Name + Value: Jenkins Master (On-demand) + UserData: + Fn::Base64: !Sub | + #!/bin/bash + # Install all pending updates to the system + yum -y update + # Install Docker + amazon-linux-extras install docker + chkconfig docker on + service docker start + usermod -a -G docker ec2-user + # Install Docker Compose + curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + # Download files from GitHub + mkdir -p /jenkins_home/jobs/ + mkdir -p /usr/share/jenkins + cd /usr/share/jenkins + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz + tar -xvf seed.tar.gz + cp -r jobs/* /jenkins_home/jobs/ + # Reset the password defined as the JenkinsAdminPassword + sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml + # Configure the Jenkins Location + curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml + # Start Jenkins + docker-compose up --build -d + + JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + DependsOn: + - SecurityGroupJenkinsALB + - SubnetPublicA + - SubnetPublicB + - SubnetPublicC + Properties: + Name: JenkinsMasterALB + Scheme: internet-facing + SecurityGroups: + - !Ref SecurityGroupJenkinsALB + Subnets: + - !Ref SubnetPublicA + - !Ref SubnetPublicB + - !Ref SubnetPublicC + + JenkinsMasterALBTargetGroupEC2: # This is the Target Group used by the JenkinsMasterALB load balancer when Jenkins is running on an EC2 instance + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: + - JenkinsOnDemandEC2Instance + - VPC + Properties: + HealthCheckIntervalSeconds: 15 + HealthCheckPath: /login + HealthCheckPort: 8080 + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 5 + HealthyThresholdCount: 2 + Matcher: + HttpCode: 200 + Name: JenkinsMasterEC2TargetGroup + Port: 8080 + Protocol: HTTP + Targets: + - Id: !Ref JenkinsOnDemandEC2Instance + Port: 8080 + UnhealthyThresholdCount: 4 + VpcId: !Ref VPC + + JenkinsMasterALBListener: # This is the ALB Listener used to access the Jenkins Master + Type: AWS::ElasticLoadBalancingV2::Listener + DependsOn: + - JenkinsMasterALB + - JenkinsMasterALBTargetGroupEC2 + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref JenkinsMasterALBTargetGroupEC2 + LoadBalancerArn: !Ref JenkinsMasterALB + Port: 80 + Protocol: HTTP + + JenkinsMasterALBListenerRuleEC2: # The ALB Listener rule that forwards all traffic destined for the Jenkins Master to the appropriate Target Group + Type: AWS::ElasticLoadBalancingV2::ListenerRule + DependsOn: + - JenkinsMasterALBListener + - JenkinsMasterALBTargetGroupEC2 + Properties: + Actions: + - Type: forward + TargetGroupArn: !Ref JenkinsMasterALBTargetGroupEC2 + Conditions: + - Field: path-pattern + Values: + - "/*" + ListenerArn: !Ref JenkinsMasterALBListener + Priority: 1 + + JenkinsSpotMasterLaunchTemplate: # This is a launch template that will be used to provision Jenkins Master servers - showing how when used in conjunction with an EFS volume stateful applications can run on self-healing spot architectures. + Type: AWS::EC2::LaunchTemplate + DependsOn: + - EFSJenkinsHomeVolume + - InstanceProfileJenkins + - SecurityGroupJenkins + Properties: + LaunchTemplateName: JenkinsMasterLaunchTemplate + LaunchTemplateData: + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + DeleteOnTermination: "true" + VolumeSize: 8 + VolumeType: gp2 + IamInstanceProfile: + Name: !Ref InstanceProfileJenkins + ImageId: !Ref AmazonLinux2LatestAmiId + InstanceType: t3.medium + KeyName: !Ref KeyPair + SecurityGroupIds: + - !Ref SecurityGroupJenkins + TagSpecifications: + - ResourceType: instance + Tags: + - Key: Name + Value: Jenkins Master (Spot) + UserData: + Fn::Base64: !Sub | + #!/bin/bash + # Install all pending updates to the system + yum -y update + # Install Docker + amazon-linux-extras install docker + chkconfig docker on + service docker start + usermod -a -G docker ec2-user + # Install Docker Compose + curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + # Mount the Jenkins EFS volume + mkdir -p /jenkins_home + mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFSJenkinsHomeVolume}.efs.eu-west-1.amazonaws.com:/ /jenkins_home + # Download files from GitHub + mkdir -p /usr/share/jenkins + cd /usr/share/jenkins + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile + wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt + # Reset the password defined as the JenkinsAdminPassword + sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml + # Configure the Jenkins Location + curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml + # Start Jenkins + docker-compose up --build -d + + JenkinsSpotAgentLaunchTemplate: # This is a launch template that will be used to provision Jenkins build agents - showing how spot instances can be used to scale-out build jobs at low cost. + Type: AWS::EC2::LaunchTemplate + DependsOn: + - InstanceProfileJenkins + - SecurityGroupJenkins + Properties: + LaunchTemplateName: JenkinsBuildAgentLaunchTemplate + LaunchTemplateData: + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + DeleteOnTermination: "true" + VolumeSize: 8 + VolumeType: gp2 + IamInstanceProfile: + Name: !Ref InstanceProfileJenkins + ImageId: !Ref AmazonLinux2LatestAmiId + InstanceType: t3.small + KeyName: !Ref KeyPair + SecurityGroupIds: + - !Ref SecurityGroupJenkins + TagSpecifications: + - ResourceType: instance + Tags: + - Key: Name + Value: Jenkins Build Agent + UserData: + Fn::Base64: !Sub | + #!/bin/bash + # Install all pending updates to the system + yum -y update + # Configure YUM to be able to access contributed Maven RPM packages + wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo + # Update the release version in the Maven repository configuration for this mainline release of Amazon Linux + sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo + # Install the Java 8 SDK, Git and Maven + yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel git apache-maven + # Set the default version of java to run out of the Java 8 SDK path (required by Jenkins) + update-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac + + EFSJenkinsHomeVolume: + Type: AWS::EFS::FileSystem + Properties: + PerformanceMode: generalPurpose + + EFSMountTargetJenkinsHomeVolumeA: + Type: AWS::EFS::MountTarget + DependsOn: + - EFSJenkinsHomeVolume + - SecurityGroupEFS + - SubnetPublicA + Properties: + FileSystemId: !Ref EFSJenkinsHomeVolume + SecurityGroups: + - !Ref SecurityGroupEFS + SubnetId: !Ref SubnetPublicA + + EFSMountTargetJenkinsHomeVolumeB: + Type: AWS::EFS::MountTarget + DependsOn: + - EFSJenkinsHomeVolume + - SecurityGroupEFS + - SubnetPublicB + Properties: + FileSystemId: !Ref EFSJenkinsHomeVolume + SecurityGroups: + - !Ref SecurityGroupEFS + SubnetId: !Ref SubnetPublicB + + EFSMountTargetJenkinsHomeVolumeC: + Type: AWS::EFS::MountTarget + DependsOn: + - EFSJenkinsHomeVolume + - SecurityGroupEFS + - SubnetPublicC + Properties: + FileSystemId: !Ref EFSJenkinsHomeVolume + SecurityGroups: + - !Ref SecurityGroupEFS + SubnetId: !Ref SubnetPublicC + +Outputs: + EFSFileSystemID: + Description: The file system ID of the EFS volume that is used to persist JENKINS_HOME across EC2. + Value: !Ref EFSJenkinsHomeVolume + + JenkinsDNSName: + Description: The DNS name of the Application Load Balancer that is used to gain access to your Jenkins server. + Value: !GetAtt JenkinsMasterALB.DNSName + + JenkinsIAMRoleARN: + Description: The ARN associated with the IAM Role that was created for use by Jenkins. + Value: !GetAtt IAMRoleJenkins.Arn + + JenkinsVPCPrivateSubnets: + Description: The private subnets where Jenkins will be deployed. Use this value to configure Jenkins Spot agents. + Value: + !Join [ + ",", + [!Ref SubnetPrivateA, !Ref SubnetPrivateB, !Ref SubnetPrivateC], + ] + + JenkinsVPCPublicSubnets: + Description: The public subnets where Jenkins will be deployed. + Value: + !Join [",", [!Ref SubnetPublicA, !Ref SubnetPublicB, !Ref SubnetPublicC]] + + JenkinsMasterEC2TargetGroup: + Description: Target Group for Jenkins EC2 nodes. + Value: !Ref JenkinsMasterALBTargetGroupEC2 diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml similarity index 100% rename from workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop.yaml rename to workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml From 87860deb0dcade877d4ccc8f08bef7c4fda4b31e Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 9 Aug 2022 18:11:07 +0200 Subject: [PATCH 16/29] Add ABS to the CI/CD Workshop --- .../jenkins-asg/asg.md | 41 ++++++++----------- .../jenkins-asg/ec2-fleet-jenkins.md | 2 +- .../jenkins-asg/test-build.md | 2 +- .../jenkins-asg/test-persistence.md | 23 +++-------- 4 files changed, 24 insertions(+), 44 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md index 146efea7..d4c5676e 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -1,5 +1,5 @@ +++ -title = "Provision an EC2 Auto Scaling Group" +title = "Launch an Auto Scaling group" weight = 110 +++ An Auto Scaling group contains a collection of Amazon EC2 Instances that are treated as a logical grouping for the purposes of automatic scaling and management. An Auto Scaling group also enables you to use Amazon EC2 Auto Scaling features such as health check replacements and scaling policies. Both maintaining the number of instances in an Auto Scaling group and automatic scaling are the core functionality of the Amazon EC2 Auto Scaling service. @@ -9,9 +9,11 @@ Amazon EC2 Auto Scaling helps you ensure that you have the correct number of Ama When adopting EC2 Spot Instances, our recommendation is use Auto Scaling groups since it offers a very rich API with benefits like scale-in protection, health checks, lifecycle hooks, rebalance recommendation integration, warm pools and predictive scaling, and many more functionalities that we list below. ## Launching an Auto Scaling group -For this workshop, you need to create an Auto Scaling group that will be used by the Jenkins plugin to perform your application builds. To apply Spot best practices we will launch a mixed instance Auto Scaling group using Spot. This first step does create a *json* file. The file describes a mixed-instance-policy section with a set of overrides that drive **diversification of Spot Instance pools**. The configuration of the Auto Scaling group does refer to the Launch Template that was created in the previous steps with Cloudformation. +For this workshop, you need to create an Auto Scaling group that will be used by the Jenkins plugin to perform your application builds. One of the key Spot best practice is to select multiple instance types. At AWS, instance types comprise varying combinations of CPU, memory, storage, and networking capacity to give you the flexibility to choose the appropriate mix of resources for your applications. However, to make this selection simpler, AWS released **[Attribute-Based Instance Type Selection (ABS)](https://aws.amazon.com/blogs/aws/new-attribute-based-instance-type-selection-for-ec2-auto-scaling-and-ec2-fleet/)** to express workload requirements as a set of instance attributes such as: vCPU, memory, type of processor, etc. ABS translates these requirements and selects all matching instance types that meet the criteria. To select which instance to launch, the Auto Scaling group chose instances based on the allocation strategy configured. For Spot Instances we recommend to use **capacity-optimized**, which select the optimal instances that reduce the frequency of interruptions. ABS does also future-proof the Auto Scaling group configuration: *any new instance type we launch that matches the selected attributes, will be included in the list automatically*. No need to update the Auto Scaling group configuration. -Run the following command: +To launch the Auto Scaling group, first create a *json* configuration file. This file describes a mixed-instance-policy section with a set of overrides that drive **diversification of Spot Instance pools** using ABS. The configuration of the Auto Scaling group does refer to the Launch Template that was created in the previous steps with Cloudformation, so make sure you have the corresponding value in the `LAUNCH_TEMPLATE_ID` environment variable + +Here's the command you need to run to create the configuration file: ```bash cat < ~/asg-policy.json @@ -23,22 +25,9 @@ cat < ~/asg-policy.json }, "Overrides":[ { - "InstanceType":"t2.large" - }, - { - "InstanceType":"t3.large" - }, - { - "InstanceType":"m4.large" - }, - { - "InstanceType":"m5.large" - }, - { - "InstanceType":"c5.large" - }, - { - "InstanceType":"c4.large" + "InstanceRequirements": { + "VCpuCount":{"Min": 2, "Max": 8}, + "MemoryMiB":{"Min": 4096} } } ] }, @@ -51,14 +40,18 @@ cat < ~/asg-policy.json EoF ``` -In the override section, choose as many instances that qualify for your application as possible, in this case we selected a group of 6 instance types that meet the “.large” criteria. +Notice that in the override section is where you specify the instance attributes with a range of 2 to 8 vCPUs, and a minimum of 4 GBs of memory. ABS will pick a list of instance types that match the criteria like `m5.large`, `c4.large`, or `c5.large`. Then, based on the **capacity-optimized** allocation strategy, the Auto Scaling group will launch instances from pool with more spare capacity available. + +With Auto Scaling groups you can define what is the balance between Spot vs On-Demand Instances that makes sense for your workload. `OnDemandBaseCapacity` allows you to set an initial capacity of On-Demand Instances to use. After that, any new procured capacity will be a mix of Spot and On-Demand Instances as defined by the `OnDemandPercentageAboveBaseCapacity`. This time, *we've configured the Auto Scaling group to launch only Spot instances*. -With Auto Scaling groups you can define what is the balance between Spot vs On-Demand Instances that makes sense for your workload. OnDemandBaseCapacity allows you to set an initial capacity of On-Demand Instances to use. After that, any new procured capacity will be a mix of Spot and On-Demand Instances as defined by the OnDemandPercentageAboveBaseCapacity. This time, we configured the Auto Scaling group to launch only Spot instances. +Finally, the configuration above, sets the `SpotAllocationStrategy` to `capacity-optimized`. The `capacity-optimized` allocation strategy **allocates instances from the Spot Instance pools with the optimal capacity for the number of instances that are launching**, making use of real-time capacity data and optimizing the selection of used Spot Instances. You can read about the benefits of using `capcity-optimized` in the blog post [Capacity-Optimized Spot Instance allocation in action at Mobileye and Skyscanner](https://aws.amazon.com/blogs/aws/capacity-optimized-spot-instance-allocation-in-action-at-mobileye-and-skyscanner/). -Additionally, the configuration above, sets the `SpotAllocationStrategy` to `capacity-optimized`. The `capacity-optimized` allocation strategy **allocates instances from the Spot Instance pools with the optimal capacity for the number of instances that are launching**, making use of real-time capacity data and optimizing the selection of used Spot Instances. You can read about the benefits of using `capcity-optimized` in the blog post [Capacity-Optimized Spot Instance allocation in action at Mobileye and Skyscanner](https://aws.amazon.com/blogs/aws/capacity-optimized-spot-instance-allocation-in-action-at-mobileye-and-skyscanner/). +Let’s now create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (*to avoid running instances when there's no need*), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. -Let’s create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (to avoid having instances when there's no need), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. +Run the following command: ```bash aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${PRIVATE_SUBNETS}" --mixed-instances-policy file://asg-policy.json -``` \ No newline at end of file +``` + +Nothing else to do here, it's time to configure Jenkins to use this Auto Scaling group. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md index f4635da9..68a2fd53 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md @@ -1,5 +1,5 @@ +++ -title = "Configure the EC2 Fleet Jenkins Plugin" +title = "Configure EC2 Fleet plugin" weight = 115 +++ The [EC2 Fleet Plugin](https://plugins.jenkins.io/ec2-fleet/) launches EC2 Spot or On Demand instances as worker nodes for Jenkins CI server, automatically scaling the capacity with the load. The EC2 FLeet plugin will request EC2 instances when excess jobs are detected. You can configure the plugin to use an Auto Scaling Group to launch instances instead of directly launching them by itself. This gives theh plugin all the benefits from Auto Scaling groups like allocation strategies, configure multiple instance types and availability zones, etc. Moreover, the EC2 Fleet plugin can automatically resubmit failed jobs caused by a Spot interruption. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md index ccc9d646..8df4dae5 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-build.md @@ -1,5 +1,5 @@ +++ -title = "Configure Build Jobs to use Spot" +title = "Configure jobs to use Spot" weight = 120 +++ As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index c3e2474c..fd802073 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -1,5 +1,5 @@ +++ -title = "Configure Persistence Storage when using Spot" +title = "Configure Persistence Storage" weight = 125 +++ You're now using Spot instances for your code builds – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by an Auto Scaling group. @@ -40,7 +40,7 @@ sudo cp -rpv /jenkins_home/* /mnt ``` ## Provision an Auto Scaling group for the new Jenkins host -The Auto Scaling group that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents. Additionally, you'll configure this Auto Scaling group so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. +The Auto Scaling group that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents: launch only Spot instances with the capacity-optimized strategy, and using ABS to increase diversification. Additionally, you'll configure this Auto Scaling group so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. Run the following commands to create the Auto Scaling group configuration file: @@ -55,22 +55,9 @@ cat < ~/asg-jenkins-host-policy.json }, "Overrides":[ { - "InstanceType":"t2.large" - }, - { - "InstanceType":"t3.large" - }, - { - "InstanceType":"m4.large" - }, - { - "InstanceType":"m5.large" - }, - { - "InstanceType":"c5.large" - }, - { - "InstanceType":"c4.large" + "InstanceRequirements": { + "VCpuCount":{"Min": 2, "Max": 8}, + "MemoryMiB":{"Min": 4096} } } ] }, From 48dd93371cee2e2b44a0ff737ccfd133f64da1d3 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 10 Aug 2022 17:53:57 +0200 Subject: [PATCH 17/29] Add FIS to the CI/CD Workshop --- .../jenkins-asg/clean.md | 6 +- .../spot_interruption_experiment.md | 105 ++++++++++++++++++ .../jenkins-asg/spot_interruption_fis.md | 54 +++++++++ .../jenkins-asg/test-persistence.md | 13 +-- .../tracking_spot_interruptions.md | 36 ++++++ .../spotinterruptionlog.png | Bin 0 -> 109177 bytes .../fisspotinterruption.yaml | 69 ++++++++++++ 7 files changed, 269 insertions(+), 14 deletions(-) create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/tracking_spot_interruptions.md create mode 100644 static/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png create mode 100644 workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md index 1c1da386..c8d78338 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/clean.md @@ -1,6 +1,6 @@ +++ title = "Cleanup" -weight = 150 +weight = 160 +++ Congratulations, you have completed the Jenkins with Auto Scaling group lab! Your next challenge is to remove all of the resources that were provisioned in your account so as to ensure that no additional cost can be incurred. Please note that the commands below should be executed in order - some later steps have dependencies on earlier ones! @@ -11,10 +11,12 @@ aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenki aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsHostASG; ``` -## Delete theh CloudFormation stack +## Delete the CloudFormation stacks ```bash aws cloudformation delete-stack --stack-name SpotCICDWorkshop; +aws cloudformation delete-stack --stack-name track-spot-interruption; +aws cloudformation delete-stack --stack-name fis-spot-interruption; ``` ## Remove theh EC2 key pair diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md new file mode 100644 index 00000000..4ec06939 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md @@ -0,0 +1,105 @@ +--- +title: "Creating the Spot Interruption Experiment" +weight: 145 +--- + +In this section, you're going to start creating the experiment to [trigger the interruption of Amazon EC2 Spot Instances using AWS Fault Injection Simulator (FIS)](https://aws.amazon.com/blogs/compute/implementing-interruption-tolerance-in-amazon-ec2-spot-with-aws-fault-injection-simulator/). When using Spot instances, you need to be prepared to be interrupted. With FIS, you can test the resiliency of your workload and validate that your application is reacting to the interruption notices that EC2 sends before terminating your instances. You can target individual Spot instances or a subset of instances from an Auto Scaling group. + +#### What do you need to get started? + +Before you start launching Spot interruptions with FIS, you need to create an experiment template. Here is where you define which resources you want to interrupt (targets), and when you want to interrupt the instance. + +You're going to use the following CloudFormation template which creates the IAM role (`FISSpotRole`) with the minimum permissions FIS needs to interrupt an instance, and the experiment template (`FISExperimentTemplate`) you're going to use to trigger a Spot interruption: + +``` +AWSTemplateFormatVersion: 2010-09-09 +Description: FIS for Spot Instances +Parameters: + InstancesToInterrupt: + Description: Number of instances to interrupt + Default: 3 + Type: Number + + DurationBeforeInterruption: + Description: Number of minutes before the interruption + Default: 2 + Type: Number + +Resources: + + FISSpotRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: [fis.amazonaws.com] + Action: ["sts:AssumeRole"] + Path: / + Policies: + - PolicyName: root + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: 'ec2:DescribeInstances' + Resource: '*' + - Effect: Allow + Action: 'ec2:SendSpotInstanceInterruptions' + Resource: 'arn:aws:ec2:*:*:instance/*' + + FISExperimentTemplate: + Type: AWS::FIS::ExperimentTemplate + Properties: + Description: "Interrupt multiple random instances" + Targets: + SpotIntances: + ResourceTags: + Name: "Jenkins Master (Spot)" + Filters: + - Path: State.Name + Values: + - running + ResourceType: aws:ec2:spot-instance + SelectionMode: !Join ["", ["COUNT(", !Ref InstancesToInterrupt, ")"]] + Actions: + interrupt: + ActionId: "aws:ec2:send-spot-instance-interruptions" + Description: "Interrupt multiple Spot instances" + Parameters: + durationBeforeInterruption: !Join ["", ["PT", !Ref DurationBeforeInterruption, "M"]] + Targets: + SpotInstances: SpotIntances + StopConditions: + - Source: none + RoleArn: !GetAtt FISSpotRole.Arn + Tags: + Name: "fis-spot-interruption" + +Outputs: + FISExperimentID: + Value: !GetAtt FISExperimentTemplate.Id +``` + +Here are some important notes about the template: + +* You can configure how many instances you want to interrupt with the `InstancesToInterrupt` parameter. In the template it's defined that it's going to interrupt **three** instances. +* You can also configure how much time you want the experiment to run with the `DurationBeforeInterruption` parameter. By default, it's going to take two minutes. This means that as soon as you launch the experiment, the instance is going to receive the two-minute notification Spot interruption warning. +* The most important section is the `Targets` from the experiment template. This is where you tell FIS which instances to interrupt. In this case, we're simply going to say that we want to interrupt a instance that's `running` and has a `Name` tag with value `Jenkins Master (Spot)`. +* Notice that instances are going to be **chosen randomly** + +#### Create the EC2 Spot Interruption Experiment with FIS + +Let's continue by creating the Spot interruption experiment template using Cloudformation. You can view the CloudFormation template (**fisspotinterruption.yaml**) at GitHub [here](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml). To download it, you can run the following command: + +``` +wget https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml +``` + +Now, simply run the following commands to create the FIS experiment: + +``` +aws cloudformation create-stack --stack-name fis-spot-interruption --template-body file://fisspotinterruption.yaml --capabilities CAPABILITY_NAMED_IAM +aws cloudformation wait stack-create-complete --stack-name fis-spot-interruption +``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md new file mode 100644 index 00000000..9d6e97cb --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md @@ -0,0 +1,54 @@ +--- +title: "Interrupting a Spot Instance" +weight: 150 +--- + +In this section, you're going to launch a Spot Interruption using FIS and then verify that the capacity has been replenished and Jenkins is still working. This will help you to confirm the low impact of your workloads when implementing Spot effectively. + +#### Launch the Spot Interruption Experiment +After creating the experiment template in FIS, you can start a new experiment to interrupt one Spot instance. Run the following command: + +``` +FIS_EXP_TEMP_ID=$(aws cloudformation describe-stacks --stack-name fis-spot-interruption --query "Stacks[0].Outputs[?OutputKey=='FISExperimentID'].OutputValue" --output text) +FIS_EXP_ID=$(aws fis start-experiment --experiment-template-id $FIS_EXP_TEMP_ID --no-cli-pager --query "experiment.id" --output text) +``` + +Wait around 30 seconds, and you should see that the experiment completes. Run the following command to confirm: + +``` +aws fis get-experiment --id $FIS_EXP_ID --no-cli-pager +``` + +At this point, FIS has triggered a Spot interruption notice, and in two minutes the instance will be terminated. + +Go to CloudWatch Logs group `/aws/events/spotinterruptions` to confirm that instance received the termination notice. + +You should see a log message like this one: + +![SpotInterruptionLog](/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png) + +#### Verify the actions taken by Jenkins Auto Scaling group + +You are running a Jenkins instance launched by an Auto Scaling group that will launch a new instance if the desired capacity is not compliant. + +You can run the following command to see the list of instances with the date and time when they were launched. + +``` +aws ec2 describe-instances --filters\ + Name='tag:Name',Values='Jenkins Master (Spot)'\ + Name='instance-state-name',Values='running'\ + | jq '.Reservations[].Instances[] | "Instance with ID:\(.InstanceId) launched at \(.LaunchTime)"' +``` + +You should see a list of instances with the date and time when they were launched. You'll also see the new instance launched after the interruption. + +```output +"Instance with ID:i-04c6ced30794e965b launched at 2022-04-06T14:02:49+00:00" +"Instance with ID:i-0136152e14053af81 launched at 2022-04-06T14:11:25+00:00" +``` + +#### Verify that Jenkins is still working + +Login to the Jenkins server again and start running some jobs as you did previously. + +You should see that the EC2 Fleet plugin is still working and is launching Spot instances as build agents. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md index fd802073..285463af 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md @@ -92,15 +92,4 @@ After a few moments, your Spot instance will start up and should attach itself t 2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the Auto Scaling group that you just created. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; 3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. -## Test the self-healing architecture running with Spot instances -At some point in time, you might receive a Spot interruption notice when On-Demand needs the capacity back. When this happens, your Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, the Auto Scaling group will observe that there are no running instances in the group and because the desired capacity is one, it will attempt to launch a replacement instance the pool with more capacity available (this is why diversification across many capacity pool is a best practice). The new instance will be launched and bootstrapped in exactly the same manner as your original instance. - -Test this out by terminating the Spot instance with the **Jenkins Master (Spot)** name tag and verifying that a replacement instance comes up to take its place. - -1. Go to the **EC2** console and click on the **Instances** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances)); -2. Search for the EC2 instance with the Name tag of **Jenkins Master (Spot)**, right-click on it and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. Within moments of terminating this instance, your Auto Scaling group should detect that the number of running instances is below the target that you’ve defined (that target being one instance) and therefore should launch a new, replacement spot instance. Refresh the list of EC2 instances every few seconds until you see the new instance be allocated. Whilst your Jenkins service will be unavailable until this new spot instance launches and bootstraps, service should automatically be restored in a couple of minutes. - -## Terminate the On-Demand instance for Jenkins -Once you've verified that your new Auto Scaling group using Spot instances is self-healing, you no longer have any need for the On-demand instance. To prevent it from incurring unnecessary cost, it can be terminated. - -1. Remain at the EC2 Instances screen and search for the EC2 instance with the Name tag of **Jenkins Master (On-demand)**. Right-click on the one instance that should come up and select the **Instance State** > **Terminate** option. At the confirmation pop-up, click on the **Yes, Terminate** option. \ No newline at end of file +Let's now see what's the impact for Jenkins when a Spot interrumption comes. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/tracking_spot_interruptions.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/tracking_spot_interruptions.md new file mode 100644 index 00000000..ab3ed2a6 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/tracking_spot_interruptions.md @@ -0,0 +1,36 @@ +--- +title: "Tracking Spot interruptions" +weight: 140 +--- +At some point in time, you might receive a Spot interruption notice when On-Demand needs the capacity back. When this happens, your Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, the Auto Scaling group will observe that there are no running instances in the group and because the desired capacity is one. Then, it will launch a replacement instance from the pool with more capacity available (this is why diversification across many capacity pool is a best practice). The new instance will be launched and bootstrapped in exactly the same manner as your original instance. + +Before we start testing the resiliency of Jenkins with Spot, let's configure CloudWatch Logs to log Spot interruptions. If a Jenkins job fails, we'll be able to check if the failure correlates to a Spot interruption or not. + +{{% notice note %}} +In most cases we don't really need to track the Spot interruptions as Jenkins jobs can be retried if it fails. However, when we're starting with running our Jenkins jobs on Spot instances tracking could be useful. Organizations can use this data to correlate possible job failures or prolonged execution times, in case Spot instances were interrupted during a job execution. +{{% /notice %}} + +#### Creating the CloudFormation Stack to Track EC2 Spot Interruptions + +We've created a CloudFormation template that includes all the resources you need to track EC2 Spot Interruptions. The stack creates the following: + +* An Event Rule for tracking EC2 Spot Interruption Warnings +* A CloudWatch Log group to log interruptions and instance details +* IAM Role to allow the event rule to log into CloudWatch Logs + +You can view the CloudFormation template (**cloudwatchlogs.yaml**) at GitHub [here](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/track-spot-interruptions/cloudwatchlogs.yaml). To download it, you can run the following command: + +``` +wget https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/track-spot-interruptions/cloudwatchlogs.yaml +``` + +After downloading the CloudFormation template, run the following command in a terminal: + +``` +aws cloudformation create-stack --stack-name track-spot-interruption --template-body file://cloudwatchlogs.yaml --capabilities CAPABILITY_NAMED_IAM +aws cloudformation wait stack-create-complete --stack-name track-spot-interruption +``` + +You should see an event rule in the Amazon EventBridge console, like this: + +![Spot Interruption Event Rule](/images/tracking-spot/itn-event-rule.png) diff --git a/static/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png b/static/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png new file mode 100644 index 0000000000000000000000000000000000000000..12f01ef10cb62ebe254373b5eeff0f32ec37bfb5 GIT binary patch literal 109177 zcmeFZgW5ZNf%z{ei4%v1w|-}rBTKiB9)$XET*VEZPL?52jLkBKz^qeuZ-@OT&DN9Y1d zGWt;Hh5bD#UN7@a!6Sl`m$RrU^7H3rD6tg=MrsJA9td+3k|NU2=3GSfU0-`Y56yaE zLi(%>8QJ+|jRV3QVn0Ec2q(TV!W_S>%-W8OU2z-P_r3MXx4T&HHt-L3*5tmoQ3b|N zhQE7zkZG);qmeDh~wqUP5n1-UbR0%_`YspB-|Wl0avURP)F86Q4xUwe*YTb zS%@{l3;5kL_=gbwf!iQH7~vKC`yKov@fGnusn27-KL5`>qVYd3imHpt%EG_Z&76Ti zdlxGQ*F9Y{PIy++)*3plI*JPXW)2`06LSYsAd4r+@gETcfG0ov76f!PA@c;;*}L$2 z3R3**4Sx9jKaW`{$o}<;tF0h~j-m>gxPvp0jE9Adg^fZ8g^Y|0;B0QeuO=b&Z*ll9 zK?*BZS4VzURu2yk77tDq2WLxGc0N8nRyGb+4i0Ac8_X_Xdsh=rW_uUP|H|Y)`A7g= z%$%(qU9BDL$^OaL#MHsfRgi+>pMw7T@4xB^^tAq;lI&gn-7I(mS^r64WoKby{qMZt zqJV#%@~c>T0_}7qtU++~z{?QgVdDk->;1n{{-?zMC8_g2l6>qO|6B5Zr973?bOAbx zJAmLtx(fY|W&SPvzZ3s02w?rE<^M|)|E1=CJ%#I82nE3U-!>CM5qv2yfPf%^AS)rN z;rVPo3&{tk7pwQxJ4ABwABduXEiL+}k`XCul@Og{3g$Yb(nnl5Cq_sjt_RxTp^}&* zqU9S*czS(D3E0r;#C(HD zhVb+XN5Gabc>VWt9?VaO)>t$Wh|k_4{NL*nVwmm!OA$1PDrBI}FEW2;r9$R}_6gzj zlWSH20i{4O^)2#WwM9Uos{Fr2{;LKVuSEjtkg+HN{>n-u;3xNA4I}%A^-S8fT1-rg z?y4JAE{P!tTKxLeU-`-uyfA9_k!2Yf+1S`HuYak9)2E%AyYcwplN>gSISq0rHDUQ9EaQb7##8l@eT@6G?VIqVd+|% z;;YW5210o8jxT_>{4(spQns8SHu)Xpo4?fZ`<@&D>ExborkMVzE`Nq1VE5s#ikm*Q zm8U;kX+HiV+%_mrYsOB2fW#qn6^sA0pk-3HC0BA>qS8M<(X>b;Jhw8)EICX z0iafP|S1ua@5hQT6Z6R|L9^b?YoWr zz{VD;lqDo?Ig;s1S{lLs#6nbTh{vTO!9i`ZA3wZvcS^3(k&VjW$`6jp7(d+_#d0@M z<^R7uB!Ehq%m}BCN}4m8(G|7}{UC%`^Vb+TeL*z*in=^@$@_BrsAKf2%NDCjUzxVX zw~wMXb&9q&D~A&@NWMgGMe;j{q`fZ>w2y5ji$&dRW(uZkr%OCfH;2gZ@KR)~-|$u% zwkWMA-DEFGjQ(o#zNGVixSMR#$8#eR?$GCQy`sYr)UM3r0**J_6ihs(O18KksV*45 zA(6phNTiep%l`cR`{V9dEo&PxMv^#tzyakKkT_g;T7xHbkSq->^g>;zV3wO4=Odb+8|VAP7AL~oe>(q}nlyg)Wz zIRxkbD(BhTP?iAm08!D^IRQfq^=t6P$<7$hPV_RA>^jM9nOVvIA*B8?gs8F&Xt^Dp*(ikLwf*KfO$=Rl# zR+XMoiSBBJenWddov+tr-j6_`B8DlOiG1k?)wTUOXrMFfB)o=H;Hi)K9{W98JcPbV zn`Ju1l|R}fwLbf|p-y++)qzJs-AN6FX3BhnL(z&+jX8bT8f}d!jIixwduK-$i`Gvu z@FhB#2PaCtP(oK-qON$Vi*3$-&(Y$!qin_PlcDvIZfma&swmV7g zB#P6Y%KrTRW={j!?zC>r_&oL+%XuapPml0_7*sBFCl?K%?&Bvz* zVOD9BYQ*6(r{=rE1~XFl-49ErCE3sa{P`M38O(sEJ3R2=OGR{vdXbisME~T*!+l61 ztG`jHW?67Os{CL53u`f~dkf4PA zwj9OA2-$Eb!y+<)*5ORLvP%4MMp=Q zD;qJGyzw_z`)atnQ0#TSdy`pjza$5>nSn&4RBd0HAU&OSOvQdz^xz~Itg%(BzF;d- z&Y33NIosj9IR}7U%FZ+ST(&QXh20H0><7bBMCS_?g$oDqMx}0q2NRB{!*4~PgJ(@+#nH5b&ZRzk3w7(3& z67YiarSpBTqRpmSSGf6be2`oUJ!Az6-s35S+6#is&%R!x;_h6gL_1nm+?o*URhvTQ zg-tiC9yWTx&aRGi{12+{Hu~ZWb%`W*^xC}8V#v#7xy!UQbM6`kaxNO(_ISAM{=^OF z_fUmH9WY{=68+;An_Se9A6$dsbuM|q^U&Dbet*}u){(4szz^+Xj$%Z5`GUb?ib4`a zeYG>llOlWUW)1{PAbI;&<=^sfri2Y9-fxH*3F@qDGVE8&-VLhQCT%`ImLZ;k&WG&Wv3(iavN?n zc~#T0$`(xh@HEUJuR41y$AJD`x5C=&<3RPVEHLk1(^31VRyk9I zmWqK!X^=tTm!L1fzu+T+{_Na#Ga+OVjfQZ1+!Xm~tXBAXpU*Y*#0!?^YQ3a@ROXA^ z6EtOqjE1OIs36NRGMd>JS#coX4*L*PEVJ5qcBac#Vca1~dLM|fyFJ5P5z4L7!#?1z zK+SVHK9DzzUIwz2FTz&R0GJXL)?@tbjhGR`V32(8g(Bq!$FrltSf7*7kw=RguMWRvQhrVY|eTE2G*xOHtX%b3QrO%$q zIVmEbm>XdENna0cDbgrD1lTV%6WdBJws&Q`L?@9G4~e28B{lXs+r||1z0rV?;`MTB zIyxK~mKqgynwlBQY}+P1X8TSHADQlY9)$;%n9=t5j`8JO9tnT7p0HGDAn~<&_$-NB zKfm&Lh1xHe@QbQ&##5{;rVaK((1x|T;dnh_DNDdZR%js!pPCi@cb{n?G>-wEmg&E?Q;9*e2ua42}&8PZlmh!Cw$_Ade zw93>jbmvE4NSqBP>^7%ZEs4$%uTaB6a{TPc@)~KAjX?0y39yf{d-rv<3vg@Hd zvKV^*f(>fZ0RWkw0|QH(r_#@?Ta*}#)q4|CqGp90ye~;r&cjuKOjLfuDYm)YBH z60`cW-jzYy2J22vtPA?Rv*C{{M`N=LE@J((LbHN?cc~0om4*DQSSYW&2QagEMzZhL z87qA(^|BLKjoTxgFl(X~#ehZoVNODQpR5*kyz8zv+CE5DpT`S>n(?m=#M-6RE1slPy4R z6YZB|zn zY!{Ihsf^`GTK)R|Y%rvgz+-Y%p8J6tiqDX<9H0bpYWhn5 zJe}7;ZldU3uB#nC(-Va>ggR=^_SvN~Q8vIF6eluYXT4MF=UbwiNmO?rAtlg!x>h9r(nKSDL7dy>vD4LYwMKo$M61CsiV5^TH$Q7q5y+tAqCX# zzGOh$NTr*QL2DE-C|vbATytk;WtjmQ*nzJke0PR=e|B1>vDI}`%G4Dc!?A!m)8gQN zbJ0S7($iq;c!MKP#Kne5P_OL!ecqTXZqV%LK#cw(%_<7%K!xpPaTpMy2)n zoX=9_9@n~2)A?+p2fTmeb?dPao1rEp`zI7KTQ6J?-+cS@U1X|^(GC0MS~|lu{@@0^ z@OYlE8`#}~u^`d-Va8!rW^2@#Em|q&IbyML$7qws$=92d#!ii&YHdG>RYztvD;9Rv zH!G~~W>zHn^%vOP=3Rl+$+n`4HaWunm7ncbn(}uU&*M(Dt3=J_zd6;SBxI}64>q5_ zx4!Bd+r0~8tfX&p-9drw{gTW|!Pw)D{5h;ftN;TZHYd4|!}xqR%jIaUc^u|~rF{96 za-v*OWQfG~YF?mZfKTy?PpMa(?YZ$-S7*Mes%!ew-x|v&@1AlRg^ir!5%)Oz^+acU zt+Z@v*bw4KOe&i$=I^`SUxXIP4xZ1|ZWY^JE|-q%@!F{wY>?ow_v&mq`HA(Wz1cY*j*T7M%TnJm$PA!kKY7i+wVYV6>XeQs>9<&{~F9dt0;GTDS=dfuvuWkWA+ zPpp#hI%Vq{J^Lk~+2g3abgT6mm6CdI?v|5~&#Qdi@^I1W@>3I!L;uR-uh2N-QoxPB zkPPwG>@n7Edi)4YQwrqn1Tb?LClmC|jFO5elknRV!vtPNAL!T10d5O{gEq{s&R0%b zXz%tQw_l?em@a3D>n>VHu;dNsYNKbx@@I_mI1(;ZHghxjZ|BVzu9xgfdGEgss9(s= zj}x&RM@bImFXbg@SamAN@p@7MiE6^6Dq1Ag7}9}_ENvU$qket7y(%6h&KAbaAg zALI;&`Ic4NNGalLeg0_<9%D%Y%G-;EOW$a_xH>6DXAjP3GOc>&ZW-GD;8g#VY~;06 zQnVzcRu;lDE^oI=Wu9JP|Hn;1vdyGoG^XwYdeLRO>P@@lc9+U%fuq@qQBd1H*7x4J zJGPVGgn<3=K^CLg;>Lb_Mi69k;P%y?0hn*Qv<+lz?C(u(Wl;aMmA`sTsP|k zHYh)uEvSx>xX0^L?0%rf<@rklz9;bN{%T@$7fI=))$=&sIp@~p_}v3QMv+0U;5GBT zB?0(|>%fnk^+ie}Q98i?K`iNXRooxt%^Oq6GGTZOIHpoxkjS7ZcaYH7amSxz{K-#! zdPbl$K`o*ddSon^%oYy zX6zmE$$U%F`9>!!ZqN-oxAkJ^gYE?@Xt7Q*M@wHejf1+WsikCitvjOBtk0>^ph;|| za^+W^q+X$X5o1D2OG|i;fU-S~eTh+n{Yox;9jzKe(zgBM{x&h%JA1f)5sbRig(3VE z06r<++emp-)KtB{C#9Dx)g0WMe6005quNH+7wynAIjJF=kukP-;1x!gfXK@$(Pt@; zn4Vf>jnB1Os{;v+<|~RGM6F8Dd(!&xRb6I4aTGzC;QE?QTIcAb<9z9u@e>&1_W9n7 zO-}2bpy!`Y9F`*m^-xlHXx=wBrwbZrqSGp;qn6K(L?;z0y}Dj%mi%YEIdj&hYFwJa z%v@0)dH9ddI}z*nxGLuC?5svCnO;+t1bo4G2EHPU^Y-qICO+HBk+1tIBdCRkxbM@I)}xrgsE z#hp!94&@PWTE=B$ym#B3%~r~s6uhGvN|X_xIQBT0SBz>}XG#bOvC-yt-xV`;vR*2s zKA5Ww!TmE>6AC#=Yr!3YI_ZIv?{7)7HZI0AeCs~cL{@m~3Se|eM^a!^Iz(6~>YDJ& zM-lQuKnwM!O%5XCMl#-v;si{FIl(YaqZD_E!xbddN3J<3tK(s4)aHB`qjr_r@mhD{ zmSo_|bJ;7R3AduHYem6hk)T!1-ng|Zs*gxMt<{=86$?TsIX-aqYj)dF$P#eV0F4i& z)t)<{X_pInFi)*ypEAPESGQ_HGgr_S!PfE_s+D&1ZbwyX=nRSlZAlj*BM|Q7H5|9K zpUBK5dL#hPP#+)P$!5V(CT1*KP@QXCpBwpjzP>if>(?5iA6!QDTt+Fa86jCow<1_O zUzY2vv0y`SfeVtSGllLLskWy?#OlTMN+--Z-@e*JsJ9esDn+DmS&kUk%;hyTyGvYS zz`#hR6p9ukxq|yFY4ll~nv5Hw&ql?HAJvxdJ2)G1)OHi2HJ~a|3|-R|(#po;shW6G zt75wE!JU-}9iS$u6kTg4Q`^*tgy({z>?o< z80F9Sb3d*Rc?KT3V~Ngaceh;Er_G5gtbcIaK2pOG*sco(R*Fa0-CBw)s7+-H@O~v^gBNWrw!pYeYvEg;a=4! z?0(|-IC+Wo5_SIKE530^&4AOM(KNT&P zUtGGdX&=pce2KIBIlVp4(@DQvn>;#qFZH}vX;k>pN0GybL=vBKZfqS@Z9E|$6QelG_NJ$u% zjeA|Q{KXfb*+$upqtj=BQ|+HVefon%t0W~m0gv3^MJ4@kbARhJJV)1QsTA~(~KOY~thwmDmf$@_?r=cGfy(?o*Rl8!k;JejjD?AFE9?OWD z_FCV&Aj{4mS>mIm4hN2QTp~DI?OqIFsAzXBr#<$(t*m}IjeK^zo*6#zogF;I9qlkq?b1se` zgBpBwwt0?YYNf;DCzd-9zS^xz}>;3Jd{rL4-6X84ppAfL^y=6Cq(b%U#Yo%eUO_vA-x z@hR>gdEI8$y%HT^@QH-6dj)!Oa>-34$|$?or4M&>O3l&0AGS9acq+aki8`Rqp2u36|GG%`)`tKX_ELakO{0*gdm^ zmfF5sGPL?g+?OXE)dGt&?u0RxCb#0;W`i9Y9?H%aYV19g6q4iz$be&=7e?M+Pe4E1 zl6*V$M`CdOhA!jg?E&|n{BpQi;yV(7gF)UmTzY;wJ(bb*QRhNqk4!ru&6PnOWwlcL z^Pe8>XK%jg=nl50eF*sH#H6#zpsg?&hcS7kx5}X8^Dg8v$`Lg7YW5q&bfeu|4gFrB zldIPF!>H{F&&+RU*Z1d zqA=c0F@8=F@`i&wxKS@$2E+ZWzAmJf*Z1a(ha})ki$Ru5OCQq-+~jUJ^Pal5Z8g7^ zV!#O|T=`S4t!_P;)mTZNy9FCHmImj;gF^jgsI8h;y$?!*+x{Az)|+7p;*(vFgI)mQ zHIp7M^~wgiL)0;FmV>FrcE*Uou(Y7tC7*)uPN&6wZ>mh5V=lb$O@Gtdh)$aN?EMvg zl`PsO^xc^%%h-I_RGA%hf0jVQ1EG_vYe5ZgfNn6uX&}|W|AMWZ&?KNr3tH-gdA5+w zt?3?0ZRoWG!m*8n`fM!nLD*J)>Xxa?J-2n!Ixt!@vakLmr+_ z<9u(=G_-(xBU84`?(WCN4$QmGi}EQf>K#ID8n*@w_P!T{+e(5h9*}ZfH&C6H)H+0g zjuHfq9vpH4CFc@EB>?tZn0pB++UOw8akaw$_)*HXAo0bi%JcqOoTCYe@`}A_kVf)_ zbj)+QFU-1QnDXaG<0XOpTR9cI>;V7d`AB6wD4BzUgZ*>mZ@nreuvIp@Z9H>EdHn&i zK~DL>UcL9sM#w=;nI-Dd(0wcTGSK$YB*KD}G2${4 z4nUOYT=vD~T~jubxra+Fs$0njz__HqF=#$UeBM`V~hG;y@3*wca+0)BEd>dtg)+v~d4 zH5Y5t6b*W!VhfV2UVWcroL(A^NB8sS?Z(=&0)^P>FTJ#`R*xkQ#V4Pz#<1md@w8dv z@b+4#wp#bSW(<$g{>P}njj%{!fndX? z<@V4*nK&_GEdx84u|cb;xKZmbZi|!7QsR8C^U}MMY)+mU-n3r%z?8QKsnx2iDWR`G z2p!vtjjQ6r=?!ZsN09>+oA7m5DHBIMcQXY`D+R_qX?&a8H}|FKN0}lK5`gO$bfOR6 zQOp@hq~DbrX^4x9Pjxlq{TR%H^7kDB1l_U-^$afS4tJ4MjXrfl`c(2|tm=IJ{=$qLww|!aAr|kmd)a27H`D>a+-fkHGyX-Ojl*yIfBi(4T%T56G&^n4xI_ob@WX(26eb118qzjtt>NA@bBRgSz<^4`I+!yP;cQGOM;0Bz2S}X1&K( z^7?d3`*4}VhoU*p4L!BAV}n3{YewyQ*Y_;fMjFV}>dFvaXET*)8UcZrWuVkfOUct# z=Q`UlV1RO|H&;qgyZ4586r%fQ=V?kiB%m^4j_kM79abDO=Y5l~tnR~uX4~pj!}>;@`%fPx-ZHScx2PpbvbVW1$`B8X3q)fzU1B~D-AuOu( zSop~Zzd&C100k~prkRH@0ig)|82iTAOJh13{EDty(S15oRz?AdU;R* zkL-9*8NKuNN74ng>A^{i+8V4`9QLZ%{lNg1p+7=wEE$CW>suGt$y!(h`tgQZ)%i3} z8k0o%LC-*b0eoYR*KCu>?Sno+gqw&&kIz)4p?XE{W(;@F7Uf6|#B3hkcUylF(zE)^ zyIJAm>hIMjj$Y}p&Gbk%C>*UB@vvl+>~Zp=6u8?LM8ztr&N7@>ldR+<@ci-okY>^%9#Vf(a0(vBsdi*} z)aJdO<^{}65D3lGd#hpK$wEcj8=voF?e!08x>m%K z$sgry;J6ZX{n`lZDr`no3YHSJU#lil6hZwx$q61w zLBkHthB#WMGl8yaJ?nJG%P@3;!OK+3L^KQA;>ad!aG%+^T6NgukD&;2T#^-VU`0j# z`;8*pZnw+ECJqckUs3vi6@RtN8_i)+)2K5-sNxDPK~aKiC}MU8g*HhF-9Hnqaf`5W zu$gSnbb53&T0@=uEq_MtNCi5`ZKuoWe>z(>oG^fH=H z^ut==SKv) zI2MXqTV=e`j)xQbsQ6D1_u5>FZdkBl7PxSj$A!QFZ@VQey!CHJJ9d$zEP@j_q@)+e zw+V0BZqc`w4u2+{oTYKrc^?^RvUu+SONclI%K%^D2W6sdvr|>ZIdc$-10?xHmoHtN zlFcE7kXD5t=5+#IAdUDAl!VVFGJHJV+p*%`>mVZ$;WZLxFke#t>y4mXj*j1b@)2~O zNvAlE+IN#CTRXfBj~~K6P5BrIQbxU#0wDT9?0pe#EAa}N zX-kp@gmYPQ81(Du-biLA9NDKo>44ir} zKhADRC;87ySLuW?rs>ZFr}C>D?KWFplV$5zQYhclUgveByIMZPEvV*sLuRr%#D$*K z9Z8Wv)3Lzuw6br1GL_x6=9o)Oy<0g{n;zxwb`39qo@jS??PM5&M32RBVPY*u&N&7lf?qDS!hsCiGrg69YT!zMOV0l4yDSbe#r=0O zZ*gVRHRF#&9R<07G)mOcn%#E_rqTR2$SF9Z z2Y5MQEeafP)ciown|ujfJfLmQS;^V;_a9t#S28#(7-^M&QzJ8y-j7v@^y=Y;Qu4*e zK1Oy+INC$@23p|X^$sh#!)z_?=gu|o`6w_PMk!C^G!u_Uw6laAH)l&*gPJB&q;z2Q zD#jx&jac&MXIhH3f^?|J0o!SQH|7ji8j^PHZl9+?HEyU*X&^m%a;1KMl@MdtT8$yp z24_xV-3F!?OMqZIQ(AeEc4OVc8d)}7o=a~0Mswz&^jp^qHfMNb_SskS~|E1@Eiiw-PBq1 zrR=BF@T<4eT7+Om$-4T$yKmHS7ZV!qO-hM^uTz_Qej-)jHI&s_3~QOaf?daF3BD^} zGVRym&ZpZNxWC{4U~IUH=u2%UoXXR)dJ`|H3tDPM@@Lq^oAM)!P9f^! z?Dy-3RcPfWHwOa?ofHGur`mPB4sn_Fe%jr`h%&q%B9(2%(UlO6z%Bc1B1`4`&Z;kJaYW%JT8^WKy{wh3ik*8XP4vt ziMGD-*{VJ~-o*vKUM^X`iqX;YB9X-gPY*KfaoB!a{h$r^0`IkLU5jNhs6BD@SILi5 zPJ}H-n5Mw>?n_?xF6z;Y>ufLqXs3|{DG#l(anKm)f@>CsUS_6smbL6?&7{w~$^4I; ztyurKH!Sp1HP0E1Ty$h@&i0;huNMs!-*A|gn|$dl3&&d`RR>$GlLLt$xqapN6desL za$BCQtGXRzg!TK8^C?-Dodq{x<^2=ve;^_2K7LlXqIDQ1J5h z_3`bW>efVi!7CJehCVn6>&X#hi!K7m`(eyhNE(Q~F+_s3c;~6W5sqVdQr`53U)qO; z1{qV9D}J$Yf>pgnDjVi%JAB!!SKnRy&J~6YCM=YqMiJ58DdH9|uYeeJ7LcaEgV zLSIwzguU-wl=-q3$-5tie$2bsss{{7#LF7I>3u2BWpcQ;B|TST4)sElzPyP*H{(kV zwHQ-9_ncgG(3JMY%p>PKoUggZGxBPSo}W%Y(@Srar#m-Ia3C zbT0p8I~apyrdE3eeRy`x5%+n?XWmr@g?h}M9lJT)_O(j-gjoCtI-fPAbYA6bx-3m+ zbaM)>CS_>Y=eo0asdBM-L2?rE@e+@lN%ja>vg}8aKb8?E8yDfK0!2w(j`T1b#&|%2 z0{vvqK?d}nDg zmQ^(M^H=9FXkb`f)zv30mjrWwxr}16iBTSEGa!y8ytzfL@Y;_1tG1-I9s5pS^1BA)cR@ zzW{4nR4-mrkLvMlw1jzI>nG_hs;jbV-7xj1;L9-`H=O0(EmIpc6V* zGB{uiISGL&7WsVWtHFPx$J{aB&e&E<*svtKQ%7(0tr^*DUl5LV|5%0N@s`yQ>vhOh z_$fMyVpTx@K@kQNh>G+$(n!Hnx5dEe7`W{tL#3`wv$EDYOsq zDQIl@)j#}(klM(n==o0~OmH+A0qWa0q^CHwR*V;4McOee3EW;hr9SA8#TE+-LE%uP zpyR})b5!uQDgaC%6%n(PTMq=#XStxGZ%W!=L>46;TsCN7v z1J?o2+~Q8C+sEeXKYZ7(Iw;-|ze9YAhC_}OX{C>S#wu2_t-Y)P5s~8eqWpxI9Te*l zzA@QhFGQ%$S|xVA)0wd0yD8M>NnBWeYVHnE%+V~c%r%+3y6oC`OvGVHE~l(lDXc3g z1~8iKMX#vdzGM=T6;GxqI?}^|u)u1^irrGIe%OVkl>~DpOAijeWU;RhaEJh{*I_TZ*=FPmWZ_r@Jt`OqqjP!$>n-LFZ-6t*^z2P~v z>dp+woXns-V^!jX$r!qrmY15*@9JDW0&N<$sfaI;zusj&Cbf*_{rrLcpKkIW`^db0 zu?Bi$zMbLdb#lTD;`fc&=;{cnZ5eaXdP7^)A#s@#Ovj^rC`cXSTC&}>mYq1!6;!$G zEwy6TRM=@Smq!~<0p=J;C+7wO6%^@-i0B{*_WpV^i8dXL)k+KA35E7;Qqlbh6R$E; zbna-XnhM5kVOLS7Z9bRue0(&tc(kv)-d=sasEyeFG5C-uobevw;${jvq`$A3kcRBF z|Nf%J^FDDzjM@c~|699i%Fo-cfG7`834}Drrj=`S;Oi?S%q2Wud&2Pe^t}ZR2*j1C z2v0h({H1^R&@h?e8Jt8EdiHPA|M^pLO@QduguM85n(OI6sX&ZpTyX1hGA4ocWeC$IIb? z0jK_Jm)g$Ku*R(Abc2)$n*0QhY!xWIcYg+R%p{qxAP~k|>&>7y-;{{ER0*j*X)I8u zCv1BNHw2S${56W?5sxcHXz+$v#LS2C=wkQg80b;V@%8B+XOE-@V^$yv_DXWdKEss- z7Xt~u-@?N6&U<|stYsufT4uR5)yZ6d?qLN0ISD6Y9GGf>h88b0QPx`m1*j~(b(c+d zITtf#u=Y(LPDbS|6zS#ErO&FHN0<+GP37$nJ^_P3T{|v||x6QTo_Hlb;^rS&4e|Nh6b>S5$ zQs^N{%v9^=QO{P%&)olapai^-fa{>oOq*2ow5hiQe^ia%a?35qqVE-W_Falz#R{^g zXvqC%tK|TFo0pm?8vKlq7tJ-m6ZX-cH>V@-0fA+d=N0MVP5vVr-&VjmK3Fw!_8z5L zv*L;x>N_EAT-#l(Pb-jhoF$03;RLZsU=O>m9$53MaSCsHXqb%9@AM5CNafRJ z9kbGu=S$5rAao^%R&$xTBVmYGngKlyV`8u~?KaN!X=>j)T>~_ay&0hU^sBo$4CRKz8{Zm=>mr~nBzLfM zU%-(vmf>Wmtml}Ly532=ogCc98@ua7T-WceKaWuCpafT5OB43svGCAqAzJy30^p09N12)+htGjgV9ewAZB;mvfd1K)m zW}}5wEGqPzf=}8Nvh5i6Q&5pUR1O2E-ws&W`|vkhePugyeqp;Uc=a@|4`QM`8a{>ExtTspV42t*!NTheMWFH5h|xSFam z;>+C=>kPdOr-vsI&vtqiu3UH@>Cf@?XOSR&CQ%v$4Bxr(Tf*@K4J9^w}6tWZoa2 zAYZDH2mp7I3X%b79)5;}mh90pF)0Me8pd}!_Zod5^|%cP#J-@3+|{U5px=q+AfCCa8cU(bmo&o&4j=O+hqKFU{`-dl{wlj&@R%;-WPdyU# z^3SUHN!yMzY#=$z$>_p@ELmEWMEx`Z&H)KLw=dNbTB~Px)GX5|$nnppERXB&)CDVu z-K-xc6zycl3YRfxR~e{g3;)bL}?eA7`3;z=Uhtrpp2RYnYuMP%nzs*PN#>4&K`>5+536+vY)es zAPbjAWptN~6$7g~*7RP9Y+W_)U!uh1U7twx-?l$eYLRu#1w9<_RTs4e_kQR9x96d9 zLp&}N>EWK`xO;cMSEau5JGV;%;!J(N8coC}WjUG+QKght`44NsG-wuTvx|W{$e2@q zAQJcrt)Si4tLMYkKRgKIhf7Aoc2Axj@LWPbBzSUBOOZZC&uBhXng_tp&0eBQrd_cl zQza&@;pTC$?2v-kWHbEo#$ig2!Km)7V)h3mSe$^iQmI}EVw=x-xB!xzH9po*&Cz!g z*NLP0?vv1#6O0vKAiI(&iVw~^u|Zx?A#ZuA)mkSBo^v$OL0z%+4oEkgqV96m35WX? zqx-ljmO7u`13;7BP!C9Xozv_%cF0t9drc4F)V?U!!jZY;vMx5y6&pAud+%}4EivBQ zw3C49uYX!!)Oz#$<9}$0%g+&IwF2xWnv>MXw5xRA%LW+Kx@YlMFsJisCinZ=5%=w< z>H3$>RIJE!DCdaIR2t@I+L7Z6Sa401t48sA9LQ#8r&$dYIA|{VC*Mm)JlsXl`18hv z({i}&0p}nfnvA4`t$u?=yN_jVw*KZ2Y-t&~+5pr8s zezlc6r0}YR{rO7&db;!I>crRk|P_{%VQ(ozVW)oFS$zoETu&0{z2JW zqITUd{d>!&vf~k)&R>$bKa?b$u9Q8PuPfl( z8Y#QSAiYV@9wDMykMz%APq{b{5MecHr_k2MvzaPKhT1Gkbf5ZTi{xMsu8$e_YxRYbR2=Ua8}@v;LgtbGxxi@{sAX9f+sKLffB-K&h?4TR4eaa!82Z7xojY z0!*(eF{ysx!jmg0qDyUwCL-_MK-VZ2@#FWt=76{>DJID!X@Gv@P87*9nVHI-$)rS= zIxhT9q(I6UOmV#$(nWcVvH=jI56Y*6T@J{0qvC3SNkSdbQq1;0cz&ArF!pNyB}1}I zhxg_7Vp7@J7iOQ4avj|MjFb~+JzMAQ-jo3x#=>a-)f+dIC`V&%B};FtE|TajG0YCP=^kJ~(MPN)as+4F6bt4?(e#lX%kg0%z|F>e13 zO=Kotyn7JI{D0Vc%b+^8rfoPOK!60d;O_2j3GVI^+}%Bp5JHfRy9IZ5ciFhRTX1)K zC+9vn&;8_7eO2GT@2&c(=EuO^d(BL*UTeDh>b}}+eW;zqJ(gbS>$L4`7XD3SXp?*? zNKJ^>mYH+>!Sj!QEjTP`gumR3WO2w61~ zfX7G8ecBhQO-tbfM&K5Ksx(TRRwS%}m_oIfNGf-BGx@Y%+0)jman2DyvD`RNy_;_w zygga3BzEbKg?-1Zh>^kDe=3rtOD2hy8-m3wFF%TUceAm*IZzpQwKtXmhej#5;&YGJ zUpIV5FqJI^!XGPJ%lwsl&gu?Ebc*f`2datRH*<13$~Ar4Q~^L{$=HXbn?SHtAu;pz zSY)47Va^j>wgjzy+tx?(rt*4=x*iLxBD`H{unP7kB8K}pZM{s5SdPuNRd(z9BvUwTqqtW3 z;e*N@4nN24u`!Ku-6qw-bIGK0mGted8n@F$ouXh2gu{LqoEp~bDT|R{Yi;=o_NH6N z(!x^2dec&h+00SJe)cy^t(VVSrS}Gd^VE#iO@NO!;5xh zq1OC$t^Kw8P1_q;ilcl_5P_OxF2>5ZVo^Bw?wvg%PH%PeGrB5aHa>C2g=F)qRLBHl zLdnXmh-dj0S&=WfbvfU0itOq;XC4D!vXw;`PtPNdq3lUwef7xj<+|0V|U>yjkw*CgEfQl(F`1Z4k@>(;b=LswYT_&j@eWu7!m zJ~C4@cP@@r==-EA9ky7gmtn+Yu~0+O+N77=K;R?z{8*bD!&rl+QY09NNT6yD6kUox z-e3M$^1RCD-JgCWlw3}nQiRxqGMXvjtMcweuvjXnuW&Ny3y;t-_e`pO0p{#ZUL0ug zx`ngogUj3QppKO(mmE&yN|Ce!H5Xx0Du>jZ4CJa)1UT-K9LAX6{MbQM)rk zvQw)sY!lZx!xp-&Hh6n0xjY0E!p=ozyH+}2^`Yq@f$2sDOCW}~OD-14Xw9385gQ-p zkQV6B8!8T6gRdDS%^RVH0g^t@-7PT5K`q4G zel|)_o2eV!rTAJ_l{{@cRI4`MDWc;-C=iWS#|L7xgNw`awwXSY4J@#?fz$8?d?(yPE^trpmynNqm zddy$Y1_)H9iT3-JQuGW^&8sAuy2c;^C+5*lA3!XF&7pjh^U!&Bwvd z9{1M5`DxK=!h4HZ5{8-sHjUl0k40KD!>SI_@$GV-)5db~JgGQpeB%bC;sdJ6l`}pC zoly{j4e4trgEI(YTk#gl9z5Ub7b2%B1RWJAYAYb9YI@L?&}W)gm{h zBAyY4(b^F9^El!BRFcZmD`^p5u`W?ZS=2FMH>gLhir=lGa^}_}ncugC!!av)C1@LQ zZdxl(d=6*y(&d$RKOP!rZey*Wv=p4FU7q1|^}Qq|0&n^Z6N=5at2%L*FO)heYhn(= zICO#L74-VDC2e8_HR_sCr6DnsnyQP4@M(>TA~#n8t!-Nr1@#-pogT*L#4Vb4HU@Zq z`^AxO{Ydvas7=Ln6OF%4Rqsz%c3dgeSAuJ{Tg<&MJ`n3_Zf0vgk$F?pY@)B{FOkU{ z&R0wHNFSL7D+uA+PG(7b$QDoKbxyoGnEL=ftNBrq+i6eK+RZB~#CWm6xv=8FoFO2z zp7(rb$RtivZp-IRzxgZpa#ESU=RQl87y%)yph?jL3rC|+RmOYr?Ji56v;8Yn5%!ku z0VxjM-MuLsc1)3AAm@!sA0njAd;p>OJ)`|NXtH^Q@bT)ajFw6e8`MZYQ-C6VrV{*6 zsLFVby|ao&-#c;S`YWqujIW4KA^ZmWnUv&yy!Z}hpw_WH7)ydWoxJ_^<@3yNpv<7wtqv-$-nuT|#QDVTju$G2!$*9z_T zvWGaA#V+}kF=Td>siwMEHQcR~*i&)WfSMAO$-B#aEDD9A(hO|`+#4=6|>Wd$WUd_$T z=FoC&Bu6!g-_S30q?qV;0;JT;2Ch{xfmuIdjGXV_xeZi0rT8s9~8UfdlsG$9gV2O{Q~`JN2C4%S{Fa7 zYLtLd^(X&>AFIPQZKcvR44Rs*M|I;_x>M9116hmr8S%P7Wf0%VWTP{@c{M;LSd-)C z1FNf;UO* zVZMX&Z3cPO7VyImvm^1w8$<&B_n9rY0~@3Q3LAuS<<1H{0^ls33beS{N2`E?abSt% zaD%oq?nao6BL-~@a`=X_FEKzY4V|&y_<;AdJt*N8$A_3X(*j7#Rwnf2_c+_kpN|{w zs=8+m3R4$qc0I~s=0yN6ErnuNbf4F*!?7_e>zuN7#Y z?+j-xdkf@JDfy)LJn(S@?EuMTc)ya%&N!I(@*K9{ENVonSrB|T7sAf4Kuey8zyc^5 z>hf5h#6nm5PHK)?*>^j|Z#-7$Nx`-)zvl>|c(uE%W*W3u=AKk>GZbGbSWk<9eMM$G z6qz85{O+lHN4daFA@wHjT}q=}P)RxSO!Ki|LC1{K!t@aF4~Zcg&vk6--lWP^In8cW zH{a)8YFTvtnHtHN+(+I5ekQSQlTmjUykoP+|S1IgE1vqkg1|hzF8KkGZ?$fJQRg!q9$xjMXC=7 z(A2s!zYLQ;s5G0hpzbqd3O+xmtdZ2;CfmXc@Pb7Y6Cai=e@kHoPO)W*d27DZFF+U` zmdy8|KcTs9F_F5o?#RMQCM798_SNohm0>~>AEz&JfG}dlO;}UTQW?MiavoF!yX1f> z&gk_U4qt~%^UB!CJbR_Y){ha1oWdF3GE_3Y4965Q`#Q>HQER~nOKT7q`$FS?o-9<8 zV|yvvEjcxr%T{4JQsSz0akVVM=Xv=9E~a3-P3VK0v=Iq@uW(MU+d-pImhaAH9L*)7S*}$kMV7;@XyW=o0?FR*-lW4V`%~CQ zDD^BX-{BvOT4-Orn3VAZzAd4y1ejq^a!tFgJa?!vE$z@s<7st}hKVbdDHktP$>%zkt*Q4@;baXl^^q3@%>6~>;r7Zjj5ZoWs{9WzvY zduu$e-UGrHO(5o*H|}SM%4D<(M3POg!P{t;G+yK9oq`O=h6bE()p13#UI}KAHm(PclM`=HiMXS^ag=1C|Dot373r^21mCUs zkBLg|blngP@xN>=Qp$0E~pBf7y`Rl#%W=4i+i<2siia=F30 zA{_F}rlt#|81a;xGL^&s9S)ZW60B%TPq&johl7a@LiNgn9^u7KTKE15qe8EhxH#DK z;vIpM2^Q~Q+E929#=7R_LP(1hS6~14RIfrN_D!ldf%w+)TyOp^M!5v0^RFPpMl2H! z9@Ih%0n!V$fL7RGG&%kWqO>#d&gRpM&3O_n_1s-tFWREt zuqkMLwWR@lVJ z-jUPe^-M7wSAWAcf{H{)8mR7p)hKZ#n&<{I1-A&npby;;FBRSzYj3nX2L3#_ePbnZ zsZIE`gI8kmZ;R55dD1# zBt1wg-^{R5l=V8;?}8&Xc)RCBBvjD?HmUK#Xz!^SizC(d%`G1z9e6!Qs!g{Cf1xRG z!8e@ViYA{M$b29&c)@IM{)YVgZivHa87V0j$J<~11|yMItg5;~ zeMMhnGDi3ZWhd}w8-V|af52A%7fB=%qZ9zEXs`m5$bX82@fSuD4*f5ttrw7kC;-VU z20X-{P~8j2w}5Nb%NL+)Iv%ya@-`)9PeruVl5CGKlxhinszWQ?o2@+oY0$urH`~F4q8_5bxip1SThQpt_ zh~)VTbj6q2g!~uinkd*b)RCB#|L-4tr0_fPem%DFNic@`!<`eW=^ca1Hq!MNY6?WX zy=IL3nhLsS&ntoqp-0aM%e~2nj>9>( z&NK{IL_|;|DVJ~4U3a0{1tnv+gPTWz`DX3Wf*coLrF_K0@pYhZW?RSqPF^aDY8zG9 z*zL%saml7u%zN>8vZUscEnyb%`YoP}PwkN~wdz1l=+xEO)V|qRr14!DK*%~?D&SMN z5>^@`oGJLk>`bmx?~)>kM)MwiZ<>gq6jxG%*L4l0_~Vdvw73Fe?^^;i>sZTXSH;yR z3Wafw$8-d1{eV*yahPh$@%Rni?E9yu%^12u{-a{g>wYd3YH;|pPAfxs@t3b($1j0k z5Rj-S2?T=;X!yZ@0prJC+LEDs72w3g_7;B(@o?cIp!h=SHn~X*59hC!zBoSVBYFrs z?|k;jEr_9(FUD3`S&P^>k_TLo@F=D4l*g=ffg54&(hfCh?$L%Rt!`Z8fg#sb2JUEE zOlUOSSUEj5=u`f?e1#s8eet_Te7F=j@RrroZwfqL^?CQ{y)$)m$+i{yWhFT+} zClnvMFnPtR`BG=>Jc)OAX1T*InC+^`^8-*a z1`^$?ldACPMoD6}1wd?>Qxpf%A#0WBUJ2A5VX6Yk$g^iw&iI~?XIZLn%|yzGm9;S8 zAA;XK3%;r24@m@raK@wca;Vg%e^Q16fR3_;>BwIGqB1-PwIjBE0HDv!5EQDu=f5Wm z{${W{V93O&6Bp>=&cahc?! zoNPN=sa6vC`GZ5$_Z@USy@sP5F(J9M2-BVrI!3Jtj*s0RU*m5 z18H=P)0nU4T>)>LlISuNZ4@k|(5bVq$CkuEirI=}yB$E}KWShUY+N)5XKP{Og_y=Q z;9Q@5w&;FSR-ly~Nw$0RInZfuT%^S=O$pkSgF2q0wBYm8s`{_Z&yQ5}cMATCS$_8-_16AHT_zs)0pI>@={@fs|L4Ou|IobG( zCWD0Q!jt z1bnhe&+$?uEjPT1i(O^Naff4|=cF8@!6&{onehy&zzpd`ExETlBg@R+GdMku(wq*u zBs_-kk*4b5_vV^O_R;AtLOM~tlfpqMCNYZ9hHeXMYl_$`1s!~_qHj6%d2S=Drpffw z+#vKAM3hm;I@=k{?>G{O#0)}VtJ?Uy`vGpv~kJ` zr5HVz6qvMsGtTVo+6pQE6 z({+$laQFEN!$IYFWu=j)S9-*1x%?jB$NP}4n4N`St)&$8^)e$11j3U&(3xm;<#7H9 zT!I>^-md+Dbt~!gZhcVa0+Z{!@??h!mE?*oL#VTNG}d|gQX5 z580{fDUDR=YQnF<@^{qL)V8=~IK>$UND-kdN+X%{Hlt0a5- zVl2jBB`%z84iYK#fWn2jjq{A}L7R*9ZT#`hA#W}E6PTQntEn&dC&MkASK=PIKOWP{ z^S2T(E3!W|Ypm|weP4do=sy69%<#C zuPi{P;pC++VlS+zL97cbF{>J_vk3n=U7s?jS0(LoxLkdWPW?(+XIi zHaZz)s!(~&N8A2M;?vW;I3Hmmo5x69f}!kYMm9~VrIYFu8gZ;(5L{65j3_~cYD`nV z<96aR&S(17fB`@HLYVAeK^}PoC2|Rivz@+34aGtgWW{_VSSq~>8Hsd;U;%b(=608Y z7@9(`?n~IwN80i*6+%~fbM?v|!|7g{rzcVO!3;RdY!ot58L;3r%Ak~VNpCmTvYM-A z-*mHQh^;_Tf+s+O3pNb$hsu(UAV!fAlcK9fLPMC9BjIhf%Iag|n0?J`$jQI_Y@g#LrE2$s^Dbg5>)pA2;@^};_uW!lYfO^sD zwC0aw53m^ypTVS?(r~8OONXmP_x{vJRzt{8pWPPt88PKk%^PF`Zw_aM(8UVK)w{X{z%zJ4n zC&F8M==+mxf5tbR@WpHHp%ci8F1^?K6MO0Tm2KUKK#VPSWbc^7acG-ClbSoV-L!kWytf3e>znOE%=sX3Zx$i3~ zOfBoywWhHM-N?`0=yhA`^IOBWl4Uw2`;N6HhtU1maOR45MPkySF&5?sa_dr_PO33& z8+*kj{Qy_{(4#w1U)|X8f}g|YPuTnmRW5hF70c}e;pLlCh}Z0^>q!dDQ{x2@5EZhm zPK*Wz!YnZU4~SD8_xlI#XA1|ncEr1&;+s(l;1uL$rr;e(PJ-P#Xg)o@Vu+14?$-laC8OhIWRPza7rdOci92rh@M6m`K)y3a4OH^dmFq->fi5%P)XB?_S* zjOZ#rVYVHjxZjpg%52>uJ;5%E%Qjm#)ttDG&La5$f_Clu0r&m2kxo?;kHdT8O;ymE+${l)fFq+?wt2NE(Ov~4m z%?qa9G6cqVo8=}Redv5E*GV<8hdi!QDTyjFshC@GM9n_a+G>L_U#bnyW39PgWPB!J zh;NIsSRGa9dB3>%8M@s9v36@WswdtQ)6rY2>ezsLZ;CnwG4|NK+?t^ zM38uqU8C@FI82ds{KSpsRqv7!L!L8=ITAz>B?+TB#K(n$h8VU*j`;1` znySuQB0JSdPs2Ou>m8OnHhh#a1Yi0s&_JdUw%=;NrEiH@Qw7Nj6Xg|>!DJ~_JDJ}; zmbMz@Qsl^Q`Vm6}pc%+556%U<-KfCoGQ(IX+kJ2zp@js!~?O zBod~1-=!RaKde#KMH$azAUyNwb$wbq1vX{~C$|=Q@%wbbWhE#knQ(tIjXe}H7~Ie% zEY=@v`4fx%jY15G+T1Unt8uwPRYs|gD}o^`tX-c(1PKOn^GbGLUDZv?$?t4uTRNR2 zk(e-Sl*&BbGn!U7{dnJFV0IXI{uydg!#vu0<)-0G?w{V7N_ml(%nG{K(Uo2>dB5Rdf}}Zx%C^T zNCM6tV9}_Q+mth!G`}#dAkN^nJ5?ABn{DUos&J^9*1u!zOQbeljYi*Lr4N*YKqZ%n zO4(D?sF+PstT(7X(U4%MhLiBz)ZC0~_RM3ms~BF>p+hB?o45r(M~!`GbsgGP@BJvZ zSZ9yBuinOe*FWMw*zTa$eac5iawrlDyI0LD2LG|_Y(oi8o{-1c%zRsjxDE5}s(X@g zn)qiF-t-PHc8ivth5578xP zW~JSqaH#SK3-gteMkoI_m!1?3;T`WE~B6PKFpVt8k-T>M9#=~rpYW}7%X>NfSU#8#mt zN0T@h^fjeBU@CRl(oE1h75@QR|6^f9Y;%_adQ6E=km7_LNR*0s=?RT28Ukypg%G!j zUMb7zy0ky@xJI&Hp?TRNC_%68!NiF*eb{TBq3t;N4ntv}kwR-3ou=o&GJ`Cn$|(3| zuG|u&K+6R4$vq|dMCN0VC}Jv~Z^Wj8>lr|a5pWEipB$dK`WXiO@{F_$t;T#zSd-n7 zRL|%Bb3=>`%PXUIw0`Q|!j&1mDC8lfj=BsuG)-HIzGs_Qp@qm3Yt|pP;0N?f)l_!2 z%6LuOC~>^+b<><+5mm~IJKtmqcFfNfdYZPo;_HW25my=vlPs$0yFctT3OUH?h@ReS z)DH90M9e79`IEkJ5V9wSHU?*zzrYipmd4t;xIM2p?nCZR=1(bK6@vZzhO#IsXfhWc zj0>^4ii#`0O~_!{mwu|zEp62!1ve(k2dk|JIMCEMHMSfYjIAhXr)7}sUPwXJPV%cb zmOMofzJ+r*C8bE}But=<&AB!%zJCAVm*z7z+8pqH#`az^OGYy`%Ngfv&kHdhNhDPt&4H?B?==I+H>`G=DC_Zm%Cy8Wtwn$(%R2BYk^*#l$ zC?v|s?!`+LW{Zm}XB-Yor{PZWy0_YWgw7VKQAFHgXPI*Rao5ZFG`# zk8+!^hr$Ft53g&A=2V$O?ZA~^S;`j|kj&F8siy4OCjgU0jM~;ab^pvb=#^oxH`1D* zB+{td`|;LYJDBIfIv{_!gPSD_AWgVOo4sI0bRCBv}(9e&4{Bjc;6mCnm> zPa6x7!v;eMj>{OjaVa~zKfhqIop%F#_1i~4q&NB`n zj9QseOA-N>n^-Egc(Ojiuf(U+$Ww#6))FgT>=;U|T)i%}v3iN^`CPYl551_xW_aIy zW@6#A_}p%|R^~Ii5I(OLA!@F)tE}P{Q~#T6!Hl!z`ONvL!oCwG)GVNwVq05Z-j4~& z{~g{n5-5Aw@_Rj37>b|D^W$h_92vetr-mV-%wtC%(g z-_EQ6!}zL`2lAfRHppRWMZoUtYha>wy@yPu4}KI1{WLw|ocHp>c}Dkmo~lgR`F$dC z#PVtn++ovN<7Jk1K1$0G*z1?qBM7mU{DQx7G9A3(DA2K&;@^gB64oR-<$@7%ka@nh z__8%yXgn-0fab&3DiA;4MT z!R;=E2M(5(?jxSk>$?7508~DnH16y(L(Kkj1=P<>C*4h z7kJ-n*$?%!l1Y6H!G1er=wISQjFj*>s>zyJ4Mgx2cEasgy3bM&f2%ZriBs9fiTGA?*Ys+;ZUbixO+1+t z;-_-4`ts7P0}gEXi~l+VIY~|c*Su3BG*!+SDTS>`YPC11!;?-!0EG~60si~&16VWR zHXU0bi~f6nzs{T9LFNgzyqgKCCjIYU{PH3GAAgjZ1QBFArrqei(VgC<8Tr>(ysi*# zrmIUrzpRtL4mTgjR4ue=|I+PHMHos0FN%7t{Wd|0l>Ml)ePA;2B?x`Is}#^p~l<)lGtD05q5%5CM5a)fAfE(489#%a>YG9dJp~HasF1Dzmx|#{L=`W zQREj`lGWYw=wB=Ke;;0Cf|#m$5=;dDQVEA&Dscau<8;<|%5C6S=)&2TJ^2`2zPvC#f zZ2*xw1(O|cJUz7vY%LrI0<8;nI-WlALi{;zUm*N`{*doc!nA+fmYJ)*CvdNn_sgZH z;YAy*6M0?lcpuhV;IuodDGEv>M#Se2f5&9dC)V&pyES|2yLB;F8OgUl5XmaBZt8A^ z9}G!M=I4r29W4feAT{i9Iy;PO_Gfy>=~VU5Y+g9Wu7B~-eK)(VeZ2mky$X2W1qs%b z2@WT<3PZoerpIgaAqQx$-^h8rSB{%4HfE2g z(x_ahNF?0rmYDK21h=qBtS`|SCpG8Y5g_kX62Ar4<3yK_&+VNRo+JFdx|sAju|Pdt zJZlvykbmfuW$*VtV(uXEv(xvhYD?yHmu&Ev7u%c6qdw-yv$XbE^ASUQTvpi8JZ5Dd z2wnCSqmE@boaNuBO;Z^79@@{-y%p$SjmNxH-$0VoEK${tVyU*d3(={Ch>jWu2xuZ^QS03J#Z zGGC%{iL&mpZ*GZpgO^By(=iz=N>|)we`JEJqoClYrbLQduRD9OAPl<1&rq1+gv`kA zSnMm_-ml>}f(?7*Ts-&GO0;8aecw@wfgTHE2LdC3-WI!a&A~wUcX6ZgQL-k^_~Hvy z#p&^o!{Z&iM8+}{TcK(!7<~Xd8>F-as@~r^N@+Y3PwND(DS59tMCMweHh7B3phY&} zbE{%w>5(|;gTi7WT@NsCUmTo90PXYoYl`hXAr{4dFDfK`*e`G1WQkzHef{~YagHJS z&N9o{rld7j#l0yfg_lwj2%SO9#L;Tyf8TP?r0pPPGMv_)t1?b1mv7s0UI^1UyWX}o zNqhNOtdGEQ`b`lIQ)VyHv*nD@&%9G|>($YS3iExJ38D}Twt{3{XVKM$2E!SAk)mW3 zdH%gm!`t(%>|m8=%fzcb%G_F;rP%xl--+RUk|<0#y0fQ(%XS8En@?x2H0xG|O9?I^#=Fo%yumy55MMF(e zm(DTi!C$B*q_LsD_B?Wm*g(}UZK#Wd;d6`}1Hofy{KlFdqg8_6+peGZ2SRPC@Uj9Z zVTG%L74_-9OB{BQ#9%*EnG~>`Pa>n@e6`gc=ke2{M}N^SDw#`>y2(PUb>n9H=hqeX z6Bw0GmzaE8aruT*5LhFW&bcGWF{sa}%;sAIvCQVJ()cuF!RQfsxnWW2Oa_Tnt}7)Z zt&gm_k};Ij@KceT9(^wo@_Q4wto;RdzbL{4jA($PX>G>ew4{z$g=IP?a$1yyyt^pY z2sEhoJoeeq$r>>w7ElHt&QZkfq(W}fKpJHOrJ4_04Z<_e;UQS#{SP7oM)$)@COEX{b_oG7V;ZZ=(AN{J z1wY?cgG$c1fHnoB!#~_0ApA!*{1M+r?1+0jx|V5j`=gG+Ef;UQpQ+MBFRT`@D0oGxRNHWKsK1a6HRcYI$mczbB`Qiv98V|#H*8HsZbCD=& zuWpMbM*?mNEvfytSuJl$l#8|7%hS#~FjpQvYRiTUYblT|3Q?1=FKO#}-F4m1_drN^v*cW># zq!UdRCrpQQ%(N9eU5;ssxxHvPlyASZRqV=sGa zf(70P#uwhvSRIJxPuXB3aJATh`MNfn_r_Y2HKOOZm9Dwc~p=S z339`43wgX*%9HA$XMg->r;7=Kax~=+Q?kX9a|R;2?f{X<_Ye^gZJx)IOfN0$j${PP zPTu)fal1hW6?Y&oyyS?#VfyU)MfP$*MMtH?Fo-J7qA zJIjH{k6A5{okH>yH$x#RTvuMAmhDXLvY_y|tO_DBx8Zcf)$|nL`2J&F5cB)lWebk) z{k(^NZD~l|iblRFd+hTlL&4A>DkSnEmBI5my!3VY2A0ftt_;l%TRb$W#(35?z?skL zAiuP^O|MS7&+$}zPO!o*r1TD-ng?$32+s4o08;#61x@Bh*Nr1}W9~y|xDW=osyv z4Xy73;l-eoTziq|r6l<%c=Wh6tcMZ}lMe2pRXmt#T!9q!l5I{6C48RhZ^^XDL|amv!0 zkk^n38cW|bS-jAbld^eDz~Q<;!p_!y$)FfSg#U}8&#{8~`}6t5ur2{Yci7j(wNvGDt>cy{unx|D^C>WQ^n4p~-O*Hs>ouPDdtv1H2Y(2R$F^9C;pv z#hF>s>P4p{$>DtsprMW`EG=;p>i^^H{qpyyV1(=Uj!P>M_<(}hna$BtL&aK0&wsc8 zGDdMyS(@9Z0HdEwqT>t03D&n;`o-2e_@S)3B@W1{GVaW8W#7`-%$;OOi||YQLGct5 z-bD}b0?t4C6#7%y1`KT+OYS|i1aiTag<%9qRwS=4)oMANwWe$bG|pL@4@aC(-0&eg zv^?#$fmjPrq_$@r&TF1dhOB*%YE79m#x{!Hg11Y1V}!&%7oqWaOMRp%ChQgzTX;|~ zWjS1hL?UKL#~&oM4@uerTN= z$#H@R=tmmNm>K8>B{}V7f+j+CmQyoSHm*za$^ zAityzC+g@)`NMA!Cp`NVKe#R78j1gXrDyJuJrGDJCOwD+O3(n2ovB8{6ci1y0xd6WhTMpdgN8+ zxF@jsI{!p<>f^$D%dPg$_smu|FdEM@&>)di&Zp9T$IU(_kh^a57dow4z8((NJ)gA> zsJSZlU~7e=vxkTCw+UR9HNB4>zE9a`lQsmY4Ht&ry(v+eyz7-1OlF0!1

_Qec9_~30{+sW)m>m2d3k)L{mU7&g2e@zR|9M%#A%mzyy`79aix9b zsJ)-9HzfODCiCxW9@s5q`&??~EPHG#vvnV#*|~(h`ZqKML-$m&fi|p>{5sq-Sum}@ z9XMYMIs@+2_enRBl%_YlPj=p zn^_DHe$Bs|1;kv1Nfgu@3~<47@s)aP>oeQ#>@G=v(ldO23`Hxe`LyD2>ZO&j-(`W# zqbr$K81GqwW~ImVts*c>JLM0-m<*EpHF(lK>cY8?4^m}KfX^F!4E>gMdORed5@?|c zzjw) zgD_3fZTGG7;Z=~vlj(|2zRr$yZ+!?Z+jK;&cQ5_fq3cRg13H7k^ycp3mSMf)PMFnV zmBzAnYx!b}w-!L6V@TUzN6-w=id@`1ym8O8W9o!)h?HbJJZDrXOD=VyQ_z1Py5ON%CJsRF%25E66PI059Lw~ zF_V#m9q~#GyOO-#qB+NGlS7Z3;g8MUy|Rvn_9b<$&_5_V+S8`7;RPB>i znvO(p1fxZ8S6lWsIPc?`tKayne!SIRCG1bSW;0*NsXmym)PjabP!@gSbC=v6N@4&y z6~+LylM<0#L7xwUW2!Zm1L-s+0n~aklRkw^kmr%W`Ht24S-$^42|b_L7K& zNrKU+HaksZvh%m^?~hxJv#2w|>+ILXaUT(d>Pxr#$rLW!h7ICO0hXP!4N{dsFk~sq zl|VLct?fb%+_QJR$CO@7wJJn+k!Nj5B!eD*<$Q;o7uf7)*H@blw}L>2=T&_ZKGDJW@zoSpqwAw3#cE9%u2b9e^?>~bt2yT8 zqtDqVI02hO94oB*#>(bEm(ZrBg;ncjgk0y61CFpgR*OYxfbE->&2l%JH#^t^g99Bo;V_rk2)zUG)d4tDe<1dlK;rqi- ztGRZCWkHz;w$A-LN8>5y`9+k^#ZRWnr~ce#+}oMr`vEJh3a9$HSR$qAPteC~4q3jJ zL#{bK1#QrI&J&wUW%`*UOU`S}h>4KtZ>#+iN#R!wujXofecQf87!IDTb&5&Pmmy~k z-6gP?z7hptna0*yujr3YdYDc3$|W0355$(D$+x_Zc~PB{a5<4X67Mt#o8c%m#0{>> zGCz-f$>q57M&5X7eskg&P;H(PSeKvU)s0*fjVVi8r>dsa7m3`3XSjB&ZA6tt7@Ccp zkgNP{kFQE%(P))f)0OAorY$~EJCGy#5Y^##e~+`zvR$fK$|`i8wyG?lDPa<(TP9yJ z2)3>>wbNcy1Dg3U4Gb3*Di@-U)|tOCfIYT<%RFxIBE`3xlIlyn*k%A{K#5*m}H2lnglA!d)o;2&HE~ z{G7A>ya5u}%xe%c8Mt#Od$_+MVB8OYLw{Sdr0M5LvnZ(VtQgQ|#qV+vU+$q(@6Zh< z4}g1A6X@+p59ka;9FSt@jyg)aVAYbxH%e>xGF_}RppK4NKw}i>QanZG5QIjfIjBy7 zh|g_xMr4tXUXK>r>V1EeaCn2tBTE~2$pBznm5=SOMuZd!)C!5;VA4mv7x^v~jF==9 z)kDBp6cqOUGsYDGXZlfhFqvkRnZxcldi~CN&R*imjI-W;GMgnv{}Y(zOk-sEEUuxt z#&SXPEFXO~ic?ESKgg-ZELS{Y@!(3nxHXPmhk2$de1b0q;2MkaSrt6#YgEXD<|ZYR zOC8MOAs}%-Yhxzh^`Lx>h{wF^Yi>tAxF4{}d1D(?x?{a{ebulGa8VtAS(GyV@-_E3C7K-)@Xn z&Q?Jjr3>Kz8X)~qTF)apn+-@JFJ8&;Mv*BVdbPjlaL^0&moe!l2;pb@cKko)m>W6d zeBoI%yS1kChXbOQvSLl%Jfp>ILLJYGTPWn>W)7S*Fw^v^PdBFR?0s_fOu8VW#oSCy{&mKOFGbq+^5D1H5n)9(CLEAc#kPYFv)^vm+>K(U3;!#V? zQ48nu)RK@$0D|c=F^?_|9anz@K}s;%n^(EXHmfz_unu=d->b%1zSb4*H7z`!G#<&q z3NZj!n-?BYhqK0qiDc9T+1W|}*+VZmx*=e$x&pmYO?eBLIoA2;ncb!p=$_SoHzj_N zFCKxl3tiWc?w3(Bx%_da=)IsVg6j3T9W&X_`g6M?WX1h~wEBGEajhBe6Y^`+Lg!4` zL^@`&w&$!>8^t7-r}GceNp#USSY^yYZSRUT*FCZKqtS!U2iij&&hB(DN^6yv@79yo z>K%p_Ty;4?D|Y)ew-w2>1$mgB3U_5kwNjIE&jMCba)qEKuaoFwAzzj2;;FBYde!(( z3g{poz~7^cLM4nwb&Bv;EEj@ccnGMgyz@5L!!;CyS#R~|WQBP~I_DReRAuVX2KiEB zk*Oj@GoQ?7&083R47`{_O{Y3?#5A@4$`k;|LfO@MVV&&9$Q)-W@kGH1c=0KpL*E z$us)Ft>riIAd?jRGW+d;B$oT@lwoA;I3NA>5bDU6M$BNfOoRGoVK2?bH(?#E{%3b!YfcJh+fGWhV=P6YE?g87HC_#M;h$GnmGKvmpMc@ zzXztDZz?X23gWGe9A%7}{B7>!a7xkvf5-QWxWzZ>?(|wU#;pDM!v6E}bI4`vjn51R6zWvG@w*=3Jg^f1;{_1|&cKDTET`=J?xwGliVvZULBQi|z;JYJ|KW2E+|+jrPKSL1OxZYqCrj6#al zB4tt>Kz}=W7pZ5K7FLs-G|Q8~;+}EQ%6#;odVAHVRihDz*sXK(lcA^D2pN;9Y0o>} zxoKyvSV?EY<~v=3)f_N^nar76e5O?ALi)SI&WGO~K8|Du#DG*T?~s;iQqDMcaGA*> zfxT-5^8pW~a)ni*ZX!3!zIIG7=S>u~xMO}|S*a$4p=HAkEb?#`xazdmFg!35^wEIM zaB~tJwd85YU8jqz`w#=y{lDEWGvmDm53C}`(`aq9nZ)qi1v5)^#(=YVTe z{~Q15+J>E31;>N5|AfEfqp!R!W(m5ZaXy-&r?UD)DiQO#K1qG8`$oy@uQL!s@l>6jHEfb*;Q$H-iGS%WiN6S1W*lho`s)g4j$a4N?z2utDBqwAddO-RdxVJ z2~Cp|@nzUU$v8@7RdBj(K06wh2KbY<$G|qL7K`_!4g)AT@uY zPO2F+p3TR?Vm5SKW&G8m<6-MvB+#zP6Jia@Rw4}p@CKVYug}@yYw&I1xW!xzi+gz+ z#Hagl@9Le6PV(xmZ@e{%gX2?=_)t7g!dp4Fc<{1dhR^(96GxBlp(*`a+CQ~fpC zMpc-Iz{3mEKHt9^4+i;#T_SiXk-rtAom!_s_y-Ubm(mrxi%hLPy3a1Ls(xg?qxSbw z^b>&LP}g2RWc~qoVf()#hG>Ww9&uCsPtvCYM?d1=9qryfv9>TjgkpY)u83h3*i$Gf zFo2`?H4EB*(?k5jL}B<)Uvn})-e3FozrlY0*L%9aqf6aKAwvCT>gb;O&>st+A9yXUwpHh^V*x&P*q!NyS= zZqmc+T$-#^6xQKKgZW-BiGyBc$pp=Q=?Om6_+iJQ=6U0N%(EldOI*27ige3}zmf!o z)=_&tFVec6t|p1S!jnm)QzBMt_I>Vt+y#$NR93N+ClNrkC@k+z%OxbkJWk zba&lXQD)_nJ+;g6^EkEU04?BUD6HZ4;4Gs?k`jM7>1tzjEQ}yA+yQ@2UCkbYLpfk1 z`6yXA4=X($ovMN_-JN) z$I>}GRWP#KA^0x!qX|qSwsJUET$TzJfY$gvi!3&H7a5OSpF37Ooh&z+?Ce*^ZzpMd zK1tPSd}3K_bz-1lMh<6nnQ{-trzH;kuG$-GEQRA@7Ae}9k*%$eC zdinjXUq{SS;~myeB7I)@gLe|3x%^bPqLI4eoX7Jz1M)27zMFJxI$u>9qsF4x`SRt< znx{$(cE>rut>?6+J6o-R z^E+$po_igwKHDu8+8OJcZb&d4Uj~rk1EX{Tm3w*RR)wH>sLRQU^-_F_uE#<3h4-{|cR4sSpP3R(QrBN`7jcN4Vr>#5YT zrjlRX0Ka!1@kx)o?^fKUb02p;ae2F}d#oC`xwt#*#&VQA?!c1HcQvo<=(Oh8r((h4 z@i2*1=_JegokKNW>16(rV-1rkyUiYUpJTd{wmw!YvEG3(gs5LI-Ew)b;qg9>L`gc2 zRrKXE#8Wzsq5#mk#{wSDOqk3IkxiW{xZFME8GD!AeDOxA5t1!65cw9iV;fDiX{j7s z+o$NaX7V_yuayGS4(H1l1%ys;n8sGW!!Xm^DwB6SrBUqTyPLhC?6;t^cpgWZ1n=c* zlBq&#Ni|4r>cAUti^Vo?iB@Ze3l+45Q5o|X?ORTf{IfU`CMiW_`(3GACO@0znx5$c zO}o&k)%fmk665*6JKMo9gTR1Yd!261+uZeecAb%WLUb}NF@{7ItD0QV=@xn9ZnXW& zgK6asFZW6lUNU%+02qi75A*+nxrnuy38QErm9JHi3XCrEXc(nM_n>7?dtR|BaX!H{ z2v{GnxqM+3#5`8;1EqGU+?~;pnOf$xE1jtuv*2TSiu*VW9+90Z*P(?;i+cgVUJ81o zLXR9e-|;4 zUDW4^@X)0m>J9Sh%-3eL=Ri{_EPC%@-*Mk?ntu)u>GCYu(A+Xd#0LoTN*jeKpw@A5)ITN-TE7L=l1ym~ORwo|BW z2!`rKdSGY-?^QaNnRKscv0pUm8TW=@4BR32HyMF}%GHL5TJ4t2qv?DqTs9BdAsA#P z3P)qyXd`JtKHM@WJ`wmI7=#p1z1$ZH7TNp_QFiTr<(!(_P6bct>FsUP$|VXt0ge_h zWIm?=S9hZsggb_z@W?|ae2wzn8Wn)jVSA>7r4zhbK3>A8G!B)5@&aj4^HZ}&f%N@V zO13emLB7R1NsBCR_~V?b&HA@Fp}VF~Ce;YJpWLXa&R`)z@zC>6Fe0EM^B1YkXXYx$ zt&9n0_o13ycF%EuR<&34Vx>#P*FVv((WA&(w`d6C5%(BpD&IY~R<&U;F^|JcJz>mI zlMi40^p3*u)XHvStL_?vtkr4@qB+OKC+GsLt?w+-YrR`2aA!CbUyF_wM|ajQWm5ak7Sn~C;1qHZuW%CDhMR-i zI|>)CwpG!+AM`3xrK&!AUw+5`7N3HZQyiiwM$MF{SgZ!>CB&Y;inv>-Px+)#cVd1T zit(07@A?sd1~+|uPaLGT80t_oCEVRZ}b{-kkh(B)cVbVM^f{`7F6np&p5vAPr< zo6kdzNp zfmE@@zV}WjFtdmyOVwq9AewB7izff1w$U+UD%$@(R;rUlFeR=R8F?8W-aSteGYPux zNxR^pr;$}C*g@LMlf8OB!m}RHM9lnpa5R-&3WLO+8-jFrj!RAcU2A|Vzui3G!j5!x zwZXfUCm6uvNZ2?cjHZuQYYhW)_MnZawWBP$0d^+=uvCI5jOme|w|?7Rn2JvN#H87z z8BZfmqsE}dF;@jkUZrN#noigsvpLLY)@Q9ID;LP{G`MU3_?g@VUANXhY?YJz{`3e9 za(V6Yt{=^U(BFl$pmxP{^K*!a6$v=P$y=L%l;3PZ>Nd~ntYjbTl%QFld$7?oh|*fkpSo^%-TO~ zG+vQ8YB8!623#Jm&hNNBT|n2HTHMYGKW;E5EDDVsz5fY{c$@M>r(yw)~ve`M|VXW^nqs zmCX3~CY;^yWQsL9WoF18-Q=q^`wcF^Vi#+vRJf z0cZuUF3qp90OZdgJ&qAp7)L*AyQr~G5T^{(T;3>xE(gY&(ZMFI(DWp%(~O|QDO*4!o;CYYY=928CKy& zBr3Q-%@8bNzZ*;liS3EzHS^2rm1H0c4e>m$KmyulW=aqc;YUtX2oW*i{S`k$p zy?JLfFSxBm%4d^1;|un7jA`(*k8M-i0SG z=&-axQ|dcg1z5d{{5drB@dk(4qV2X^c9QYt|_BqvUSM_2VgG#7`e`;P1{yQpetKz_GNy z){wMan?>TNVs;AyMU$}zc|BLWEpS|#-KJoQ#J=KN31G$ z<8UZJKSD8V$}R|)KG$f3lL=*k_D@3MO069h zK0>{{teq&>ygz;Cny7Lpie*tjnK79wp7K4Z?5&Ai&kbAmUY4NS?_W*t^xE8pb{d~H zRt%J$+B6i^K0lU^?aecP(3D8QZ#^>6`REFr;Bnp#aBHENGr%mtM~4>Fx@{ZnI=5$6 zoIq|CRw;j6pr5I0ls$xI) zF5iZc+de|QN7Stgq~f@DB%MK@X<=JAJh3|#@h9@^tK*8>>*D!tjl!sEesrYbbmEnI zUnR83K^)P1uV#nnN8YkBO%SH_?CPc8$}~Y|t-kKgjK%bfb#;{2<7`Te)x1sU{7WR$ zfrA$liS*M2jzyU9oJS*5(2?ghc>ODn)73_KdqOQ22&=)i&dE+LN#6sthCK)bdHVbg z!D6kvI>#u8XJ5~I5joup*J<4+4!*~&hfdoS6#D?WDuya)-w4%}K;42o`S@9H4?=VngVWo19`#TD6 zC1IWahJYEgjgM^Ixmi1A!im)^0iEb3v+en*bwJ?$CLE7uIooRPt!+kA=_q--=A>ct zsyC0RIaJc_H<@pSF@&rQ%!W0Nt2j@kRU^D>v)n*J!-&KI+j-2AVeM4P9kZuL=B$@T zdCQZ&d?7DV$l`)`%DR&T!}J7hw~9f1p&yph1Vb-!7O#%DcdXS)$%URd#F4z9Q9I*F z)8+R)ka{n~sHs`m_RAoHDS+^u}^|vtkb-nRCV2lSrF9ma3#`)J8AN*xG*6OS3vW0v_HWZe(6S9;gk>9`?V>7gS zK-mKE25nv2V;UIZ-5eJ?5 zxC|aCs*{xMN*hqdh~LPY@S{z}SEx@M5AQJ6`Zo?gl9|UMM-XOrdEqBa1RI%gAaEfB z=TY+zh#FqxNXE9MDh8HKoOMsb&Y;|DDgRn8Lss`~?*Xf{=hxLXWPk)c>Wfsr9yr+N z;>1Dt(z~|xFI+&q?J9GHR1OhVhkaryWasY$r_>+QoIdMi7)fHKz!c%r zZ-tA|0?zdN1PlU#OxF(VG4yx7y%eX#fUx}NJ!Eq0ZZB2OgV8F8Mj|RL#%}>BCx3tq z$1}r?k@0Cxdx7YS7H+}8BZyDKupJ1z7nM%4sN)R;bZoIc0e=xA;aq% zhlXLuy@zaFeS>!e5Y)47B@!_z`I6N^;!&gqXV7s%=MkGZW zZWPV4^z9YVb|9kDSQ_W~bfR>0b2g8&a4$hOx!taeZ3$v=4wG)1>fPn7vMwwIcEr~U z6#N7%g1&=@^q~re0uwqwhW+0ywoo>*6yqq|!`2(ufCWW&IY3zK>~g}r_I!1v#jM;E z3utpw;;Sqht#$tL;gg+v=)C{x1*AlUP`da?HuV zB=62e@4hF6ugA6piDmsdj>&2B(Yuaw1W#D$chvH_T}N=(u6>q>BDN_H$pXGwyMpzn zXKzdXphy4id98TODs@bMRKPkRjoXT1Oz4|Mz^zV;)1rZhn9wN1`>4{Mi9A#2uyjrR z)Us7W5eefhja@3NSfY(we{4*f+qmy{cl1*z=YlbaIaCnK(e#larEa}i9)I~2p6^U| zlK{Kt9?O>RN!rQP`K1?+XM3tl=oHx~8OcYNjLU~(6^onh)n?aDF=z_qhTj%C;jR~VX*`>4sqBTMJ%(!xksx;o{ z2vE?R@!jErHC;4{M~J>_fzvPaM{|^=RC=Nq`1wZFAzVwslH4~VBp%r&HSN0NSHJ6y zffI-Hwr)#L#DK%NyfzB(J5=yQq|An&%nbenVnR)Iyrz|`OnGvOt!gys36cMfXY( zcs$LG*bZ`5OK$rq{SoaJ zTn0f-{liH$mi_G8rG8O#1bKE}$nP(o6dWrX%=9*W=av7~1Y&oYDE|5S}TIXTvwr=~7kT4&fK!RX{() zV{ujJ#|6B-t32x+5&T?8~OPM$vJOjo%#D>&` zWMO1syNW2&SIs8q$vR8ftOu&BzSRFL>W$lrOTlWZ&I+j$SQ-jpoX_z(^5J2GMZ*6Q zrZZWJ6>u#~BcH{f;)_bGFO$U-{FRIBN7(6a4)eLXxS{Pk&@!87t5CqUXm`JFly<=> zSXnt&l_xJK@q>!Ywv?Ms2d}evkv$%2>E{Mf!C&`~Iu9{&-On(&%}8oAktFrXCl#=0 z(Ny<&B++PPR_Z%zXh(gn@7_@6GLF+1mdXJabF(%C)=UKJmLm?Lj8GE3OC1M9Ee2V0 zJ=7zVtRSDaNLb_$AVGttM+eg7Gt$sc=GGYsOnMFArIAT5Z?yYk@dZ%AllUO){8+wC z;B@U8niws$2lSqMxV>nOLOKB;z3yG{$lNr6k9gk(KdDiN@W^jv*LBAZ$L|C?3Rvmc zc0%sT*REe-9c;?@BgFRDwvm!H@jH_U&gruwqV5oISdItK>BeRk%BQ$Hnx+lB5ZrqE z(UfE4i_C^&j4(4{zw4V;j-7N?n?y$!5IaDSR_=$n1k0ue5)4DkA;+b9qLsZ{@5Q8s zRW2gj<-BH}`{dL|qi{1(eoCiG5P#^S;?#jh5&xbnSMqbtpr#EsO1kZ?~S=eEnuo2&hsm6H77!b(nYd913Q{wFSx_X=~{o}IO zokPRtmN`D_7LIfGQ{CvX7*a$+?$=ea1(5Qrsq$OlXcC<_f29cE``4y9hswTRyTUs9 zMc0Xe#eN1ZH0UtaM&oes2@dWY_;4Y*-2|qT7Tp<7WX&3-6`*+1T?enMw9Xgt1}uvX zcdDt;DBeo*HrO6byX(i6=(e}qWynuASAxcfWvrf9fbV*WDRca2t6xjcU&^Jjr-6xV z#PaIeMMBN1k;1AXN<&j<+GsJ0&4Q`t0PTrHoOfCgtB+T0ke@KxrEHIwKfM12o2^f* z^r=?w*{0iXLFt_A@PM30K#sbDY>g>~m!iQqw#2C@LzCGrrH*`?AXp@;<89}qc*@Nf zUwF|<1Jk@M9CkUO&cbMQErms}C4zQ3mzRS6YDilLkG8m(PcVo#%)Y(v>g=zD04FFS z5ktIplM(5RG7_Ifx1SYEFz!&lvFZJyny-+A@A#-h(B+i-7|hgUraLQQ6F<`E&T zxZ`px_fo-^^(M+r2ut;SZL6+A>kn}-c!cvq-JHD*c3FjAFs36pPB6X~>w2So9@eO@ zD@csgt*z;MBY2gwv+@YWN1bX>JO8n&;$0=GU2nLp6ZR6_EiZ}gESrzJgGPU7__=~B z!rT*Rk&a~#^-2vwd7$0t&Bq}ep3WlbP^q<;Hoe#qmt4KWBKUcEFeRf)(j97L=HhJR zXM+4v=Mbs+XSXl0KNl;7$%)yRT9s^VSuH+XvsZJ?9(9M|F2hAYnOa%MD~UjN={aAo zdwR)ejZxmj&wSOoA!;1COnV7AMv=wURoA;1s1AYFc4e!%HbmMm)$H95tioYB%|AN* z1`zQQPFFqklg|tTyF9{EKGGD4vdV-idy4B1Z=asb<(2(ME1_yNEBi$P$ za#v)1G?xDbKtsUvcSvP9Ei>hN+CW5V#3!D>ZQ{~g`F$UP>60TGv&+78aGq%D?Dhq7 z{7>HtOakF=urY?gpNjJcdBCvZ@Mjo9h%-B-dX9}?EIMZOG4k&7%2~II;#IeRVM>TM zWm*@*F1U#D`L3KloWt^_!`;A0|J6vcYVnsj&1Rg3S0r)>w1h$i@kz&L1QWP|1b*X2 zu=uyYdE^ZZOL+v)L3DLx)5T1Bc35pD$?j)0g8n#V*6HyfWZU{n-_kmS0Q#1R`^#6a zq@woFLb*(vy|JGpnBh(5{#MiMR}Ow?v)qrJ8U*A0imK(U*In8#VW+Uwj^ZN>g*JAqI`aD#90(oToc{*B;l%%k^lG@ zrO3{q8iCyhbC!t5P1ARKW7%r7 zR9$=R3blgzh(5vGV!DUnt*i5WD5J9l$z||@P68}@$$Q<~-eP>9yxsEs@pq)A&9ymH z9|r_B`2{*t&xRpnoVx1!1UW1p&p`Av!dU!>GNl3jK7I4+US@_OU9#}=*oGiXnIHIa zNE72&@W|<1-us@5IxLA+P_Or^>H3(o0lgcUn-Q9mJqGO(%aP`dDKFDu3L_ZG8o>X1g!;5E;B0Y{y}_ayY5o671z8%7#aM@Hr*E=pYp| ztOFh3br93mT?Fx=8JlNsT+mG%hF#t4)ar+xuALZdGn{84)pfbP#4l6idjtb-r zd3u_cH^b=$Fp0p|J1(1!9r26}A3+GXc{<#~Ce+I{Z(HrP`h(0y;y2r{c$+&=y$0CW z+MSvkN11;p2Ko_WM-o#EBJD}O?_nCR)mi4E$q63H@nt#+yRx!|5okrLPKDcbtEwyj zXG54x<6GV0tS`l6f*JQV{MV5qR6{YDR~RSC(iGe*`xRO~o%(p=?oENf_ zFT45G0Q&09lfKNIgS z?VtP!&RpjEbTTWf|6~Uy=z8(U{^T39uD+Xh+97*@Pa3oL>sTEyp{b&wn6Fl{KXh+M z&d0Qvza9gccEL};mmhTuqGC|IOrfv)fX%Oh!nMude26rMzit-nbTlW9+^!(I>T&wR zx$T=|3F7OCs;hOXpbacQN6t4G05G!2uv75^w!k(TVkSt( z02AzD0(TKasPFi@`M!}@0GQBvB(64f83-B$O8VsKuJ zvHp5B0^K2_{b9>!_(X*O9lk6{7$etuf(stwdJsWS9i6W4$Xu?4g7Q_|WAbAU(OpY-%T22D6`w*euR(YM}IPBeMgpZ>w zK;y8L9n%P1?&=ohWN;!t4y2JMN-q(l8;7KkllG85x^V1;8!CJ|nyVm&&x4VRf9Z3d z@=V$JYKqjf%@!4$YvrBpPnj_M;O%~N-3prAJDw_(r`!fb&XF@@r7U4OoPpHbw*I%!F(ALm56KbriZOce;wV$gU++y_bwU)NxIqn8a$;uSx1evC02 zcIPErsx<;dzq$+E2%T8FrN}xA^j6wKULtrtkMKEnd2H$az6{h;5&lKVTC7@~>owV! zLb(d@;9*~6YP!~9%L#{GIFlUvFJE7iEX%jW{p<~~5*}rW7QsrV$d*=q9rfCR58ZzE zkdw#h;m2#In&12hY2v>FyZZ5>h$LQ2q6naM?+@u_u7^#U5Odcdu)K@>)|304E@3=i z^5-KmCJJDv%n?6Mgk?4+@81@=5d&YzrKf(!%ui?-@Q~ zA~abq%RAG$dRhNk5$x22ZRmr_tMQYXFix}!E|S0H`|8!X%kzQ`-u(8b%H=craj?x| zbc~KSEOkc$7GDj)nSr5hAJ+{k3i8%}Qa2H>wqP>ml*Q}2q940TWarJ4Jdk?YoLq|% zNt{h~*LPIzf9R|{_(OrrZR{D_m!`^xVoP{1ssn)}`KFjh3kk&YI=W6%*!2@c*bXV6 zX)c(bnHE&6C9z)H9np~QLRH+&!H_ftMAqF?G)+MFy)N!~4P)rxAWPng^CKyDZl!_F zD}AM1S~h;_y0#|;NfTF43t++OALhlE4J;Bt;}x1!_m{z!?x5ywnb5G@FQ^%aYl5yb z9;W>Ub_%`c)5O#DOjfIq#}LfRw~yH$C8I%W{V+h_5QQZ~_Kuzs!8*Xu{MEHc_R9l1 zXBX(VrCH?rULq&xxQ;*E7SQ|iZM7lDquh2koTNPI2b`Z`gRweeCZD})d-?qe-Z?19@FBHmS?Ti`1qjQ0K=jHg`5+)oED@oMlTzYWQhy`?gDSWuIR8xzUM-N9UGdsH>&Nl zv*i&Yd!bB z%fNaVP4hPE9V$hyAL+FlZ9d`{v5^en(#588*}908H#A7@=;^&4+}O(vlM4h({2`o8 zq`r{gU}arWO6WGbkQj z-Ma-;4R2krgw@&XA!{K`OAr|L)>b&p2xno_fa2=ZD0+Nrv zA3u#s|AX!JPdXR&D;Vu75`iGK=${HC|9I`E2#m*O{>Aw1zc^oizx4mV2R1$b*SL{; zn7{5}U%~t2y%+f>&&!<@!#Y58x9**2^>OD0<-YKOI+&jvM^e7 z&u=NFwEwOQ?#X(AKgeqMrzn-j>!vF5t-xoQ6o$7Fk_OfP>qC!G>@bZ9tD|u!Lv}_} zRi0p}S3(m1{{L28>Z*kaHL;_+!!ndQ!1X4a5J~;X|Bui570ijQIiXi21tX+BiBd`bRzPKRBBFo_}h+UtXLnyAHNo9Z`#hpl_wHixU$2i~sLS(_-(IcKZ6Q z8dL(>X+opP5@4!u)guEO1aPp@#`tqvwWPz@GAYotwT^Q0Kbe}s%D!|aFJ1`0kQNhB z{mY=SmEp?fpQwctvY4S{{3sSUafY3Qyahn%Jl6u%Y6`*3)4j}As=s)pN^mi^2R(;4 zW7E`PQABDF56Xj&p3rO{IYjC(poNr zPsr@(Jicr9DhI!QMgf}al;UZmi9;j}|JO8PU&H9xz|jVAYD@WV8*VR?VvM1M)UgX4blNiz=tdJ*nT**wgTB?q{wHUjF;+PQ-LI zUl8oT#LEWHwrO`EkDjz;bi>Ti}$Q9ki(ZhDDXVM7kQW_&#YGmj37uCAdZG zCPJugR05{Z*hqTip|(9n6L2a_z^M-_f4h9|d>Z*eF6$GU(RaNb)pLiMZ>9pGTTFW? zzW?q!o+j3u{nYF~Z0aGtR1y~BojPuZ>KZXsTX9H4=@^w)_mgTLQzuhh>sB*ffB%m7 zS_$V{G5M4y4#(+e^O|fsfhxy-`mLbLL>4>W{i*KiqJVDX{opJEC1P=i8;(;B9qiNXaKJUE(`(du$14J z2~bex?X)nHbc#k4M(t!|OUeruVOratP2s!U^M8JVyFI(#vEQ{u9tgj}{%*V8%j+d~eZ+KC1 z?lv5E=HLsy6~$j>xW@l7-yz8GRV}zZ_4Ao3hC?8ZJ_6xO3BSJ|HX=Uj5047vpPL)l zgorzub#>(k^WOIMQnGqJ;qrUkAV}NBfBCh5LK%uF(T{4fv(oST6l^tbn=7;X-Odq; zP3o>h7pq5>7m6i5JB-h2wPtuG=tX7{EV*iHBst6}tNsM9F@Hp-mHVksfj9w@!BY}w zMD94X3KKAZQK|ePI!m&c#wUg= zshBL2)L6k8UVZ8vcd(5Al4ZEP#pb%nPKY8N4N$^cyYk?`G*r z8~Zge#cp^+I36lQ9Y3a{y(Q1G;CnnY@l~GM9XrNpq0{5@`=$>t0+ zD5ECl#PYH{_e+C!0O??dC6{y6_8bzw`);T$BS+kGaUb=-aP=<9T%JmBRuxwrzWVTV znnJHpus|c zye>;LFp~0;J@TmB4HO*~nRpugdXCfR+Kb1=){GXUQ@N_D#ml0j8sxLM{3g!P;XDL? zFvH8WMx=pjor~biv$@*!8D}Ue{wUUAuA=tyBES1yM%|togPywA86A?ttM*M7#@SSo z;|RaRsnRXR@6D+_&WTKMVy>s|Lu3T>DdHr@CfOw=uPnngF2=ZLk{F0&N-DDzV>1z}CB zn@gyDZP5e#;9zt5L`h`4;0Q~*(XtqmiKDmwG|@kn>VOT0MJA*qop-sX;@oBhXonPf zaFcKqXaDxlCg;C=eVTcsOekRPf%A^>c)c0t!j!&kQECzP{J6DteeBBqO#^E#0s;bs zLQpOaJ!nS+bF~UCRn%6_C0zWw>aGmF-BXWANz=(fJ65nb`g`hypXJK-XRo%b!i8ui zelR}MmC-j>7Ul~Ixuk>^1dY(qyNGd?X^7MgU~IUI!1qch_VroR;t=C9fH)X{7)aV@9szQo*d0X5r_M&K#O9tqkK9&K?O5T{->020Moco-%*Q3>) z?+hHP%i4pU8!{}YnWE+X(>(OB;5;Y@LgKf!wv)zGJ#B5p>G0l~5BeDzS<|DFqEZgm z68Mzty&25b>qtqaSA)9^_7B?6sN&e0pP)r&aVg!tpg5z7t4|8PtVuxTaqC4j#_Zr( zr2eXbV|2qyj}sc5)hB>M_>lSKSD4f!g$XeH3&MOvlf*7rbABP!oqs;H^32`kF-^I5 z#H4s1Q-|9b?nO$u6yJwcw&wbHCK>39G;FFBKomNLnxkWQWRXnn#e;_GVqpk>{ltjb zk69Zo2k>wdZMR}@KS#{BXAOrl_y3&W8nTzIuOWGj0uWQTfS`F)*`MlJ#(}beN{@b0 zDnnS-*8LCruGITW4NvMod)H134ICEfOZnPDhwN$Y1J1w?`D(0d7^qJW=Vtp7Xjpm*L~eQ;=c^5w>b z&yA+KlKEn8HSx$ny74Sek?iof_iH$k4y#}j#`zqB-j;Ki1bm(tS3*z^r~M(*qWGI3 zq%HsPNV>%_NINpMVp>^ISzz{Bb4wvWw!1IT=WPViXA$Cs#-c$?t6+!@pW!_-Ex?z+ zN~_jG?d@qXAVpS!lp$u#)>nrLogz4qDdYD1pnZJ{-IV2^?R8)yhDZ|6ZUL_5mnMo) z+gkR()Pq*K$J(+Z&Y&QK2sv%2*rSZNDe&@RbLf9K9dcT7)+S%FhidcYQQ+NCp9OnI zT-$arU`;cwHE_*BN8cM=Jd>h2a@K!u*sa{DFKWSbh+{Fji@$=`N9ZxFvtJyLY z6-@A(69x)OMlG^~%<85uOwPfZTe3NTO&pxpdj-ioJ(Op^p!B8TqiJ$_EO3+6)nU<* z>L$u#he;tJ!P^xmFVvG42M=4Ql-PHI>Cg;qz7n>4z8z)GouZt~Ek~g*T@-&MefH_V z8{thzaQTqZ?1dB@j5m#5L3drCQ5;5z^wlVad>+}G7f6a_F`jfTnd^QOQ0gE2K zM7*zZFhFSvRMHtGLan3>xs!JtO>I>D`aTJ4wS4%_faz9=G#cNTO4>c2Gn~RHA`JHg zon!?|K8Q_Hnj>@%H|owrs+4Uv<9MO*@E6JQT;(us**j9#4Rfo|00|^Za4n+p2aabw z_}k!9uE%y3lSB(nMt(Y_7#$q(lm7D2i9=$U@dS1eMfG`$w4%I%h`9nj%e44S!sZi& zG3xLZ5sQAu6n>h$YbC`na`Y>i?fRiXibzczF0!a!`)2KYHpH zV6?DOuwQXvaqn8AMhrT5cSb>--0?8ep!r|H&_81$QT?f=I3g1upqd}^bd|v>6Fih4 zxxJF{>1KC9#*xB<3Gd%;`+qEe%&#j2=D1C2;BH@X^U*RlJ1yeFWH#d#bFYv2f+Q8a zRKjm}4nCGIR&lE7Ng$JEK{p(B(=5X*H&*OA5x**WC0Tq|!xPN0hfKbb zXT#gjQsK?QZ2RmT;XJRt0jePRWu{FF@N(E|pX)r`(SDk*J{zzEcy7jTD)T;1-rox^ zeyvT|Zwnf0*B~uf7@!^4{M9bdbm;kBO^8H{CTOIZq>-yj=-j=<-URt9!`U+6oK^Nd zK~Qqz{E`Xf83r6>C0;y5aBbFYdd$^Co8Y%b0x3UnVSC!x4s3tPjtr@tNx9Av>x>@g zZf&)DnjCWaF)h$^bHj1%2@wVue&c;g9lGFu2={8(e>)5Tb7iuoI+@>jkMZ;9ZXMno z=RR;#vhtGS+3=74EB47Idg)^DID_I#^8`}N>}TAR)oGPb`$WQzDiEhMJGrLre7)=n z*~*Pz^zd1K`x?7ES|Lji>)B7ESf#8bqvmvjO{uf$^>7M3^(~iWY=(y>NzCZogc}*i zhubXD{1XnJ4z#B<_N|91gfS(Ko&9N$<&)<;BBB@xNS4yJ&%DC)>E#z^%(AkEPP+dFNz z^FRpB&`F+eG3_aqElHKg_8Wa^wBv#nND22NKi{H`5FEopKyeyPpQ*y97;4wYdp8EW*_!6|oeTkP zUZ|}Atc}J($Em1-6LLiNKHz4Jgwj-cBi%Rsu<2glpheow+qNQF0BL@Q?_I9(NYW(4 zLh)=Ux^aCWC}p$qSL>2~6gUTqNUNXu1Eq4GD=Jg{x&Y;F#OvPVEaaU!57|EaLJgQyotYxs3`o| zR4Q8X8`F}7MyqM%*WA{rr7Vodw`lvlefhBro~ka1gLDS!M9p^S10FQL=4XS&yLZsl zt)DHv4=xbC9?jOH6S18N|8aTx&J9-XqdD!ruV>`ea)xza-dc)*`i@8%k^JPYUVtlC zb8~7ilac#+Wij9_vYD*ac4NVk(I6!Ix5S{EeI45Q?gzg^U)8?BRZ6>)`DKg6wt4&0 zF#&^EMKW-R-ka}%q`9~h!OA~0zWyBGiweywLBzb|LySq`KI?7CBJ60)R@LNHJN9Rb zD(VsjNo7?&#ak|TIBSQX?%bnrJnmC1;5YQFWoOtwbc;N+n6=f3&DB`HkU#&$<8*=A zU-axVd*D=esd|zzbMxhxAKpfKx7zH4`KEr7BewtBe0|^z$hqY3mdU~-Civt@RR{P} zS45334@rvvbV3q(E-s^Q6w|J3@U1?V8tL@zRVZ>szaNRv@cPZHUS-6(5_2737Fz{V zp`W{8srzV_B(DSe<{^cF{Oni zI!mUzPOo74&WWVQmY%Oy;GWAjv^Ba}*vX)r9PAS8)O70kk`kux1<4wySoN=cXxujYvTlW zYuw#6xCVE3cXxNVotfEt&OYb<adT5o+nqAsECSFn~yN{A1-$?+T$wL4?n z+cm;dw_8#&Cq-qo@0CU};yj32;TnW41}(AsVvB40O*jl)HMQCM#>HGyY9Dcg2~_P( zh^Y>G_7rzxDMb5*U(@%Y(}pyOx5X;F{ z*VWWaGNA)z;WR5M6BnkM;9>6Nq_4;y(NN|lUjT6yWE9V;y6XNBiEKnFT9q*mKdRdI zRoF4m72!))<4VoON7FB;$@#|26sx}**J|aZbnJ9lVmCT{*koXErJ}CxqhQP`<}@*0 zC{8LgD|1w*F6#u{)hkw!1+fx2gxO+qra%#W!{6Uk1zBsKj}SY4kP%HqrH=DF`;8Ew z{%zs+%z&8IIaTq+W6+muv*f3VX_XnP@?Eu0K{n|~XJyNFeb0FsfB|Ut<@=9bMXTzR zQ9Wsf3SPEZ2KLjqzfAnAQu^>48fl3QzKWUT;kl5TJ{Y@st&7H`Cr111O0eYpZVWAky77LiI#jveTLWg;n7XePgEqdm+vOe#ohQH@ zha&1I7^RpFsCO@PMf9Fm@X|0A)N9aNSdRN;%{0`ygnRr+}DPSgjyn9`n z@`LEvIl6TjuNZnaBWrI}^|Q{yXYT0XlEs;iz3e>yMiQNq%Y{bRX$sl+$t@Mn@Jn6s zPJn(VdpPx>zO!_E8jiN3o8PM+5VWobQ}P}4T2MhtB8wa7uE?vWwpHfh8Dmle=}7PN z;HB%*+-SM2Bh*sA^_0=TND!_9oc0efS;#H6GixM1!h5ri_fKeh+F8myHOc^{P*Pc8-BNDRb#w^AJbK$S2Tp~f2Z0Xfowr3 zl$rNHMZ_;V;HY^wDdfosRK(S*uJ_k15@^czect+B##qQ@DJ3=8t`Bij&LcIZ>LMGO zOb&cQ)(Mgy&xR7%{)zY%jdY$1x;e^&qxs@O@dbWB-zL8!XgJyi8{Z-{-2 zb=9b)Hu`3H0gvpp_|7IssfQo*|B)OQ2W@f1HOSvg*Jz{M{M0S@lUgbTLP0 znL0#L!n`~777+s{I5o=JU^srY$V?UQ32?jXX5fi@v%H3Eyd~`M;}lUL1~v?8mK=d5 z&A}~Jt~pNI@e~F%u4ZzHt;@WA(uh+7Xz@XN^9t~L71Pi8{%e9d3<2C%qS(?A2c6&q z0e~YtL1O##meG5BcofkZ&Kz?h^m$}Er_$W;VL$1_SK5enf^r!{X^CRKlDMF`R>=GK z%P7Mb!o@Qv;XJEk+TdWAY4Y(0(rMe48R_gFyxr9;qWL-Z_!C&1?l^rZ!=?E-Ts1LW zp&Cog7}k$&lZV8`aX~1cF2c9OVj|$Z1ir<^E`(>*xIv;4{yRs*7g9oz>z-)I*zxL? zAR|~}97AWBlsH7^nM#?Ag?T#x`S}DWE68z@RkX0{vS>6RjfU1?N-kJ{PmsVP?}686 zN!bb{2`B8pIUE?Ldwg)$1#>l|69$CIPrbyW|2!W`t>iI%%4HD_}cbJ`+eUizJE zK5_nPD26T^+%uUtYy!#ms4aKUrMobLftD5y^5bbJ*A|6p(L^WM``GC zgTWoofxR+#dVN;&eR)aRPh#qWneSX^bgeDXx%D$5yl)-P<}xPz@k0nBRCB?Iee^Y; zE22&~gcUQ=;wW#S|Bwc4fC42<%|(x9gog)i*_~?Z>8<^SOegTO!T4|P#+!$-G!rYO z&nY|E;$so; z5qr%X44g~zG1_HgJO+j)RK`vzkYKqHyaM3b*!T`{#XgM-g5`lz6R}BrQzb)CZw$1= zBhq&(l_@u{eRX+=o)7PDtv9_(^*mDl2$cR;)4$W}>-`0!q|RyyE!6jC8_D zC9ln)3aKkW+V(uTsb8q1rJ4%LI5KSy59s!6XgZ(p zGjtlc`)3YTF@n;omsXR%5xsAk-v_Gb@B2C)&f@yQ;n*?0{2!a5&}gkdr6i?lV9hnm zVk;`9INS}&Jio+1NXy`v^{l12-luxK2J&C@MJhG!=g&IddI`S;k5n;S+Fg;PqGUk4 zwKIkpF~m(R9JY(sl<6x9l8YYgW{d9!ktIrqQ$UdHCp+yRu%pqI9L=s7bwDz$<)O4fpuCr-dD5aYaMH*ZT8&gqKX z9VZ$zO^T*aY<{|KwIR2&t9ik}0>aChk9PEESKJBAPHwFLmKLC$D3KH58`E=nLI+Jk zf2!qlrG$3*tE(c2m*dUXxK|@zW<9TWx`#M{{Q=WiDO|+4*XRf^|LN zO~!cLGadLr`kty6X;JF&S%R~#|Gxw%+TrU!x`U9OI1dC^jJ$ksSX5#mBdw|b%j4~) z$5przws$HA7E}&#@AJe_T`E1GZ{=P+$BuS3Fb{p<^W~NJS?mm{Fg{Za+NIH%?AZ__ z`sqAtoH*Y=vw%{+&gE6x*_Oy5Sn9(a3%PnkjWHfcTK2V^ezI!}WWW;e!7o?~-Lo5a zkahMkHx`z6=7b<>7@CbQhu=wL0Heq6tUvE3-y#jmAQ5`;;)Eo#5iR%8CUs@%n5Sl= za{`qD(ro#0@B{yf59Mr_O6&@2i*@ua?$t3$bchFeGQ#XQ5v86P z8SH4&jYt^2yadgo7t zODpAyx^M$6C!=g7^5V!ndPKn_%=m^P>Pow(&0-|H5-0Kd;4^7NB8czx&waSb^hK}h z>qB`W5*aCD#Za8PPvLjnM^SOB@Wyjjxm`734*6ZHEd^bStp_jXCHwc|lGk~h=5Rqr z_7Mf{0{L3NaqjrRtmhR8Vi-c`5IC?1?n-n!EIBG&+UYd6Q_x6h5{K* zv=%#}h!XZQ<@R)ku`Ax9vY7LdD&oySLMc_bu|$sV!3PaY=RI1OFpFib7N*U*%(~O( zVSP-wh`D|Gpy?LHWsk|_n|6QsYCeb$?4RueM1`VN(=?!AqIv|^3yW0R)0Vz-MPDPb zJ8!|V+N{%n(@+oDQ>XJP)<6_4HHXOWw$WIHp7)P6~nU8k~P`v23~y z_!uL9a812_Sy8!|ki3Bd#|W7E%g{I_&9j0vunwLonBmDcIv@l^W9W0S)E@t!^ir95 zr0M1tDbS$)cQ=HW4KsYM;ya8rz3A6QidD{_{C)y+AczI1IH+u z2pA)Z5ct($JXS|^7Cuz|YmTndtNlK~yt+t~`UqOiyfom5FkNMaEg9P2k2NDCWfvLH zid&XqIvDVs&~Ds#mPui>8hg%pnIci?F*j{QskA2xSof*^-Bpfqavv|c#)6t9CoDTWs^84vX?@Q|x`q=Y~)h8`w&!u@qeTP7zgn!J&St-`nmM zvs8?9PrfhzWx+N^uE6ZlMiEtJJd{fU*X(}iIt9H-NP79HIIvi&NBo>9huJ?rJV=tI zXwyf_-BT!V0azXLt)TE*j>UIU#_m{_L@frzSWq>__D?s(oy z_|`BH!bK^KlbKcI*vZjx)bXwiXbT6J_$lcZQ@Hpi{9|j9;S$U>SrGkr3X75rGvN^F zvy8Q{&>(yR_a|BI8{nCZ<~SM90D$-@#y%CXEmDlV+5`+2-f4Moj zXkhB{_}@du5}jn#HU9I=`~Jp?!6*R@+CEq*=HBl7FTTn|{HTJ@OSIrY`5#4k!tb+H z=k9o(!ab4%^OsWi4QYt^ajYxFsK}M@zkmDNAAVnltO9#{_k4dm`X}f97rq`~1OxQR zCecezTMkWNenW((Vq%h>c0J)0*_bj?U1>Xqqk?h3Wq%#W;v_Y;bMU-^P&k&>&pr9xbxwkfbHa1SoayLks>UA4p6TCC z9>4K=fw;nbmg6cP6vaa}JI}f8pk+8g9nn-sN>u5r6Vt|oSvVpS4Wet{9o_V;(01@+Ne zer$YU0Mf0Q!@STAHc!K}z07yJaBNFij^2E}2)fWuG@HzvfMaRd*cFlU9+z+QNLh{= zCNw%|pIfVCGt#Tg-EcrsaX&yZ5fKwL!X|}HMH&Xu6?`7c8WK4abJosctv1`7nkphH zhK%ZEdMqR&BHr~II(4uGoACE9=c1;s($lrmYtOC?%QF5#A{);`ZPn|em|xriORgZ| z_ki&%WQvT+bHDQ1brInEpvGZxM09Cm#CB+;U}5dt7Eu4de~6VRqROJvL}GK5F)vO8 zKdk%S4UJsInjz3&+WC81&x4ONmpK{4t|?*1+&m8ofe!11*B5PV?Dp>;RGe0TyGB%W zbia>&9Z!vRuuNXJNqXiPT@hU6?OuM5Y_Tzi8Q(`xObW0-85#Z zUX$KT@x8#iv*|D=(9k;En*FMc|XIQSGyP`q=HJ)dC9jl`wg< z32qu9p5RrHGQLe@%eBt@TXr1GK3gYq1-|PMNf^*>=4$=#{gQIl+$~`sZB~Z$v)U(c&P`Oi; z$se;?^^a=e28TqVq7@L&VzB(*z|+LkRF}~_7RWLWkW+XI7>>8Fj{s>7AeEv-y-o9a z=h~;BjeT^Vm9B5brfMG~_eMXf3%YPp==cl+DMMnqzYIO%wSd$Xfq{Y4qbF@3{85+$ zXauVall9ekkxD*Ntba>I!?@r~r7D(Y!#Uzvi}=CRC1<=5aHSTVTAN^xD`v1(~*SPGt`Xi{|vkIvu%SMBs+7mLNh#R`^-$AwB5^9Qzej@=9K-$B&&|DJh&T)=DV*&->5m z3Y*Iw*EU?RH)sjQKei)g?Y;ConLIR)x>s*jUbGl&CA~+^Yd+ih1H9l;lP}A`5pXlULiOgh@_j@}ZMNsju zJ(A@|M{``-PP&ZDCM#`$g?rcyP-JQMGIU?1d&3vnqj zY<(ZgkF84+2E|oNeDUgQyFYdG$C@;JOIM^wy7A3DkF;*2DyzLYM^+!1cUmW{lt<=C z$*O0d{i)f0ZwuO@ALYq7XmRDCVy3XN!qovyN~=scP(Xr`Hg48a8cE00 zCE;sMZkreBf*|mYl!sz{_zOoEhQe#U9}m0d>N;!_9f>$|v2|n{+H|EO81#PWX!!Y-hphsQt{Jj3%Sn7;8h~yo896N3%h$KcrOml z#q~yJv?`T46rtos@^#`wySBy6(Il$NnVVima5nM%loM9NR(?v4)+g3LvobT4a<+xaa;*qODVFAC)R8A zl1>%wIy|m#UAm1oDuT9RkOTHj*4?Z##oC_&q@}Bz_I0B5rJ1enJh;40!a85wLoSb| zjBW>~O(=Dy$sz*yeZh*<%c0$z&dZKeS#$JrVv-(2+6O3~8JFLctjOO?e+kHNPMo7y zPWA9Uedg)Ox$YS5;BH_W2HZ?xXC?(-CG0ZS6sq-A9JeDggafKtw>csl%&MB5a2|gG zrmhJ=IvJ9V&_mgxYr8wIYsSVwR}GapacH|v6qo&VLVfJfOWQ;U68wklwwFS&y}LaE z(fuv5T5E5OqW9KXSpOCH%lI@Ezc&>ft>Wcdw1|`dGg1$PPfUwQXC5E_P02I2&S#k2 zRikV}w#^AhE)W_zv?|n-pBFY&&IZ46f;OZaM_s>8^7=ulzqL)DrH2QssY!%pXMhWL zBSGEiC3_cg>X$(BV{*~vgZ?@>F5_u?w}x6jERY)paV$OFc=hgZO8eP|>G+%Rj`^$M{-y@=nY#(1*Wh8q1 z1#|OuGCnrYJD-2Pv<%EuO~)~Zqzp5+%r#27;7aU2>=X8v*OQ1mjEW=&;*I4m1XaqV|9vb{4g`~y+)_&zHP=jIStK25wvoD{JFR#%tTQwP>KaAkq%`Md)$R0EVa(g=w0Nv&<~g_EMld_Z%K&f5K^|@6LsHMcE?Kj@E9i$SrXN zWUg3Nr0Io{3b8&aWi`<(_TGn7dqduGWYA|s9dO)#X)E$Jv@1(S(b*Z=4r^q(rSVmCpX81@{ ziYsqC4mk=s?Q;MQ)DPA3J08RkKe*UQW5#4uNo71e5<2#tpZd>3U3QWR$fSB%{#?=h z<|(ML{>OH-=t4~o`TK`B^GC)MBfdpFt~ZssQkYF;S#yXZ1+Y-bcx-xY?ney@WGf&w z6&s6q5&D@g)?35EBH#yChuh>h~WxJoEBJdGB|aau6S?w zy|sDEjZ%^`PAU8(AmrxOZj${vFQcv>0=k;O|GG;;5~$jkOnY;|kK5VP;jb65ow>tG%?<+s)TwmN9zH*}SD-j{ay3Cl4+GM(EeeY3g(5qBre zp9@NwOc})|Kq2}IQ+{;$ghE$ZuF#mLQdQxTO8^?;R$Z0el zp*%>6)KQKYrNmSfy0pbIFvFSWdxbSJ1|n%(K@k=i$xonY&QG_19UdME84U~Z??vXC z+IRUY(aJ}44^9P9AP|ewsSWJSQ-a1Heojqta70z?2)mwGO7xp89enO_`9= zrywom5IuPvXDpF`Fjx+xE>k`&DW0n(pVgp@-f5S-A$S0?mgCO{21n^Bmz8c7oJ(^d zxPcpNdh!7mnFJhWHm?b;zVhbK&T@$F#w;~2HFxc@rKAvwFnwSpEm%auqfWm1+r=LM z14(A%k9l+e3CG$lFf~44Ha;OtdQB7%*WScxp^DZGhHWNXOcR)xg^^n;RH5^Di%QpZ z!bXt^Ch$YVoGsY;G~*lgd>S;`K0%`w)0 zfkPzhkB1VorkI?lh&Q%)5>Hr8f@NCtqY-reOGK`SClJ1 zQ%M$l6*Pj1mqef&=;4J9LD8(ikXKoLBC#mmB2kPqQRU3%lE|15t;nOGD7}&sz;Xgk zCaGA{Y#Kj_)a_(a9nu71a`O&>?+o(c5!Xly6wB%n83Qa1L=GN?+g$mc1lbGK36!$a0L zu{JZxE7Jl!8`6}ltp7#{nMl4nY;Cdtw7u6%0=briEvnZ!Z5c+2vDScM-LoMms;%T- z=tebZ^Mq97v({YXS}O57_$%xbUt+2Kpr9&A5WYD$R>fCT+@C#f+&6`RBJ-5{6!R4K zK(x#i%@7xjC)L*4X|*!3kyvE~GK6xQsUloQ;nesi4sPl<#qUE(^%-ds(3YM+TnIupoF#(zSnA_Ybph) z<*#4x5+iHx zv*2}H2{OQFY7mAXxkf+JQ)DRR_#iDBil7{epEWb&c;QTzit#3+kpFUGehZtHWmJjK zmv}dA(IU46Y~XDcYLv*}TIdbhdE2Y%FM3%qVa64P$0*aNOp{Y;QJlBC5^wF!5ni5J zXx+BqBUew{#3sPAd%r`IPt&Y+*Ucvwj2()bvy-?WtN8P8r3w{K$Z>MzS=vL zD+uW+*P|p$h4AcQ4wg`#J7h{ivRP*)%JPReHd|XpBxb97^N3|gr#56OCZ1h*KQfkX zU2^?J!TAV(qYYHP@MrDN&6cWk1xTve%996KwObLy?h#X=b_d-hUiu&xj>U>a3)k?! zE>S8;goal**Rv*ZShSx^BSL>``dM$aiX@r%9*}&E)|nU0@a67Ro?H(LkrstMR8?sT zX7h;SAhFt1uAx3@T5*x)6~8zfJi;u@!TQBS{VFLifdgO)<+%vDmgFOdU7@HeX%=n? zgzBw#678ntz&j_=8#kK2+z)Oum_1>`VsL2O9fl>Iukku6N0`ht(jIBVQqfa%J^BaG za}I1LBYd&JDbcTMT~=WJ3awda4;rZUPIJ-$fK@(AOdJx{lZS{k-|6j;`I?Z{qjWr= zZ};5bPRWDGqUA~V$JbPI)Nk?tkOovL_ZAed@Lm<+B7dG$x5I_t?aZR_+r_R)Jq{ymobE_3+{%~9c&mfZTgt}7XggNL2!4K-{MD)E zW{>(pm;qHPD}hxQ8edX<+W*P4qaOT?y?ccX;Plq&EgU!!i%SJ+h`(7+`Pap7h_xy8l2SylNT=+1gZ(uN)o544b^eDR}mK_ zKcgvZ1ru~X*Vsmo6Zh^lQ`MyG;gM^JyI1gP)sulx!jiSzSax{!>5#Le6RCuPr-S>t#?YAC>~>iqdFzcu6idK>NN_)m&Y3;0M1bk0`aiyD>#JS zEb-J$_yCkSt9pD8b*5wHA)B*ZPXx+}!V*qg3q&xnVD5c*U!{8=QKpPeZTXB=4_2mF zMzr{_v3$IP zO9z_iAxphD`Ag*SzH30xAtHmOt-H#3kj1AC1cQC@%PuV>VyZA^uBf0RW`><)sLpW= z-P>V3n)4RpZBplO$KfOJL{|D$oNLeU`c$%U~6 zU4wG4GpU>k3>rfGL&3ETbEiw<;=n-!5h(VQ}Xu>R%Xp+RQM2}*R?%%lHux$Fe@8FEPrKZBW5v@9*TMN z$WnPPeaCa70X8MP@sP=D&GgSGN>N#W+(ne)gXnrqC#mXyl%iH)Bk691>O9xZapB}% zpIBht7~Y(?ioyb`SJ>}{4O}-|v0G)4HTI_z z$4MpIarK!Q%j4pB&*6rJWpL^ofve^k` ze}~RPK(y`!-{TVUbAbQW)vQGg+U|j#U%sT(P*{mlJ2iAFio{~?DK(+oR4wcfIz>Y7 zB3A^WSi}cj-r~!Fv0O~6!tIGRPg{T^s!HEp(v;gz>if7^2el;4ADPp%a@6d9bMXrv z$SRoQ=d50b|Brw0Ab~g#3S-gN`9qfi-$VT`D*KOy8eSymU&V{WsWklCqWZg|web!O z@;EwTmpu<_s3OzZi6q5O7F{5tG9J8QR7H|?#J$${y&6elx3Hag;%Vlq&z7L6x$axa zQrUMG$d$EQE%n}#ljZirO9ovJ)^OD|8Fmj%GO+2685CYco5y}1_yD_CjQ`im@<4x- z$+?!l*iGyGlNvA0`k|gXdAK2e5E5Pzd6!9dY~)OUNHr7^{n7^x0}Va}x3(5F@swJC zH|q6&z;mpM;VjX^4YUqM96JyHoJq`B6(X6k^-h;B)=+$;_AL zW9AndjO}R#$EhhZBmY`1{?)47pc3*y9jM$XNJ&m1521)Zz^|y9n*Ky;FIzPH z^CeJFg-^?sRrdh%FsUvlfuLr*9IZq5)D*0PiB#-1sN0&<@F3fI;SN>Kr2zu8gn_UJ zG}M|+4A)V@9N9Xx884^wZFN)8=4r0P|mR^O>ZG@UtF*8nzq-#OK1YSV(%<2x)YpD@gHt08L-_py%!(_zUR9T!94AlJ zQYR)x9uf{Ju^+Uxe8obKYJy43c4cq}TC8D=5q?n=k1HG(HZ^7cf^hKZlf22uZF8yw zW*VzI511~vp6$C%qr9c71T?T#hdO({TYv*U;sLXn5viH z6v;jQ;dX<^92}}PVS&P_+j;YPhxVzlm)ml9FZJF?q3S676=cRr-!H>s`X4FEOKg2Ea_2w z^($JRn#sDcqI^8Fa}Jc6nBKTv*($glV(gM>1psA7a%gd>qw!@&i)=EzjQB$c+VhS= zAKN?@CZB(lF!21(O81`@6A#HBJ6poh0rFhF%gOQah()c5e0hJj@uoXA41A(l;DXz)FTBX{ewv%SJ*Mgy%;xq#r6RK3uC~H9wzXYznFY<8FN@Xmo$=bbwDp6>2oeedW z^`4)KM*z>yIkFLNkm0ah%6KjZ$43K~D}))2X3urO7C2NA{v#iZ_;4D_5o8craxUFM zQ{#Pt&OfbSxM1}V(Zu}pINSt>E#@XY{Tg_78o?d|JMg%#IFQNa-E4RE|M_wc*L8Sz z7nANyM15ixU)sE22AqH>gF4*J5}1TJgu*8|M2^C9V{Oj}+eW{s{?yAVA)@5!!@GVK zRAwSX`3r~}Fuki$->Wx$5OwD~fFiNz&v^ag_}Q{~!rU*re8onB9)eHh?P#wGwEi%N zhKLyA2jajECIM`t>TS-*pi*?|?BTb4Pu->{?93ysFW$P^Kh;Iwqx-WN2Yc9ESaWM@u3MQX<3$B!>z31(5TMK&d%p| zAzU?%*N=&bh4zC|S@cK0KRkS1k9!p!Ew{VxB^u-eiDkL2bcZOJ0fq)L7d;zulD)OW zI%@5Vj9x#!aitd)oNx>lG{K+<;Ghfh=r+S&I`luqK71k&1y4q#)K$++ZGjkra*?mG zL`0=K!e3FL8oEo0;Wb$6+Qs13OW~^jy2}&@t3nVn2lRmZpeDe)jANnW4wK-pih zZY!8Ju=5QIE6C&V2Dpk>OJ|{jm{)~!%}zDso+cc%lx}x1hxicG0era%^B2(QLXtFM zYF0T25bjUx?Cjx>1Q!O^f1Yi7*j^9-NG&P_v0%nZM5ab@UuyJNfY?X2Tzi*&nZKEO zxFrv)q-;4~9WM$)&X+BYFLMfNJRlGH!|8l|v`2iaCeXqsFmm)#g}jP~F4N#*ZYPf~ z02<}@qOO)=I$Q211%GYrN&;eHj0~6aY|!9lOzCZ&|MfZY(SS8|5fL&M)WH6gY}1Kx z6{aD>9hiiao5BnQ)YjL2jvo?}1pXk)luHN?hgF0lA#N3EtU{vH)>4Es-o#e`76(U0 zA|{9md-ek*|D%6uYSF|N)6G&Pf`$5fLKR3ME0F0biPVN zG!PO`Dy5a=~7NC=~OFwP`fp%aq-anAg! zsTxQ08+wN`GFlp7rcJ=j|b?<^Px7+rj9 zP9kfdR%%b(L+8>2>ulZ1RY&AD0{rIf@@F+n6mb$eLjiUfSGu(2(sCDFAye zuv}fjBPZgw_Er=lYRbGW+3!*Je|;SP)-st8AKz7d=~)?LSypG$V+`uQ@rnb_de}9ndHmnJHo|&sJk}-FLXN?sAb-q^kzKI}D7SUo7HZb#z#3 z*bcw9J8^O73l2oCo_3-ugHFyC-EEJBqu^VwunbLla8>uSOPOPNzIDKg;bnHFRo8Mn{+;$>RTMVh?c`ikB_wie%y9+1PSKW{6C zBi@|Q^(By<`y>>*w51U`M~GX7rkqPm%I%XyTo*gXMJslyy(J2KBc7;E%q@%fOl8`u zGp@o=&3&I9uPtKL5}+khQNf?7njkl>Lufx@sO%Sxf(}~=dld-#wt}1@K)!4Qz z*DeOt7;*1^U9~NSf++`a{v=+3RkbY&h^lyV!>Q<4^Rat-NwT-4jD7RW|A8^Y`!j0hl z6V&j^5l?Qu;t< zVH@~gX@$}$?|%1A(kjT4c!b~FR^J}6eiy-NYX#_8oBHOE_8Mg>P^e08{M|14*Hh0I zDdeUEQ@1!`m%JUd4Hj?UUGm02=N?^av5;um7^M&TxM}7>qbqvc6CKdqKXt z>L?Qq5=BL^1}O7|*@l%g?I2ESUHB~NX@EFG27Z1WfFQB+suy2@~96=kJiiR8r2$LER#P!dN^SQ4nx zDAJoUczKlo%{H%2My6-FItdvS!tWQ$Vs{=Nq{RHEj=2A)j;PcK#s%X>0uo^p_{=nl zcy3FNJFYbS-`G8lQSRKW6XzuDZBAS2*gy)zH>#sj0?izSiK3ii?Wi72R`>Bwt9zJ- zs+%y2H>W?F2OmcFxnw;DwC^s8UUPfk{Yew!LU6@-))G7p{%i!bo|Sd8JwP_~W5nk5 z6~#b=%kb^3!Xu4M_HVJlRN%S;PvK4qs42Ru^F0V2Eceq_RP-R|dXilXtN$ihQ16Tf zC0fkE*#5a`+5d^G?lo#&>4D8wDhk%wg3@s%tSo!9TC^z5K}E`KlqIUA{<~E3_=JoK zV|M5gpaG~@a^?2IwPv%}QFzaU)La6Q!XmkN6j9>pjsREXdQY&noi|W< zFxWla=XTX4=j>-n4c+M$`=4+aCz~SEjAYkOSo} zV~SA*(DmNvS*U~GTH1q_k!5!V6GYG(ody5%i!X#h4 zxSeH9S%Ol=U~e6$k4<^6^PS()R0foT3Sl4TQd?tKENO1-cH2e4K5+S;ZNYy(&tt+M zhLq-RQzOjyhnl_rhFBH4e5e=TCmsUHl~6VcFQBE#PL*#>WszT-`N{8?Z9=p&xV@1i zVlXo&BX1f+1MwZVS5k>Y>c~Iz#z28J9Ui;0^!E=y+G5}qhBr=j)x@bC2RW)tO>l*J z6NaMK`zL^ZO?K@M<36h02hfbBd?)78xqhGpleuE6exsbk_0AYgm1~$g-Fb0AOpY_} zM!TbT{usS4znUev)OBiZji(vryUS!DN0+XazFQ#I5?_JYf&a~KAlatN|%7I z{8M2KR738JRmWEUSp4}rz2%-H^NSNxy&;-@6gafY060T$ zvuWr9)opa@<9eY++XQsG&{O(f>0MJMpzfJNs>`ybY1FDdc2mZIKR|a4CL`J9o76TiBsWyldj1b8Ctwm zeH0OvB>%cqX93ZbxbBQ+Bd&cA#L2q?DEC;P{i^D$r*gNuIHPSMZKzU-RcJ2RCQZZj zlD`|};gLQwn|6?N|C8tfmHmpe%#{89*Jt~%_4NnL4!X0Y($PE)RE8$EKGg zIl|zO#%34k1>lBVu%Momwb5Z7#!#F2E0jJkk0>@XI_K_SP|*mz)f0EnSuZAlDJr;7 zQX=~+E@zCd`xa*@7R?aJ7LFnk1@ICIilXH$jy1jx5)wvud#X;pR!$rgWma$eh5P%K z>`Wm8W{sAd*!EB}~SLaR(MbWh1T8ih*GN9%!9E z1kLsJ9X-F{tR<*NDZ!g5ZbA z6%6@zHf5LZVG<7o9)h!%qlIUX5s?DBgsAeZwiGB+Y@MJmU6EmN745W;6#-SUN_rdo zfa@5jD+igcfNs%ZBkC96RJ}h3ACIaqn6#n+*W)09CY@t-c5JKJqouHw4vjY08h;HK z|EFNsB>*|Dbmtjui9NUL>bNfFoZ$}Wv|$gT!m!^+N(f3+3(;}{Vw)V!Kgx}c+akWZ zcE8qRmnyadDxsj$y-VK3?ZzBQBgK>N1#32NZ=Ttq)4jadEj$w62p>ih&Lz;JNGR8e z4Q#(Nepq(q*c^G3w3k+4oBJ8mq3ct0k3lsqX_um&KWo9JcNN~Go?c;>;e6IQFW%HBH8ek$mrUvs*-x1e~8FPKL1hZncHQfNi1u+Bu79G zHUutqn^~kf9AY(U@CqG>A4KhQy;P?38e_g%X6W!TX$&r%h(fa^9+rx#h+D7}2Mas} zQu|5dq$E%PRLB+JH3SCpcqnmlsPV3zFV_uQw=y_2g+gLnLE!BZFy!Z*J2`b^!>hGK zsi{>^bL?x{ble73!Gmkn{++Q<@bjDCD6YLFC`|y4%K>gUr!us1^dzO1uQXI()?s!H zy83iVn@g0AMf!hX2H)}E51N1K9`C)y)FY#E1xirZJkA1?e(NrWKFB%7#72}P=Ce5= z16ALCc8khx{MIvBG%>O8D~YEZ7%A+4JquFyw)q_$v%QO9q+d>jB6*a_VZpOU_ojGv z3|+d@V@I^}U4z*QazD<5-gwJVZ*Zzlme(t)4y$1hFeGgbT8vezZD^3Lwa_rtAYhe= z2eU0pL0MF3Zt>AcC;2zO*o&S}48HtX<bUq52|e>`fc?qKc3cwakTLl-WpQ{v zXtu^LVgzY7s+FLKDwJsT)LRkr7w^_4(Ti@M5@Tl0Qay93=Awr75I?l&q)!&SV0O|icL{M5%Tc( z*x_FnZ!~mvZypC8M{PEg5~Z$=jr5JQJaec9PhCB35nCcbunsOP88|>6d0w!N8nehO zu^TJ#f)u;8xwju`q9;39WD1k4CJMma63yrNGpD6N-iQKr_rbdkA^58MQM6F5w;<4Q zQJMlmT`g*LJ&NNlBD$tWR7;;al9xSLt-4Z2X!9u5{{OM{&GC7yOWREv+qP{Rjcwa@ z-S*czSqn(*IYBRK3FHA@P4q`fHBoc&*ftM z!C2--`;QwJKtpKSb^woR_)1}(H`J6hjigGzEEMztfR0=IVp|uiCsj79a5!9BfdI+P z$KiF=r$6IkuQLc^5kG0Tgy)s%trI|M$&Vfi2-*qy)M=%EWEb6KQCy*l0l2dlBF6`pZUD4MXxihqJTGtckP=Icj3tqt%%y z3VJ=Ri%4qNX?Aq1*z6M^jTqhaD>>8y_j@Njy$EJF)D*f4`gbDa9*3S6hPAhfZVo*{ zfJjujKg^Oj)6)z104!;0b|TsGH0i_T$1NX`+n279BJmQgFy+$e-hnuE!_$2gWFy09*06R1bkF|ab|ZA?B6Wm> zjc&UM>-!eyQ(#D}Am+*1R;R*AJ35a^Ub6{xlwbuo;_{YfI$8d!KAUZudV4%EhkL^T z^$4*+k?k&T__lDEFM?7b#_Px--Va~5+=rC**z1F<1t*M6_)Pryz?@YN-nqH;O)PWE zb_;RRP4%A7X~!LL%z&l8K{5c5b%CS^n37)ZxO}?S)C^O?bRGr4y!n!@0qFy%q9mF4 z>h(s3z*?#@5VigiJK6^#8-*VTjf7yOQVD!mlOG3Yn%AU2r`_UzJWn-zgCCFj2o6~s zd5F(KS!&(chE58-Cgd`xx#V(3@T{tq!N_sc1>qi*6U!0KsEhN#zt6{+M6!)$DZPL?ec(>yNS0cQH_sP%bde8e%d!yaWIBSi-`*nB{?T zM6b`fkc49bd;Nr-_aEJ7A4$1!-mvXTm*_oOA^SE6sMypK;$y*A7vzQpi=ag+d48(x zwpLoZ3$-B}Tysh%MeyZ9Sm?{)a84l%Q0 zAgGz0oGWH9p`&SM&G&`tO5&;4TAIRlxuf)ZGOHoJhxR?9@UUW|8&$-xVc3r4lCa^Sgd>6d-W(C3Z)Ff-s7h*V5yB&T z-Dvc5&ha*jsf~3OX=P;B&E#rp&C#iFFwV;e8y$Gr)HbNs#Pm!Cb=U}Kkb81PqYgno z6x^I~FwB)K>b{%Y@F8q&O!9SegX$#8f(cS>z+-o(2%F<69v!LwDM?CpGTHsP1I*@( zIB*7pvmiXzZip4L!zD#EF)@1Esp5cb>7as^USDae)G6eM4?R z)7#=Z3SMCYq7m9n8{;`!!|i$`-@H#3CF9p;l=|urK(wUf-Cne7iR0;pz;BBO7UFf! zVEiSbc$IgRx@d38f;RSoHr*OX7M~AOb^qsIt6|aKgaXF z3mQ`;p0(R62Sv!keR_SMB6-x(m;j7p@ZWdi6Eec&Bn}fOkXmw0k^F?pM85N|p*@AN z{J>l0)9zSvxHwAB7;>i>lk8$QVGh&BVC?R+-IV>4`}v2Wf#8*2uc*=L1n;kDSB5B$ z%${H0XQygEqTOET)RS&_j)DY#NyXw|Ia;!&N&aPih} zN=EI=uU6BItf~d)MrJ6k%~v3eMbhrO2s}~;iiqT-MCP__I2qwv2W88b+vCO*AJ?*1 znmiL#a?U8%M;Qs{XCT8zkGSn3jqlsRhIlXxB?%8qK4tcJ5(+F(Z-`S?QeW@gL9@8n z5)86j?9qqHi3>Pe<4io_C9Aw{@bM(F&pd1_2RA)}L|AcY9LTJ&i@tB5 zU`az!Pn01@LuHpcp}WiIJxO1WUTGbSK|f>cF~szgJ*SOnm7@0p5n7YQ+Ykl)xdUTX=U`QqSRCzV^|Fr&BK9RaLdT-~nb_4g z+HrPtr!Tix^9W#BUdGo+FT8{(=1~EAi}sxisk$gWUGd9edp^n*1hW1g3kMyOx{p_+ zRf^T(Vo6@eY;X%<@<9w4Ub7cf+0-q+Vh!C@G9 zWD<(N6z!JzWuwi#%Cfz+hW_&4%(ID<&$g63+j$rXVN)3dFt1XaondDU#r$0zAV%hm2=w4+3WqAgAN+ie0ka?J-tZs%D3q@ZDPtf z(S8vkZ3dSyH4xflR)L_+9}1qfCOA!v)bj=75##Y^kn?3M#$t8`Vb}pG>ugqJ)K^Q7 z7}ZG^L2NAA5>k;s<@k|rYXK*}s%yZ@g38~<*uoQ^Q6Rs=#X3QUE3}`=n;m0MMrmO? z9L2Jm`ZBAb$Vse$4#7NDO!@fiYvT7`dovu#o^2>TOm_Zh#zsKg;El=jypgDmgwuMz zmZ#^~P@mP_QK{I+V)f5sE26=9c&~!Gp7VaPz$TBv^vfcbPW0bjzCrKf;})JSkW@N6 z-ieS(t}IA3g{@YrHFQ_QfOZ;*fH_-Fle9_l)N|ebGN{ca<6mMS%PhUdj}yx{t^PW0 z$&#rj2O**$xoAAVX>(y$r12Bjrv%VKx^SqJ!Poig3Bf&fT37p`EopRJ=t-s5l)i$f zP9a$jrnIdFnVl{5?dsfTHX#lyd8ZTcL(|IYZ1#MlbKrMdK z)Ve_86FE`wIc`cMk-p&F=z!X3X$XCh@}a;73}9kismQd$jM^dcdtNxpC48+)>+Y1Z zU3nL`a%14?88@m}UZImCmkT5x8+MiL)Vi?p)b=@jOD-}GWJH4SBSQhrzShcfDxwYU zE8rcSJ*{%@>``tTp{A75qfObkB0=xn9GA2PGcK%zVn~_tngh4u&8yY{!onQFf=2D^ z)P9HGZvWH2Ul%_X1brYT*q9ni#1$zC|BV1Z^da9wAT8}k3I8gW)rny0f0e@F66`-H zV175$7k`{jYFu}@WaQ1w1ML+#->#aQS6|BnG9$|8_V(%B&yJyK+I6NZ0cU5AahQ12 zt+R~^XEx#s=pd8&^Eg;Op2*OyMaX`+zO{Lv_h(*%l(&Gcm%TG$oE8`5QZHL#CqS?* zP(XX58JaBUx5Op@VGylSP|9Z7pzdTE@9a2hBjahXfJHqTw=g%nqQF(h>3Bhq3HD5V z7Pm|Z83RfKJ>u=%&gL8k=UdR25}m{SGU9kIlxs$|KrVgyourV^C@h!Edc`sMc0=6 z6rYTxh@PQ%Uwugfr7t^DIPW0kp3)LbW;=G~E^=^VM%J?4j;}LvAlxzx9da82QBxsI z&h|YVg=+8&n@Z-pS>1;Uqs+o-^zI)ogFf&xWYwqQEV+W=aXKEqfxe=^L|-P#&xC;% zVCk=_4C6NbshuVFyA}s;=vR@8syo4(-ead6pZ8CxtY8T%n?W<7w;4Mh-zuINVnSq* z0wtG|lJ>!Dk3+l;4%F9e(p7m|FxuLaZ}U9thZJaKF!twNEyI+0(-c`W2*7C{^d57H zrF2jlsRG6q7e<&=a4fg17OfoG>wYzco^(XQMhmiCvMQYHE=WD(Fql;6GX6)Yc2|SMH z%!iYtzt*bDc|5)hKZeA*psPyNeo-=F?pJ`#Xcdog0(9q$a2G?%39^D-eopvoluza& z5V_rS>?_Ez_I+THPA1VqxhVAVT+)nB*V|FcgYFKp4fPQIaR9%Z@AQs%c6r-}n4$OxhW2KhWPEtE1k?%=zGOfg0nQx@ zqD+0%Rj5>iNG<8zwzVGG%?h2`$cdrA3tYH5YTwymbRN03 zmXqfr*mLoMT(5TGLG@J+8O=q5~V1AA{xgul5@Z8^I#)t~_ zr9h|D70 z9dJP4lsl_1$dLK=9;s$ZKHM6E`cXKHggwX@MLH(j=HT<+r8Yi3$ll=LS zCvL$tPwU-|?UWeqadJfz|7vBz3zWAm->hXai691!&5>0|cydKiFrBd49)-n6*idfr~sM0ZUZzeWGGY7lEn~smnl}kJ*wV z&euRU%1c3~;>2PHhr1OQH&0Ka&5i(oZoN?yPAv6rGY>=-;{ss}4F@?*O9=sY;V*nO zHHogGJ?pb3u+aTrGS0%shbWiD3uuW&+wUHWiMSI)Hv6$}g+ixJFz$&z{}#7UW9&eV z0Tn`)MZ^QsLSFY(ZMw1f>)8lOK1w%ax-QEF(2>Ki0VPmJvQKX7_}_U@_2FA#%sDK|*>`}EE)5KG=oVd@PFW8Og}`P$Tw;4T{*OXfmxH^pfT@D*Te!G zx2v_mHX|v=kAeTVM`KLjHgKj<8dkXkNCrP&;cS&r(~knK#lL>_@A$cZpbKkAx*9O9 z=#NeQug|&$$-98}2F=EJ|8@M|)#w5NL=|WR%>7Kjx@PnMzEAFpsHg!TShx<2lw+)C z1yqzz@MgiB9G|u7bNXT;(~%y|LyZHNaq zLG%RhS^tjef1fEU^b>SU7R2q~`N7om?ozWE|4yGqDEHTk!FF(Xz|@MGw|Me)`*Z*M zC*O~P-Jv0I&Udls%&wb0i&KOid(pshHFl$$ew%D!hOaujp-vrzOh1=D@**|J3&agL z+B4_@RwvQN)$gRzBu-^q+AB^b3i&^)^P5^=iulAmJ z^MN;(*c0wm+1D_w6~EpLA!>~)sOQP%a_?@`_>KuV%Q$`)$$;6)cWO}XVmF(EumS(4|S#s*DQ z{__cjIgH)Esw;>yJKRZUH52W^ipLLz+!Fb<8JqWw>u6>Lbstui6ri$8TWpk$$s@-& zIbp>vaQokZ$Qj){<#?dnA_6|k8Eq6FHQn`K+v==2Y+RrWc)Uh58W=Djm6136__koe zJzM~{wi=`%<|C%BPEiw{4D>`xWA8SF*#_Yq40yo_S(t(JWpDWQo#a?TK&T(K% zqWwWCZEf_skB^?|m`HJ)tod41=FleDm`eMXwhC**kwAs(Mvp|vY`GZ(G(@@;34xuL z2Z>r|-G^c(_2HXeh3%C!yJ2V$NIff|1;P&Z!;}^~D?hEdm8rGGAiAy~zQd^=g9U?A zI_Si(J&}m&Tj|hII$wN$;{~p z6s6bEZoH6#tl=LKF!IpP-w&AxTCQhduew*EtcVzo?;dqU9GmloIFnqyDVvw7iEH%4 zODNp^>DYx*pklpAq<}W>ZqUu?0i>_qnq<_}de5FM=MsXmfe6C}bPS@zF_ptP5#rFlk+yN{&7ly+(;NNirYZYgp zsmtzs;-&8c6f2I^FjpQT-ikd2&JC*YVAv*vfjuT2A8&xB2o?b$A!6(mqub>Hn+J3^ zduy#_sbeVGc86Z74x5=0lP_wG^FtI(?e#4YWEC$y{ z+BeeFQAuuxzdc23V`B;kd|*nsRnwX$u3tgjrDY6aS8?j>7{72plxR1Bp=Bzj4$7{- zl+LvI{bHuriEPVUcc2r!!1?|pmZTqh$aGu0{#n?vw5~=n2!*I+pcmK}s*2g%1?{SfkkPQ;)E91y`-YBlD4ej=QQK?4SW1ufRLpG=L+`>^p5K++yXR|^cA!p~< zt0t56bQk{>Ulgbhs#mpdyW+b@sfbXX%mDi0!V=VV&FMj=$6cbiCyQ+&cw(xoh>VL` za?By0>Gp7T#;1I>M@q`jhFv*SmosOl#5iC;Fs)`xQ!&=wbc;hh_=kxHBD;i#Ao2Bh z1CO*%q)+~o8s~w>4?Dv)07ah<=v0;rXKyYF7~$P>w)!9t{l`pb{r!vdjn0~Q9 zj+HxhgJk-vkuEhFGe0st+<*u1afQemzbYcgO8GvG(FV3X;*F2i2|kRx8LxE_gskl@ z6|XhceBZ$Cj2#SHb92We=IIEK+rs4x_FMJ%CT%4V^W*VOS>rx(iM<#O&K>uXj6^+& zH0SFgt*cAY#AZ>GcpKD>*sskOPD*xg>Zj}9=aV}t5|qoV`73L z$7>G{#BT(;w6krk&aY3|+j?uvFHL|Bxb9v+%HRILbc)!WJl1w}~D-4{N8-+AX=p?)*5E1i$ZH>KAORCQDtE`Oiw~l?b zpi_c|4?-rx&jMF(wEcX%pt_DMqti!Z<>Mjc9XJ5@v z&%=V#(6At>8NqShJ2WJZlZps}P53(PK%iEkGZ;pClfd*$q%b{QQ_-H0!ILShw|u)7 zD`D;SC6A+|kB@g=k=B!c3ysEc->*FNYir&L;2N#_tZmVkqf10f>UjX`j zkm2A&5yX>n$_Oxk&68!(pzfXP1G$}s2?&l9Vss!gs{SdGpPM1o!)^6IhszrNEAnt8 zmAN;F=QxI^e}+8wvjvzE6eul@qU1yji_RfC7WUqkdH?#``)$kQ*HGkk)OQiG)CxQ< zXN*!+niuqPdzF76Ec%R37k;lZYL~=OzDqGcLv|*BTJA~-<~c-uIDzm=Ebl(N=A+3( z0l4j7=HV}39Nfuo+95-`|8P=|Ab>6V1WDjqtkaZ03;S%Z4_t3)8u6KgZ1_JQ9DtoK zO_23r_mbOvcs%~RgufD4UE~0yw?{5Kj?4Jh@t+{O!y4GS=h>t(i^-$1799Z%4~Wak z;Ug!vNPYtqkj$EV3ZJ<`M|x8C8fqTPwUlHK`epBJpvz!#Lt*ijL_O6~DZ4YT$05ax;e~6DhGrFeKZbA@+dD)Ji{LXFs zr(XvE`?TM{KJ8KLcyd0U=K9^6%@9zB()Sb&nwW`3uN4XCl()33zPlA`Bs3Kv(z-A{ zU2P+!VcMS-$w4fwzq_-m&>$@lfx`s>23DeF)E$S~v{kp?gjx%9$y-dU9kZl;}*?`Otv?|;WH=;y~#swI9m&dnPN~sZ0vbd=rvAHD7Gp~D2#^R*SqNm;=DKJ_k+{Cvb=W=3Q>xTrxTuPx>{KryG21 z8$rNA$D6D*o&!Ug=yX_wXb%ji*Kgpq-s2_o>_b*MakySF{U($9GjJUAQu^94O#hv3lKQNW9?sXV|fdh;=pTi;}g^iW#SSs_aymtu&ZHree zm5bH!$~9T~cL#mk&~b$3?N9D`o60x7VX08(irz`8w|K!G&}#-;#19B;sux%A@g?BX z4kX8hh2miSQwILl=;mq9J@37rA%6z>hpJZZwp1Ln5aR!ghn|g z`~T+(M}A*ns#Jj#t!7>o3Z8`_VBlxM4RHv!!Zf4X`3EqdNrU`>wXH4LE})S^+=LsC z+>ZfLLS-}u1kX30M?6#00!6Ju#fAvQLLp9cN&&tb--^7W@H`egDtwYL9jQc`Si||k zNj(UqHMjEWxZ*QULYxyU>eR6U)~d@CRYmQnKOqc3DH4F-3or9(uwITEpObfkW<|nk z{|m!B$PQl@%UK#cR;-{l?&b@Pu$h3>Qihl0r*ak zeF+?R;IQ>f%#2%+v`=;A^x^bwUjKf5v-PWJ2etN-9~!fSoNO^RRhyrh_bR#HJ?(PFbAVu=AwdlR(p4 zETI2=aCFgj=-5Rh^CMjCl4qN=J_=Ah1Z}OMCWk(Gvb<6u zZbO7~1M~%hf!`hr0ZyGzt&oR<4Jx>uM&>V5Z$-4VVuL6YST?~+oMu-Yk)s978`it` zR5&dzqr4Y({6}C2I3VEkv+gD&yoe|F6?aF+2@CP*DB3x`CX6*ojS_G(Cg~CGMWruI zLnf&#`Q9UJA^8GHNiX_?-@>fa`<&!V>elxP3G@uwoDIMs&6AgBN7kb{?vE9#L`NZ9 z`95h{a>u9j9AlTN5>slgAx-#;*Ggh7)&+w?CvUw5K`&L=x@eZ#bNCX-+l^-pfdZ%pVL=+IbA98dH| zU{~XMYqrGffTlNkRsDR+0rWqwEJg#O>QkzQYTV&f7Q#?GIsC6)zyDc&P~cNQAFVQ` z$=H8>061w3(C+~8Q|KDVpUC1b_entgJ1az8b}g2+OD{I?8}l|@-m;M4V)%W#RKqVW z99y)0eF4Gob%r>ppy@4FTV&n@;orrKynKWMNGiMsh{u+4Kt;jypMstML~|l6xJZZK zMI`Pv{x!=wZidg!_cvEpA~HU%br*iqAnwZctCQ*KkD~B?GqdfzTw;q7Y&`6!^~T1b z>+cuHv&v$6IU#}%3{O+y-XaLN=s~=$4FC$U*mz&a$loOMGPK455<}`|^>Fjy>w>(6WJMgkm{*X) zpuNb%vfBW%!S`cbC@F{2`ok6-N`uRAs{xP0>7#e4cztbWZOw^aC$|WxrQ9Y`YZ7ng zm%N;M`pVAEZ0(Y}JAZ)}s}yA}W{xkls4XsODmh6D6pp=XTWIAI&~Ps(WC$$#;S5G5 zYFhN{=?=5W1%vL}kyBS|l?@J4>-$I_biR+YU{c*N>5&g}iRZm|FN$b2kH}k|%r0tf z404^421Q6An2S?YH){zZ;xiR>)=>|tMRzVLTv%B*a?~Pj!P7EQGRyw3>R;3ZzcT;* zMRdD=^^^L+Sorj+SZ}?(#M94^I&WGu*4{2!@2ZE^-5L*BVnX@*&3s#^0kCEOf>PDh zBnc8L|H6x&w~bS46rpbd9_=IcJNW3(S^g~C_|OZ`Yy490;1*^30pesd&nV?`sL~yL zgN?eax#)I~nG!4}i>U629+-Z9=cn1)@pwRL2 zMmmZ?IM|xAgH0PObkJfb^pM5lHlfX6dF12Ctsw{qxYSu@vD>4@lsn{e`Pt{nEIPw* zhQ_y>C19MVE0D~NoBpZt&knDH&>Z%5=%%<&xPCv!PlZTRHhX|Wz%vw%KAX#5*cP= z1%5I>msrr;HlknncEN{RfF5~>=fR-sbDJ-G*>A3PGHVrVT&i&vN2tbaUWnD<{ZMMi z#2k20ZutpyE!ZL+>FBX$rQKpPN4G9uW?2yzOe-s#~Z~(3O|8ImQ|95>km~$Bc_NH?O!?*C0nGYZtei0 zrn-`;FHqEdM7s8clbCvmmtRz99W^D#Q%sq`mL0q1D}Kh!`AVsgg!#g+wb;g3Sb16l z0`>M+_U-r45P+#f;Aia}wDT4~gFmBUPyPs&f6sYot=`(xegtoieWjFP27uTRayntx z+wMg2jnMDm$o^O0kwy-s@DJFjw$EYi@db&}#uCiSl>-FJB0x7nRSk70zQ};0sAHcS zi1F;=*y3l!r#sIFh?{5{QB-ntZI6UkP_T6|;YK*L9%qyi6c;}&bk|c<(^BIMhKeKO z<%v9RqrD56obJ3WOEro28FU-`?3dCX0mu0XT*UcTr_I&y4emFR+gH!wQf_1i0tT^* zI8SzB`~LG=#-&J?)ZD6HIA7r3idOga{)HiiRzPaZ&W>(QKVbbp&(v}N1Pg`vHUUVf zk)#rd{jKhcucDi^Mg?Sx>IucYbU!hd;GOA*j5;@@}WaO;j|K_LSK za|lyj#=Xkhs!|vT-P*;(m8(>rQ`-5rZr8&f3naCM2M>ereR!F4lOUlyLl$%B6R!mk z{U9R4lq;0N@`B_BQEBtOi8~ePw)wNz<$7150E%*h&HO4E^Y1~H9vD3eU zKYw|3cP2dN@ugX`+^+67)Im48I^tKt3RX=^t!_rat4Cs`9 zA^J=NoPgA_U?nh)e`&OfpVzFX--n&gBcdAuwo}F+$m8&U!=-BR89II~#nUa5Mi}_0 zg*koOSO;X(lWs6Z^s~ZvPVm^OmrMB%^zwO|EB(sM!+bN6Kww{-ESj-Tyn*(XRG>53 zq62Qo6*DDACrX=d!aY6NGRAb7uu3b^LU_!~uXdBlCegX4>RyrS%pj14xCiwvO5_G# z;t_x8=9a#uhekxo&)kXpA_W3n1V+8$>4QI+uoa9>qcXUAPv@4L2Xi{fS-Y*d76F6w zlK6Z!+q&iSnR#VWzRr|{rvUAj?s0QNsaJI_+g%>Ia&AMkh`u?4$HH+`NxybHYpxYf z-A6pjd%rc*=*yr$8-h4iCK&%fnXuDznKtnpXC-h^%FW*>Wyzq@V{4_F^^0N;%CUH%ywe#aK zJbukV?GT##9D)|@E+MeL+4T@s=Eh}6nO8@_anja@hg{%_iW`T`!yf|#a8o+EXRQ7W3R0xZ9;G6TUqC| zU`)2ZOMfQ)PGvMaB>ZjmT9q{|p}=3s976NE0oe;$$1yNYn0+4l=gjURQKhM(rNt2p zy2$$^IH7xOZip}|Fz?-Inq(GPdSG$vg?tb02&>!DvG=zv>2kxCg>Vo*F7v=AF%>M$ za!J4|G_1m_CQ<&wkD-6f-Zq^kPRD$5Gdx9ai}~uB2-v%i_a|Iuqb5ilFxBe;4alUb zBYY-Tv=4j6CY=m|>w{Xm{<5=sJ4!_&6W8;c>&FyV+`XN7uYvuT8VoMA53yd0wwH(K zxkO&WFz4uYIys}_npT=G`B_G*--7f0k;@3e9HNd@+$u#QO&oh8>a)!;5O$6`_TQ2} z5wXQ{hVvLDE{7`*Hzc*JHN+P@?w!*~H9;(KD3aTEAAK~-#Il6*_Cf$JEk6Fk^8&Yq z_BqW_zbwM@%~|7KFzFL|kiBG%!BwS86gj~usO{s181f-k%K8DlhR?l0BAbrx>)X7O z4SP!1Td<`KpT#3JR}#<}C(oG@>P1iQet4yMPB%<6bZ?62M{neeTHv=g<+h>~dcB%8 z^p%Hk-Uq{_ig&};*rgR$cH3k76qSt}7}c*TZ`cwKGY6$sfu;+xZA71l0(Ud{;UqkZYF@Xz=C1if(f%&EMRx<$9n&U9V#DF}YNZig<1h zSjJMNZ^Ynl)8PA+*$MV=C6`dAtwq15A2M}wZr09dmn+q?Eq%3!Mu6DpZ5GhbF#S5% zjM46t$H1uNFOoSrRrnPi=4lm2aUCL6i46ALPETsP_#q8m6O^ftB=Wmm41NR*EXp>c zg-#>1uVLxZBx~Jpm$83-i?yN>2h(~4>dM?=_eJ({VR4F{<@WHZ*i3|Ctu`R})(rOi zh;ze9QThWZ$_&s;7_x*juz*;WBPWNU^vn=2T~mio{Q?bsaX*MTJ3nLLXr4fk7of8A z(%D`L_{>zQmU!WU8hb=pp1c7?RGY#8uA1(V21Q#^*N%Er_i+|7@cDA_JL==Kcghsm zK~LSn>@-1QyBkU0bgu{V;Q?3c!hB)J%i&5bbxnJ*rwTBO4n`;|X0-nGwk%tmf!WwmE^A{|o z=KU6xhA|9gdYfWeL%IOvAd3Ru%_B((YVr3FJFlvXXN81nI`RwGJG@IlRhW~}QAb?BOs^P8|1Fd1dI(^bYw%j~ITB>`1G(Icw&kCwn2Iry8Q zBv1h`!v$fX(0R!Mb@oCABp~80>U*qOqQke6xHZ#CaZv)mKe>L<<+P@b zza61~{`FYH)R;h4JLsznshZaR;~G6^a?S z0<8*hKe&S-Rx$#4Ix$$Ls|x&6g8XK2aZ2v70jq`Y^J7AT17bh=)&pE;VG(ByQ}sZF zk650n)kt?W!A+|KxKGd6f|M~kXV8JD(Q;zNU8ERgg3uS6V`(@(iYZ1WO80T;9Ltut zxQ(p6?gYL%dRxCkKp$8h;N(GVrTcQlr=cZy)1bLj&FoP-OX3%@TJe=iF@GIN!CE=* z2f5#w*ZbD!(OkPw-d6l~)j69PpIJ4#O~aQ5e3q}rJl>tp3xT>3Yrfkd1#~V!oMxET z9Sy2%D#h~k#xiX5v!Wg(q)LWR&R#<@o#2jH%of)($OZTrBoFZLrQEuuSQh$D*rhDF z3Dq|(=gpQ;V%{G`A7sO$lop7ry(P@0dCkQKc)~6|lOvk}d?-UtFlYQSM%a%EK)V%y+n-?Z1O$*3e7Y_B(CmorR9nB;)H^BE-1 zY}*=;*!#R0YIkAw`$J~*<3c@nZZCv%ouGNpdyIUpUUcpZW`qLYP3TDP<(&Rz8e`Ca zhqL%*6vIaufOtWU(JF~$8h&m2?44iiZ!l546YVu;YFbQURn=ps(UzC0i6V z!rgvab5oG92rH1y-9)3K%`Y$GQvZPQCM6#y?BaPBr`PL4>%?I)Fuv&Lyu%FwrW5Ln9+=U$@g- zeAeXFl)JW<0R9ML9t{*uIQzA?AZS@e+|Rc@=il*CsMKs{a`!t!e#jU{Ov1G~Lw*^B zv-_GGKJ^8{!;-$U<@u*~wgqqNe6zj7^4LRt@G#$3fL=C3NCBTDaJpU2FXi3AE!f#` zKJSZEs;F$RxInQ=U1r(5%SV%Z1;wj;hAjyIW-0K4q0Q}W_$?HU0@LiHy}F|Lfo zx;`mFhSGqJE9aM&_f(3B@_f%&o>DT{ZG~wHgWR-AQ@o7yB0Ox)W}-u8+CX7SsN2SJ zzRARe_tOCCo;jXqXWKim(tdeg4>dK&toXDE`5$}=-j=R6cdQt-RtYGQttzJh0P${j z-;#RTIy*wa_FhopbD3rtuu69ebm{&knZ@Y*M{v;t*$EcB!>Mw0ZP{i|eb6hZlfH)n zd-j06`?%iU2?#}EfXeV+PBtdVZZlH2=kuQ3HIRo-vir>Kg`YiJY#O+%irMpBlp5Y< zT)_`#02(mKf_^dG>vb;N$3Q@%s|tvxUd{+YI9S7-mgVh8Dq_LAAK`N9&MTxz{j# z!mba{%zf|&UjfoI!BUgRDLWZo4HZigDb?gFKbK(r8jk47Rj6uTDL0pVZ{5L*%;{62 z{w6*?Wdof`*vGFf>?}XD^cL9B3Yvwn;Nq?Z{p0}cp#7L+g@7RsmC`;4E00QZ+%D@! zxuN-+;1ey3EsHtv^Rq>bFV4y}525Hdp3#2^uh2UlD@ePA@@#o)mn}LtS3bdh)!Pg~ z@5cWD4%pd|boQ?koL3e^Kc=L=a35C8yi^o`$=6z!x%pMyjPMmV7jVb(9LIeVEhs-` zXxc!KMt<;oI+|cQN-rtj_vuU8PwNr>5zA$J_$7f(9VLnEAp0d5*!^|>x0?_7@dL*0 zIbdnD8A8RUl46ypb_IwP>8kgh?|Q5YvyPoPk-_JXM5D*^6>GO_t-!v&E|1qx&ibp)sbPn;kOQ_HUHO5?#?_G1d&e=%DMB zmTVVcWT;ndnvE`(M493=!O5w!koyDx5oV1gyd<7(7kzES;d`Bgx2^&w)$bn-R&3O} zbq8sJv>2ry4K$AF`Uc7UBkinm9E(^@4hU3l2UL1o(+=d!K?0J5Tx_D2a^SWYWh@fq zn+PYx0P5Fns`@3Q?KiF zYD^}(T_7{E-?TU5kuxIT_D%yPLUKc_#|6>~2mRpd?tv{1PQv2{`>-o3vW5Jp!Zd!G zx^uDR#F!ug8ps@bm3W@qu_`pCT~C!wlkQc5oa2*_oF zB*UFV=1mjwFbitYS)(VW7QcGjh_6bUW=Be~EIxA+!oGYL3yEDM$=VqrZ?`FIObEAhiA;8OLQMAj|bsdFf#C|2t#XpqsIOeI}moB0sJ|n>_#b z-PS6F{S zenk3x2c%_4(f>CKcZmtO^J`{x5WN3N)c^j6Ehz$kv?aMpl1KD^PnXhVCIAS#CiM~B zApUcMe@kEd==}f+5xpKNlP-)(?e;LV@|E3$#bp@t$n-y!>))Baf2Gx*dqbkPcW*wO zEZ7VYF&|hk{m&xAOoKe)3`LMVr#37$D~YYOxnMIgPG=oKxU2u~>AlW>FEe0B9_%ha z&HLGZ6Ld6gdYTeC)VRgt*OWn-W;G2gt>gb^c^rT1JREae&B;;H#kk&bLA)LY{eSQD8{{wI7;TmO zN|wX_D-=o*NFoL3#glYAGCBV5=JftubnV^7w>kq5>l4uM0N+_IcD)OWDdedj z#JQ+OE>r+0YQxZ}(+qt?NizXzfLO%X#}*6We2Gr8_a^73k~TJWX5ra^yF{L_Hq?o4 zJpGR)$`zrDKtK>~$bm_E>M$z?|7k*MXnvPW49~Q-B!~d2bN8$P-K@Yugg_HhNy*3z_g7?^ z9M`cCw_|sPqTNWNiKrK>`!EfFvC;9PYfK~!_of6Dxn1vjOdk2Xqxv2$91hk*GzQn! zVCYPwMjsQc*sIr#;9uYL2wOSG`g(d`Zr&e3^I+Foj2dZ_8k{L%J`BD^6z-395paib zbNhV?4(LEi z3Q&la#w?I)-R<-X(ECA%Y6p=pOU68A#Ai&j^ki`0|Qci zPV+4?HBsI`+yHHj$7whxbP|Ca9`x<$2IOL!UH8iYu10}>DYUuo?$_vcp($wPXGtq# zfV92F{EjM7AS;iMhDbZ*m`uEz%_6hIYGzCk&`Nw>E#l^HUgvK+JSF%IR2)~INy;%4 zA&E}6IoMQ>RVbPccfW4U(PwE^+%#`bzR+>oRJ>dpw?(L$zF>Gebx1>KhrzJ zL_2=ilF%IMfT62rxpR1Ls)|A6TU|o~uy<7WTeN`zV%j_tNr(P|b04&gg=T#;V^5vy zSxY2`<1NOIzSPwDQkC%-2YcdwS{Z`U-{pSsm5<}AuKO_9^Q9Tq1$*}7R9s+_Z+iMW zbC{BRsl^3ofaUC2)-V}K(_D_2AeS-J3=%u$CL{zpoX8lwJ6W9W$e60Tejkj`MY4_l zbCC$bf4if2dgmsNl>q$)iHsL&c)hYcO9nFCE2A7(;<&C9U&u-@rI-gIqU`U=4ji{P*I+SM`iL~ zy5@N;b>jH87R6B1G&Q(?K2?CeDGT5@j?C^w<_XN$RH)UuC@Y=RZx-QCU5-F(Mi-s}DTu34;E&YUyvd1J@3pS`!PN!InXV5P?v zN8(JA5n~s;C`;P|J+u$M4SmM(vky~LyEy(N7j|CR$}<}rkdLo2{|!|y0FT;F;^!MS zhZ-jxO4ax-_Go~$6~ss~LGE&QQ!w1~LAgqZlG~Q$KgG{4^+kc&hlP)J#*>TSIRCeC zxTYS{885jCBu;H&<)!rWh)~E7b1|T;Wv*4!*2>WHvavx<24s(l6}2Yz^pdZaqOkPq z4d)~9+pG|A5i8&;>!+7lkLWf8K7;3=$-@aycMRuqi)pZp|C@uEnEC_gWNiCK_SrT^ z{!cuyR8t03k!7wp4u8?4!a@`pN7#sOHbA?=tZ<~@6EQOfTV7y2KN0KEDjFN$B14QNXQm`Thjs*0eG+5 zn04I0kLaJe&PP@Z(1fxu$<24iu!bXP+NK&-n>%d;h!OvqB7c0-Lmg0PmjfEKAnM;| zRBDy?7oFBSAdU04CkD0(a2=r8tER!#c!y=&(6+bpj$Egn!B(8tgdaX=S$H_)ZMJs; z;0e%JeyFDuXfGgoT+^RSeM%x*j{yo~WZmPS56>cBS8I-Yt19yB>z=Mubb-w`!IHMdI)F4W%X) z`J%rJ&<^&ZrhAGj)Y1ZdZ9D3c!KW;ga-4!%+TM1szlchE5(6Fbs?tayq(GQ z*RyMbbDf?|G5>>FML2=B${m^LWXAi{<#fdJ0()oNmRb`JNSqV(b*ruA;NEh?$`mMg z0F-;_g}42hCm4xF=X-!xYtwVGF^~EZkX6^^nrQc4v8Z8~XJg%fv`v9DSkk7RNpyW~ z9o#d!SE-uPsg5&MtTe99TZ+3tdo0`!Q7^YtiS;Yb?)wz9h4Rwjp-)8tOem1FOQpR6 z10qA4Qd;hwii|}0 zD`;O&-@uX+ylwa6Hb5l~x4`|AqeStQ|BE(uJxzbWw~0{{Y|$mLxbJ&s8Z8K3VLHmM z_utBn2j_zby5rzMI4d`-6k^s`r#YEY?OC?Rjo7Iw^*=nH$uk~nC(xhylHqjqG|n5V zTvA7R|?U(PDDNtmPbNXlQn3HT6=up2mn0*zbMFx;5W#O`@W~aF$vI zd@WDw93}iurjLSJ2~g^%5-W%fpbXI4ozh{0;vx{({Q;`{C@bhWGJp3MF{3|_8S?A4s3`E`Vw$HTxsmV%-6uk_-h%g2IhM&Zsbs?6U&cU6xaKb!Y_(u2NeuX&ndsK?TAx z5K9)-*4`L1rBipXJxm`fI$)K!JO9WVxuBZ!={H(A;b{t^jdMVMv2 zaRYcCxArb~SLZRWu#gfK%s~5WMLP2b-&>*3ogR^!ZETeNY_&IVYic7gor<5R<3>2Z6$iR%FgfjQ9lqfrVLgT^!h z*!>jTzsyuk=YzNzBgy!V6N1<8U4PM559`%&nAf2{sMAP2EkkyQr_kQEo$^W|4lg*j zt2H=4fq1V3wFJ1N?2|vAc>vfA`%xf^moDCyV%xPpEiuaH?fXT}~0) zC)eJj^FLq1!!r)vZ(y6fmY5%%9%}2D57R`!#q}3K1ZS#V%Tz79k1g+CCfof4iOddJ?Yl)#LW)u7&^ETz*<(|sj{Rfxj8Yfr2VlWXq&dJp%rN-NLvCX$t*$bhJTLR7o z$o9-R5rcA5ItrP%rqQdM!T$iR5ufglqkePHm124z-WgC#wR-4Gkn7KKep;L_UQKT( zN}Ji;iZH$?W6cg=de@7y&1 z45}ZD4YYZN1&SNdgS#-g$lvvi;I3>az!!p_GdxOziQp|IZeb%KDSWXUPwanm~C*sw|O9O12Mvi;*bG-xv17*w$UqN;N(FDbT${ z(22CRe;N^kn{1Y#Wc32(%d8~FtW6eMJduCpCIAJpGysA)ZdW%c;X5X&V$&8BA#$1f zx2!fx!Ukii1nu&SYMPpQ4+VM*g`)9D1@fg;#&sBm;)>sDv`Ymjv_6=&!Mz>0IF4CIt0V!wU! zHHon}3#c`=`F$YgeqStP&fZEj5`|@}SiLIz54MUTIQEZ;kRE%pBJ^lM{qPcc&t6T}S@FjkTwT!xe9;6B-QRk1{F5==7WL3=+ zN`|G(xPWBSK1+b6!oQ*6>;&&Nsnq(h@p(IZaKiEVU*;2F0ufrEjFyY^;~}`j;Yxjm zB)r4t-6F*Y3+2pm9Hu_F&JrfpOPf-?f>8bfZ+0cd*YdoI3KTMg%VlOtwhr0Rzb|W@ ztv;rEUxuqv?)+5`>D7mn$7Cro{_*>~{q}e?u@{H8R@g)pQ^FrdHC`Gjo}G zgC-77zaA0F5Lt%0*vq#F_ zETCYedbIvgj0l}Km&(951xMYRK_F|HC|N8l5V4m4+9W#>1r(wYrG7vUDbmvOHY2@9z`HxdD|!@)2(e?UR80U$|gVRRH4&$^p%4s$nzi!WdA6I}PI1U%AXCY)`pjP5P6lkW1HmlRRb0)s{BfjFVf#Z~SB8 z>M0&ovj6LyO0TseYJ_&U0e(+P!CW;@`k3JD=7c!lH z|I0%k$$zI6N0YeNFP(y;sN@hjs`LFvS0SU}+dJZFb+~tY(IeB@m16I2zWPw>ybrSTd_NYQQ^i?|cKhS*?WlB*cSSUaA%N6H=^!JD3rz6LsEeeV} zvF!@3RBUOi-WL|>Ce8))0AeCeqS*9l(CNCcI1<;tC8HGX@koxQxX6lHV*#TIu2+Oa zzP%D5&g;CrCVqa5IqfL+UY;e+mAY#ngU|9!1)@$w#6ZoLxIN7`>OubyQRsZ6BmoCV zs<|fpwyWzqons%_6;+XM;Rz1A-i7Tbi2vHEIVUf&#%qbQK3*6BF|6h)zYwV0jT~3w z90dXECl1Bw$bL7qtC2Lfjq;`M@=g6{f!Et1eL9RZZt>A#-L|?aU4?2spq|4PNOGvA zrYk!u`{VnRfr8E>&##kXBWKQ!^#m!arbrd~nda-`h(IeHeW(0Hg^wO+yHwmjk=+EQ z1}sW=+DhKYK@|$`yD{-kE8vQzjTo#c_qXvlA<16Iqj~y_Yc6E@3M_0AxfBEBHKrI4 zB(DJKFuGcxJ7sPLJ81qmeu1rC!E(pZDr^_X3K$<7$}w)EG@6WBS1~%#3;_fgI~p;f zp6h+^zU5Xt*F<#BX=_4Th&GhRp@wdM^SV2tO=qWErLF6?zH&FzXp;M{-f0mloUjIh zuOeM-bTaikY5aO0)~}oQJNdTj-}S{;DNns=C`_kYV(#6yrexI2jd_PUbUMl-d&dbF zqsY?Q`(L(eZs#@Bv!k_k-o@TV|ATM2xMdx+JB2gw%)cl2%oP&~HrCK#=CXD8hM^3!a=R?wd zKUtswg&Gw#jylSs7v1daeK(&?AD(ytuVOPBou-#l_yJa-yyGFa(9j&S2&)7N-N0av zyN18AJQrK#)2>3T8kD-@R=2Q1Lni#Eu1{C(n6sPxB>tij=ZF_E5XuUAF<0p0IJ<&A zOd%Ng=VTO%9nX;v9eptvO8^Nyq^rrCa;P0)hG1mWJrj2 zn{Uzj~7)&t7GCXni_H=Y|Pi)hjYYf{H=MEL}Z!u>Aed9`)2<)^@u zjpoT!96C!H*_VT|c87JWMXj3sT{Bd2EBI79)mEpO+x2&_d5%#svs;bIb^1|O{8h0` zgR#v|X54sRz!7;{;_=GJriuqtjL+~+w9Mz(WHp(v-qL9S9G5)Z-IDPFF{!R%pCSKz z5mWe>q&JZ4y_h`=l;#8C^0!r-5}35X)B3^Y{w5rvR6R~y3&-vlulD(E znM)b<89zV$i1ruW9w`((lGpR~W>^`Agj9Bc7iU);jpu3lUhbS?Lj{7cl6~ zq1QvnWGs`}E4dY9u=?ZBS4NI4BJfU+O?T7Hd%Vh6;Nop`nan8Aa;kPJW$-Y{e8rRV zvKLCyjNPM@Koqu9_etVT)9EV0oEZZpG;PN*ZD5?=rf@8-2<~fhP?Kg*S4wJVk(*pZ9)gL zV#Psjq=+hlIcJg>OWUO$J6zhw9#18NqAc*h4374##aa)aJ#-r++EFZAg~Z-|t( ziLUtH2kHUVPvkSgJXr|LU;jmj0I85C7Sw2OEa!soiqV+#?}-=tY(Yq}QBpYq-1Hqv zJB(7SV;{n=_kX->cwd?mt~{}C|9zjoe?W{-3r6lU0(NML-(P3=pToTD1%dJF8ZRS~ zlzLkVr$F%2gvqep^rC-*{f_nLI){-iBPtc>z>V$!^ak(emtF+SAmfne`10Y&3cGWa z53!BoO^!6O!SOREFpsKNT_vqCm6waJJ=$s*%a&X5K~vwA-lhV{kYOWV{PFs+ApRL*bAi3Q+(^kD`$PBqrKase7sa4@1td=d0ko;}W6zLUtCzv& zhWOT5=T(#*nUuS#DWRLQ>|W^}A79Zg2RmC-s+ErSGAV419DNtqL0V78c!SyvkP${m zjx-VYoTtQ7T&l)b;lc^EI|Oq-4M?^RHOSsfu2k=+JD9S}4_jNdd1;e_!!6q%9vM4p z8(A3N=Xl94rqikR{57|~X7~gTW;AHn;`2it3o5Ob);sa&+3&3S`g8H4^cVYM zbEgbi!^&z)4Q5mtOlCMVm%@`0EYJ0C`;B-9xEk%bX1b+tCk)OSc+*0>69YH;OjsuM!$y0>3)EGomZDO z8&U(>lcN_6qnkPml}(`3H~xDMzC?48hJ4TWoov*`fD!6!M>tJuE|Nyte6ugEn@4EM zcqpa%ht|$;;;U4Sn!5Z{?&BaYUoVmC;kPGkyyWZqwcx%F5M#(IPQD=_@Yc&`Xy=1= zY>xXMK)k2IEe&>b6(jX>YYykrZ`tgRAkyw`^(Nx1@`fQ{*GX)SQno#Ijvf=Zy@|NE zX(hGCgm8i*^aSq=PS2F(tW0Yq2sX;i7()|{=NNKFbM&|shh_JZ*|rG!tGi9*Ylk$Z zs5Uk}5hPvM#pLP5B)ITgT?!v*&5hL>aCQZvo@JwJdDo0LUmqLH2FkO!9psn^LeskkWKM zo-cuRRWEOgtx`rjTB=_Ca?DRQ87n>8HWMPx<9kbERNwO}zTpG) zNFFOZ_g1$m_m!6jZ=ZN4OH|ZnmEa<`57t{lZ{@iR#9LyT2pPz{r)T;(dsn-r48~ae3r2Z7>Vg+b3SBnQ$6&53V|lJKA2UTv3r9jUFegYmM^7=2sa}fKfW^8 zylnf4`CVWPiZ#8y!Qt8v8UdmlZ(QY%f$8B)tj>twa8*xZITGHr?bK-s3FhIK%{PzT) zM)LL9w4+u`94K@Vjy{~r9=^SO0@GZbJD+a}f4MmoPS7-j6mtKP!sT_~dyfptdS2B9 z3c(qFL+7{Nzff0eb;?eO?|O&2DHD7$n``if83a1I(Sj7jbh$d@H9T{1cis51LmG<537ad4)k#7?P=Z2K zZyXCJvcclC&QyY`xX{F0Ol*^uGFyH9VsUXjp$d>`o%@b&#xks>8@hx2VHNWXma~is z6tjcHk&k-Sy=w-YJzsBZOVid^O)05$qsL5Yi?}GYyIjs+x}04U?)A0{ew9W)Twv;L zzCG)k#arzOdy~L+Nx+2`mT7>)d_{XOTMMmBYOmF%_=x%bpUKjrV*i)e|p!#4~h-?I%ql5Nxp9gb5VSmd6wDWxmo;%0VTU*B*weO{qhvb~ur9Zi+^_9A?8q-pT z$Pp=cap#)-T12eXYlCdkY>*kXOXCwz70sVaJU{e!}t zA}Vvy>qX7uR_Aj(>QcB!AK$W&!B~=Hm}VA62_rg_Cw((;Q-4v?g4fD5lZIscF3pX} z_Tmp^NO$cVTYd}|asvJCU|h?zQL=`>D{J=%&)wD*Dy$+DPTR5Tp!o?Or2~V(ITm%9dKqi*B3uESYe06(J|d z6VB#5Hs3@_LUx6`k1G5ik?|qmnX{v^(PT$q$8Ut3(EI0?z%{g2iuHpGX1wJ`A)*nU z^sU_vn-XcrMPW)W6*?o7yT3J1o*ZRx5xb~#*Kb)~r;C3}_eL%g{1hQX{Gh@r?`@M# zq%r>aIy5$3^j{?uhyW-dkpOi!`1{#$J;ZFdu((%gdFkE|#|!mgg7K~)alxN2LGw({ z2=v_-+OPhDcUiaBHKhpcNO%Vn`|T_5SPch`GJs| zvg42L(Mdz*axWEtbiWM8;um`*ye1FaDH?h8;aj>FTD3(*Lj%d2@vXtm-Cb%>eL$~5 zDJ>9vTdkB%>{qyNda!FmTmgBlOFPa7z9|VGbA#{h1*RxRzDf`M~@la%Q6KRNnq9oL$dAT4U0y2KyfzwnLLt0o-@_cV#n7ma9@voez7 z@5NKH4RsD%d&m})+bU)2bp!Eu1txr=Ra=V)YfDgU9VyAUJ-%P8rx~KJD}JLNm{)HY zPK=|nG4jH!N zdKP@lSx`|UQVekD=~8a3Ec%=q?#N^~{B1cQ4#eSs|M(g?mf4$Cnx{CW+F%%8y;5{q zs)%m~e23t%=+g#?{uJuD|<=STvc= zNpxZQaA_wgsnpVXO!7D!Ob-m-`%`^-Ul8u7WYKiADY1Cj2=7>uovHIAvOVjdq0M#X z8Q2={#CL7Y<+W6oH}&rO=7?vydxJ+W`gbo`fs#VQX|x9F5=Y*FGWhwA=kmz`7g_bv z)y0@Vhx9`er@r2>{Mo%msN~@i`pa#xSXeNUO?w@DvLSrTJx;w9kg=4XqmFVh4scZ4$TzCWWdSxzHS&pTt8uvCUA;=_v~ zouVWWd1_pMPSBI;_5yi(Wm&|hij@40H^_H+4GU1aG+1pc{b*^A7uMwzFRa>4OKK|I zD2LvE*|Q<(&9WP~+9lmCT5vw1zjU%(K`T@s?PYC+&JYKOl*F{Rh4O)Eo7LlQ zs;uD03dnxQwmpR(wcf3@x8CoLZ2=M`mA*n1ZLmMHQ&jkUn~xQz6I2Yz!I)8%65mgu zpiwFCA$iNuJxJJjl=8crsnMKck0I+IfPGY+#1scIxJ=nM1T5he{6lRU;gqnBWDSr zi0uHFMJ9#YS72Q|#s^Xd`z0n9I(f7e=EDrtYirdRn*BHe${Y1BzUF7v!M6EG9y;#| z0>;kQjdC!yPKJTM(olGxs9QO^%ma$?s|MT*@S46i>rbf+__FKsnqvd3)sCl?QKv2!gT zT4;YQJFss;Iy+S)IN*9F7$+P2xYE&SzH#?h+~BrTHF2h*1Nz5l6I=<9wF$q{t+Vt$@M(bWk0uig+n>^z zh2PWk^Ar4cP5xLM7bwUA9}g%wUFTSOgKobNNr%BCZfL7{&(o%46Zb!d@y}m*zZcAr zHtj<3w-rnmwBAoGH+RW8Htm6IkpDdlpFpwCczuCvRJ61Cg3!n$Hfdb6YDwMyINX1| z`N94YE^_!*sAzJ**}8LTido?@G4(G$|L>g>$n{H)rZvJ<=GU4}b(JkOk2P8#f6)BS z(Dm@22mjA8>Jjn1WcW}+?6T}&ra2(AUq*P1zGE1LQ{=x^ zEGijTls<0nmu~6Qnkok?-h_Pkw7)+zVaqQVf;Jj81Ejk`-e9-k2DuB{{2wCh*H50X zpsnKd)qhm^`gp~eC-eAbd3o7%tYuD3tYF_@4*i093GGFNMI4L>d?FMZ|A6`wXc=`m)T&Bi2Ek_4h5;BK*s(7LMu9 z{0p}Ddm0q?A!9Lh7yOyk<$=Ak_3Me1$~@hEuO=EvC+0NID)WG;=znJR&o@6T zNca#psRSNxD$CVh72fkR{Lcz3@Bs)CRZ_R5diCm>?^ibR1sA&i^GoEi=?O`#EiF3o z@Fj8oy_i))Jz#Uy{9%fE3@R>99Q4jqZ^ z?O&ki^g7Z{Y@J1`48>8YHEL|FeWrdpa*;ohe;v@2#@-NYWcDf}#2pRwxBs)xTp^hB zJkRZafPQ}fz8pf}Y6U5DPVwK@(&Z8S~rxzdPB6mHX`R zfBX=NIc>>N6*bhW-|rX6H%SWY$;njr@RZ-@u0!ZEgkgd$J>%~e326g^$Ua)M-~4?L zC}hAVoqi#Szwai({a@$WaVL$0{P#Jh`=u|)v<9Vqp9T3}ySe~qX3qZm0Q`c0Pw3YE z6DIphH#Pl|uS<htbClNU#q8Ku8L-+>u zt@G=eYN5AuZ$x@mhD-bPL^;`I@?0>@RlG z4RIdda;`s#Q;i5syW*%35cKQ3rvuMFGm=6LoGl@G2mmbmm9%hj=-m1$^kI5P6F~DU zKeOrh5%>sxjmR)QYy!yKYB$RJx{AjAx(+9$>0xj=D;je(zdZ2}Pl4GF{vC4Mb; zZeFr4pc-8iA*+mr0{r2z^)8cY+nw4M?pZjzrl5{7TnHM^vG^voOYLsYVhPV}VzC*f zbjlu_G4VX+appd4`lg(i!LTdHFvQV4h`x z_6~XqxBGCY^!@41?5T>1{iL1qp*475jYL$w*vXTTXNoTDVFky18R;#j!AWGg^b)B= z9J4Ej5_cuc;O;EXl5^m5%HM?&SnY$N>j@k+0A>;3v8a`2n5*sI5*)*Yaj`ivDF zS8VZBqmHl0==a42M|hy5H0IE@@p=B^47)u7(N^OxWd6wr2jW^C%;va_0N~D3I!n9s z2GT3F1nDg1Up()x7bZ&3!8!@tCLr_4A`7-}_XW|3E63iuCOt*hlcT!c)xoQK6ZzX3 z%o`j$sn3U;PV*D7R=TJTZr6J@KP~Qmx&c_&)on$Bn!k=iSqavqi%b zyh7jdjw#KZROcnIdiylSJ07`tou?*B_Fmmxy#2_=-m}*TH#&V@;yJ}*k?!z8Wz@^O zm1Bm3##6>2g@}Hf(=)uGG@xHChJr_S=dXwAoSRQUpf%He1d&KIVRW&hA<0*o|wc6`?r}Ip_ zpO^X8Nky-p-O+-C-3w1CnrM2E2P_G~&MtKkAFDggNM<5)CWu4TynBRValgZo23QNl zyMC3rz0n;M4KZ`Cd>*$le6r3hq+OaBu-l$~RSX9?1MU7gEs_u8m*j_&V)!Zs7)RT% z7_>SF%Ec=Dtf--17z-`uOGxvc8D9ZPSD;c~xWcd>O{rLZ;i+Sr4bBD?>H26PL?oTr zr@WjN&+{(GmEw^1MAW@*B$=sqy<1W)Tf|W*rtUfI2^ou8hw^fV$w;-N0yg;`8NUm# z9Jjii>RdA?;kJi#1B++Mh+WSNVah;5&QD*+w+BLwcchcn_G78UHrF!8UUrNKeqD>t zY+FxtKFjuOuFE=PC!|eoBN7hYblE+J*?8**h55m8ac?5m7r?Qol)LBbcSh?RsQ}{p zThF^I-~1g?#b>=Sx~Ds^t<;?b9M1yWo2cyva8l-TYXWlZddw4 z*zDFn-ULSBFmB6EmMXrHPG<1|%=a6fx>giUCm>8ZA(r+5SA)eSz&P4wq0Wl9C7eZ_ ziwiBupu-PVx;3%N@Y4r8{j^77F{y-`8GFy^FKHe2BS%ZyTOJ#ELpR@@2@Hqgpq82) z`G0<+u)Q8`-I=dC@~6>mCP}<#lqpRbW#K84!=)0oE!4~@@EwS-a5Ef!kjtrvNcpkk}44Jrjnc$r~^?4{R;~03sCX!oIWw#bnCWXsLttS|B zAh_f1@=X`l4bFOZ7&L$^HV|JoG!GgSMw805n4+5MV1h0|6qSFPY7!eRhOp0cZsln~PhX|7c-@T&AzZQ6v01CenRxX>&~jGn)qH z%X-KceB_@X2mAgA2umJ3FU0g*s^*M-C;p}Jd^_3X-8&IquZOvZ^vNt&MTet{SS~^9 zKoED+t#?*C&;jLqx?(N=y+RJ%>u0tum~F~$ENDJ1b5%wK+SO#{Q$>0_NSg!cwD0|{ z8p3+ZlFvDA6%#h5nt9l~PLa@>8i{M&EfF+aA#|med=u!kMl(#^7Tr#8R)SZZRs5Qd zO^!tou(6pa(_XrcbKmps&UP=Hcfg$Do*tsUJ^n~rns9p>iZPd1gx7Qo`@h)oJ z%5jiMs%gqJX%ZsUPgy1?vZLizcQ0rpe5M_eCmbW$a;@4t-5BH2LC2u+t%o9&a-(5p z*}G76X)pSh7wA5dTHcRC&P3BBcr=PfZO?V{3$Pwpj^5)Z_U;#-`2`0Xgk#pOd;7-n z9{K_O({v@=OR9m{CZGMxw9|Zjz-*Sq%`}r$OFsjc7Z+5|@N?9>aEa}p{Y8XSWjUHKs z9byb9 z_R>uotp(SSPi8K2jcC&KKvzVs+Ic6Z63w>EMrfvyZ-o=mZ)a?lnS_l8Q>@A^)hoV) z4>TP!(>yo(Kk(Shk4s!>)*7c$WMAiEep4LMxGWiXAsVJ?Z82S1-&=p5m96XZh2UnX zX={sIAlppvzEJOumr>fQmwn1a6g+Ao#NDJ5&nGg(0_Gx;*9EHmM})7&Hk zMp8AG&!IZHt!`6`^&IhcEoP&qss&n0DPOz9&iBuW=k;P%?h@~=dGmOus|!cIE-T)J z?mydbq)8{&B^t|j<_e@%p24%Y=k`2EO#wkyXx;xvaHvp@6~l*0R~mU?xox&zcJHb* zx>yWHt?9caJQFn02q>+1=fxEwNz^e3HU74J3pO_<7Zd8HvOc&%g^P>5tH>}p} zz3uPl8W=Kzkgd`O;wY?oH19e;iBK7v>G)c7u4n8O1%=YgD^tCbG@jVyc0HRkt#w!| zdRZ-yw%9nhS7|VE#Eh-negj?HV(>;Bo!lpVGJj2ZFxGpb9VA3YuQ7W3{eIrGYdlxd z;0(lV$!Ngs&{tPRZ!uj$rZeQE%$UrnYHeSd)CSnZM5@F1S1c-}8deGQW=kEQcp6WR zY^ZuswiJ9e2N*}8WNi(m1iVQP*XfeEo075oSu%~seSL)zD}>#d-W&opmp=SzHd8d@ zn``@%A6|hJczTGpXIspoV0IQfFfy-DPbjWb%SMEU2gAsb>RV1W5kclnbMcIk`Qx@n z%6X>+sui9wwQVdaNJJufO7$#1SDi_&nXwcd9-IunoNf$h?}aN}{iiSfI9#SL^n1gp zjSn}*GQ7AksPBBtHZq>?#vh&x7^3(e76GBy1=JIUj~4=CYo~$Dtl&1{@ft^Er4$H$ zwoW$+uy|a}SWiA1z%L4+vzP9<7u?H8WK*X!A9BqPyBM#xZblB7sh|xC_53BC;QM0roA!=~u`s6u>Q) zH?3j^@IxO~*^EwXt0*#QJcqq`^bv$&LV>u$EefCJAua`X^Tdy@P0sc$nUD$Qr+N3_ z+mh_}b;Vu#A4}u~6-)H}BL?lxy%#FXCddFQYBJ%pkgn0|QREy=d(*dr_#;PouQ~zK zp^q=LQZa#g_hf~5%*YUY864Riag*tq{syf5%y4RfYI$AZG@i2qJ#)>^>3HmkDx;r% zla>{GK94WG-Tsnm-~5_V`XGTlPxn-NoXly&p5+l}_3 z{R`A8v*-&ysAPTfK$*Q8A6mvq}kmtks>);IbPMu3}ej`0)TS{{fCz6d~5O!~0+t`fP*6q?q^n#||dVFj;s%cU+8- z4n0Sp#nF3MPN|R=AO}COr{}r{p(N9#`SzdNCBxbUeF$1BLCzC+0&Hz!6cV_wSixUN ztPiUUA2Y(MyK82PXon?rspm`exEblG;aCdo@eBKA7)3ll2v~&c-HOMM)%NQMKz2@- zrkN?0a45aWppDBG38Sa=D592wW?X8z?&7wiQxKjW+SAC7+^^h+KT05W{S@;}0J4q% z`VyI8fgEu^Q~MqXhZYRMtW>Jdk6&7s%=Cm1PJ2s6kHAZ5$@W@%CeQm$zl`=)GLA-c zW5n`HZLCcS8?-{{{ukG3>2DnHiEMV!v6kV(OsOh8qnnjyuN?;yNjS;E2}mHi%#;eW z5YfpVji!^KVp(hK3WX+00GL7hin!@Y-k(U@{f?Muaa*FsFB4W@Aw)21=na$Y=f!N6 zq2{=WTdwW$C0_5N7B0?_p7VG%mGO)(v9B?QnbbN`(|A2jU}PWDF8OOuO-9mcyxA-a`EfVhWYEGgfCcpSP)5NqM2(s?FQG)5G~ zLThBX4IfK+XfY{OqH1H{M{mwW@{p{~lhe2(Icg8R#``*-u#j+tY-ciL8m9Y_Ho08S zxLA2&+Mi8^mj|{>gyP<9vt_u^U|=Kak2FG8UKQ&%IQ3rnSh-|7k6k?01XAM(^~zD0 zFlaqAGE{lCp>fJAe^x74??7DWja;NyPb~3_CtwsX51(YQ+CcJtSzQHFjMriaRP$NM z?~5Od)7_}Mg)dV+C+jV*l2H(}^`r}WAo6)H0KjAziKa*~kYa@ECjY@5ZrzF0s|z(*ls12UUsFF!$~w~y8VM5EZs3S>#;-G`C2Qhn)mmcMw-m_{(h=r z?Lg-Z`wZyyTPA}Zgr1U7?V6T-Tj95DrgGKIKltCKFE;HNG0~U|r^Y;$!$gwCuu0C5 z4qHzLE5-rVH&Ed}JW(K~m)?8T-Z3lNQ;DAs3W_L;!bNW}QT9uz-AJQ2XH|wR;Qt8Lt7X(vi_v{VL zY!vfl{JYlQOCDRvju|&K_0Sy_&1~c`V5?Uf(*s%H>KlaMW*uG}^jW;A;v|>LBX{|f zk3(sf^^J92Eco^8qw@UHKuvHoV(|yHS(;Nygt-1#eCQGPjlr<{2D=sLwh-(o!F38p z2j7QvtsOQE4C$|;-@ROJ`mlj2$owUmhXfFL2Pa)X3+jrGy-AijH<^(;f6}& z+;~{`$t_qvzx8Q>M=0oBl@-;I`_Z)sw(H$i($dM_p(w*ZKKP+4iwxQQ!tsB+A)_

K;pDXTJyZH!17jLj&jEm%P2jrY?-?I*rqktnxm0LE*iU0=ZI;2*l1{GfDw zuesi~Ek9uX#$2O_ztvrXsJtZs8Gm-AWJ764At>5t)cXgax}sC}&-II&a2HRdHOIvp zSYuwXOO-~qOMtLBuT{tJ-j9~Uf$2~KO!t-wJh>;iky&jUwQfwKkn54RHSR&3#LSP6 z>asucjNy5Q?C`;y7g-wQZv2ccI?<0_^}0qe-nt4=fc{%NHI9;Gu7&BVuNg;B+HE*W zElIB|c4_e#Fn>&jlqPu@AK2l-ukj(Gl1dTFAlbGtcDqLMhG=H7v|?)r%y8G#r>3-Ij%JwTU=b3hW0(% z`k4^%0Z`Yig&=?MR9P6ZQK{a5N+}|C>UBSQFVt<++UQzmUJIO(idYg81s<#QyQIgl zzG!M}O4ULnfSWV)6?bTvPsXxVaK=eq}~1AzoHMJ*&pF^GG6(2TKGd8E5241rP& zOy2git$7{_X5oMJ`oh;MXI!q_+Uk6b|GH_h$&pH<*4)g`5=;`tC<5PI@<4LBS-i13 z*+^v`EUFAvzs>UGWK?xM2eXiphOyj_2@(|aXfa>!OV^3VY@05W)$V;Q+nZ!AXo`)x zwz2`pKlVZ`x%=~-2%wQ^2iEt|%ccllOE`S2jTqkWAJ629Mpf=GJTzUvc!2Kp;`3QW zB=6ikRkL@0Fr!^VwwiCpx%1*BVUCz4nX*`Ossz1IzkZd>()8&lhwQ1&5;=M@v(@ow zyNG~2p~Y><)vs&)rOH2|T7)pkZPt!uTT`drZv6oF8er#WJEPv5IgV@Zj!$^koA%s z+W8uC9lz)544W;Y+Jbp#>JV4yJdc0IHuW2`k$Kr(L$}~6M~(LBfw@#5^~<015e$va z&uMdL9rVkmTuiR`=g^L2+qrfd-SB6E0&=&VX!+r>Wx*nB)uxFbplS3d2-uDXhGUPY zcge|erdxKeFpI)G#xlsh(Qj^qW=HO*#a~??wYnRZW*M|_cz<#XGdp7My%|jdw|c1` zr2=r%Ec3pK`V?hx-IPYMSvMz0d3B1~WBq3wXu#;LrLBGTJ^OwfzPpnZt4tw-@!+aj z7<-%jPIyA@3Tt>_n&dYalTZhIRm1fD4_W;Pi_lY$;5rJk?k*m>m5KZjob&4+BaP1f5JcYbW9+q#ML#S$8Zs@Na+KQmBUsb|<1-2qIc$}olcOB>pBw=g#+R2pTuOt% zRLXZaAy|WUlw9UpRXbF{{qshDq(iu*LhcIfIF1`@{u|puL3laEVZG8sF-exD$x;fR zMUOBfda3S<@=3?%&>#(4I0gj zKvk#3RO2~fFIVzqQ>S|W$%puNA;g3To77T!LcnCfoi1{>BkF`}?jKgs#Z{JAZz`KmbinPoM0+nI6Ax2^ZM{{lC8}AiVlqJ5D6q?Aasw Q0{AB`EF)AV@bSz412LyZhX4Qo literal 0 HcmV?d00001 diff --git a/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml new file mode 100644 index 00000000..97db2e50 --- /dev/null +++ b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml @@ -0,0 +1,69 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 +Description: FIS for Spot Instances +Parameters: + InstancesToInterrupt: + Description: Number of instances to interrupt + Default: 1 + Type: Number + + DurationBeforeInterruption: + Description: Number of minutes before the interruption + Default: 2 + Type: Number + +Resources: + FISSpotRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: [fis.amazonaws.com] + Action: ["sts:AssumeRole"] + Path: / + Policies: + - PolicyName: root + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: "ec2:DescribeInstances" + Resource: "*" + - Effect: Allow + Action: "ec2:SendSpotInstanceInterruptions" + Resource: "arn:aws:ec2:*:*:instance/*" + + FISExperimentTemplate: + Type: AWS::FIS::ExperimentTemplate + Properties: + Description: "Interrupt multiple random instances" + Targets: + SpotIntances: + ResourceTags: + Name: "Jenkins Master (Spot)" + Filters: + - Path: State.Name + Values: + - running + ResourceType: aws:ec2:spot-instance + SelectionMode: !Join ["", ["COUNT(", !Ref InstancesToInterrupt, ")"]] + Actions: + interrupt: + ActionId: "aws:ec2:send-spot-instance-interruptions" + Description: "Interrupt multiple Spot instances" + Parameters: + durationBeforeInterruption: + !Join ["", ["PT", !Ref DurationBeforeInterruption, "M"]] + Targets: + SpotInstances: SpotIntances + StopConditions: + - Source: none + RoleArn: !GetAtt FISSpotRole.Arn + Tags: + Name: "fis-spot-interruption" + +Outputs: + FISExperimentID: + Value: !GetAtt FISExperimentTemplate.Id From 2517283d01cc6c162af979e97dbfc64a1e18dc0e Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 10 Aug 2022 17:58:44 +0200 Subject: [PATCH 18/29] Add FIS to the CI/CD Workshop --- .../jenkins-asg/spot_interruption_fis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md index 9d6e97cb..d0db6bb2 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md @@ -49,6 +49,6 @@ You should see a list of instances with the date and time when they were launche #### Verify that Jenkins is still working -Login to the Jenkins server again and start running some jobs as you did previously. +Login to the Jenkins server again and start running some jobs as [you did previously](/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.html#verify-that-the-new-spot-instance-is-running-the-jenkins-server). You should see that the EC2 Fleet plugin is still working and is launching Spot instances as build agents. \ No newline at end of file From 0926d4629954a1851c631b5e220034b445aac87c Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 16 Aug 2022 22:34:41 +0200 Subject: [PATCH 19/29] Upgraded the CNF for the ECS lab in the CI/CD Workshop --- .../amazon-ec2-spot-cicd-workshop-ecs.yaml | 771 ++---------------- .../container/Dockerfile | 10 +- 2 files changed, 91 insertions(+), 690 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml index 63e71acc..28fe1382 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml @@ -21,156 +21,14 @@ Parameters: MinLength: 8 NoEcho: true - AmazonLinux2LatestAmiId: - Type: AWS::SSM::Parameter::Value - Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 - ECSAMI: Description: AMI ID Type: AWS::SSM::Parameter::Value Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id Resources: - IAMRoleAMILookupLambdaExecution: # IAM Role that allows the AMILookupLambdaFunction to look up the latest Amazon Linux AMI and write logs to CloudWatch Logs - Type: AWS::IAM::Role - # DependsOn: None - # DependedOn: AMILookupLambdaFunction - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: AMILookupExecution - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: arn:aws:logs:*:*:* - - Effect: Allow - Action: ec2:DescribeImages - Resource: "*" - - IAMRoleTestEnvironmentLambdaExecution: - Type: AWS::IAM::Role - DependsOn: - - DynamoDBTestEnvironmentTable - - IAMRoleTestEnvironmentCloudFormation - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: TestEnvironmentExecution - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: arn:aws:logs:*:*:* - - Effect: Allow - Action: - - dynamodb:getItem - - dynamodb:putItem - Resource: !GetAtt DynamoDBTestEnvironmentTable.Arn - - Effect: Allow - Action: - - cloudformation:createStack - - cloudformation:deleteStack - Resource: "*" - - Effect: Allow - Action: - - iam:PassRole - Resource: !GetAtt IAMRoleTestEnvironmentCloudFormation.Arn - - IAMRoleTestEnvironmentCloudFormation: - Type: AWS::IAM::Role - DependsOn: AMILookupLambdaFunction - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - Effect: Allow - Principal: - Service: - - cloudformation.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - PolicyName: TestEnvironmentDeployment - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - ec2:* - - elasticloadbalancing:CreateListener - - elasticloadbalancing:CreateLoadBalancer - - elasticloadbalancing:CreateRule - - elasticloadbalancing:CreateTargetGroup - - elasticloadbalancing:DeleteTargetGroup - - elasticloadbalancing:DeleteListener - - elasticloadbalancing:DeleteLoadBalancer - - elasticloadbalancing:DeleteRule - - elasticloadbalancing:DescribeListeners - - elasticloadbalancing:DescribeLoadBalancers - - elasticloadbalancing:DescribeRules - - elasticloadbalancing:DescribeTargetGroups - - elasticloadbalancingv2:CreateListener - - elasticloadbalancingv2:CreateLoadBalancer - - elasticloadbalancingv2:CreateRule - - elasticloadbalancingv2:CreateTargetGroup - - elasticloadbalancingv2:DeleteListener - - elasticloadbalancingv2:DeleteLoadBalancer - - elasticloadbalancingv2:DeleteTargetGroup - - elasticloadbalancingv2:DeleteRule - - elasticloadbalancingv2:DescribeListeners - - elasticloadbalancingv2:DescribeLoadBalancers - - elasticloadbalancingv2:DescribeRules - - elasticloadbalancingv2:DescribeTargetGroups - - iam:AddRoleToInstanceProfile - - iam:CreateRole - - iam:CreateInstanceProfile - - iam:DeleteInstanceProfile - - iam:DeleteRole - - iam:DeleteRolePolicy - - iam:ListInstanceProfiles - - iam:ListRoles - - iam:PassRole - - iam:PutRolePolicy - - iam:RemoveRoleFromInstanceProfile - Resource: "*" - - Effect: Allow - Action: - - lambda:InvokeFunction - Resource: !GetAtt AMILookupLambdaFunction.Arn - - IAMRoleJenkins: + IAMRoleECS: Type: AWS::IAM::Role - DependsOn: - - DeploymentArtifactsS3Bucket - - TestEnvironmentLambdaFunction Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -182,18 +40,13 @@ Resources: - ecs-tasks.amazonaws.com Action: - sts:AssumeRole - Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role Policies: - PolicyName: JenkinsPolicy PolicyDocument: Version: 2012-10-17 Statement: - - Effect: Allow - Action: s3:* - Resource: !Sub "arn:aws:s3:::${DeploymentArtifactsS3Bucket}/*" - - Effect: Allow - Action: lambda:invokeFunction - Resource: !GetAtt TestEnvironmentLambdaFunction.Arn - Effect: Allow Action: - ec2:DescribeSpotFleetInstances @@ -216,100 +69,11 @@ Resources: - iam:ListRoles - iam:PassRole Resource: "*" - - InstanceProfileJenkins: - Type: AWS::IAM::InstanceProfile - DependsOn: IAMRoleJenkins - Properties: - Path: "/" - Roles: - - !Ref IAMRoleJenkins - - IAMUserJenkins: - Type: AWS::IAM::User - Properties: - Policies: - - PolicyName: SpotAgentPolicy - # Policy is documented at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#spot-fleet-iam-users - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - ec2:* - - iam:ListInstanceProfiles - - iam:ListRoles - - iam:PassRole - Resource: "*" - - PolicyName: ECSAgentPolicy - # Policy is documented at https://wiki.jenkins.io/display/JENKINS/Amazon+EC2+Container+Service+Plugin - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - ecs:DescribeContainerInstances - - ecs:DescribeTaskDefinition - - ecs:ListClusters - - ecs:ListTaskDefinitions - - ecs:RegisterTaskDefinition - Resource: "*" - - Effect: Allow - Action: - - ecs:ListContainerInstances - - ecs:StopTask - Resource: !GetAtt ECSCluster.Arn - - Effect: Allow - Action: - - ecs:RunTask - Resource: !Join - - "" - - - "arn:aws:ecs:" - - !Ref AWS::Region - - ":" - - !Ref AWS::AccountId - - ":task-definition/SpotCICDWorkshopECSAgents-ECSBuildAgent:*" - - Effect: Allow - Action: - - ecs:DescribeTasks - - ecs:StopTask - Resource: !Join - - "" - - - "arn:aws:ecs:" - - !Ref AWS::Region - - ":" - - !Ref AWS::AccountId - - ":task/*" - UserName: SpotCICDWorkshopJenkins - - IAMRoleECS: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - Action: - - sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role - Policies: - - PolicyName: EC2DescribeInstances - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: ec2:DescribeInstances - Resource: "*" Path: / InstanceProfileECS: Type: AWS::IAM::InstanceProfile DependsOn: IAMRoleECS - # DependedOn: AutoScalingECSLaunchConfiguration Properties: Path: "/" Roles: @@ -317,8 +81,6 @@ Resources: IAMRoleECSServiceRole: Type: AWS::IAM::Role - # DependsOn: - # DependedOn: ECSServiceJenkinsMaster Properties: AssumeRolePolicyDocument: Version: 2012-10-17 @@ -339,15 +101,8 @@ Resources: AWSServiceName: "ecs.amazonaws.com" Description: "Role to enable Amazon ECS to manage your cluster." - DeploymentArtifactsS3Bucket: # This is an S3 bucket where the deployment artifacts will be uploaded to - # DependsOn: None - # DependedOn: IAMRoleJenkins - Type: AWS::S3::Bucket - VPC: # This is the VPC that the CI/CD environment will be running in Type: AWS::EC2::VPC - # DependsOn: None - # DependedOn: InternetGatewayAttachment, JenkinsMasterALBTargetGroupEC2, JenkinsMasterALBTargetGroupECS, RouteTable, SecurityGroupJenkins, SecurityGroupJenkinsALB, SubnetPublicA, SubnetPublicA, SubnetPublicC, Properties: CidrBlock: 192.168.0.0/21 EnableDnsHostnames: "true" @@ -360,7 +115,6 @@ Resources: SubnetPublicA: # The first of three subnets defined within the VPC... Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPublicARouteTableAssociation Properties: AvailabilityZone: !Select [0, !GetAZs ""] CidrBlock: 192.168.0.0/24 @@ -373,7 +127,6 @@ Resources: SubnetPublicB: # ... and the second of three subnets... Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPublicBRouteTableAssociation Properties: AvailabilityZone: !Select [1, !GetAZs ""] CidrBlock: 192.168.1.0/24 @@ -386,7 +139,6 @@ Resources: SubnetPublicC: # ... and the third of three subnets defined within the VPC Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPublicCRouteTableAssociation Properties: AvailabilityZone: !Select [2, !GetAZs ""] CidrBlock: 192.168.2.0/24 @@ -399,7 +151,6 @@ Resources: SubnetPrivateA: # The first of three subnets defined within the VPC... Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPrivateARouteTableAssociation Properties: AvailabilityZone: !Select [0, !GetAZs ""] CidrBlock: 192.168.3.0/24 @@ -414,7 +165,6 @@ Resources: SubnetPrivateB: # ... and the second of three subnets... Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPrivateARouteTableAssociation Properties: AvailabilityZone: !Select [1, !GetAZs ""] CidrBlock: 192.168.4.0/24 @@ -429,7 +179,6 @@ Resources: SubnetPrivateC: # ... and the third of three subnets defined within the VPC Type: AWS::EC2::Subnet DependsOn: VPC - # DependedOn: SubnetPrivateCRouteTableAssociation Properties: AvailabilityZone: !Select [2, !GetAZs ""] CidrBlock: 192.168.5.0/24 @@ -443,8 +192,6 @@ Resources: InternetGateway: # Create an Internet Gateway in order to allow EC2 instances to be accessible via the Internet Type: AWS::EC2::InternetGateway - # DependsOn: None - # DependedOn: InternetGatewayAttachment Properties: Tags: - Key: Name @@ -455,7 +202,6 @@ Resources: DependsOn: - InternetGateway - VPC - # DependedOn: DefaultRoute Properties: InternetGatewayId: !Ref InternetGateway VpcId: !Ref VPC @@ -465,7 +211,6 @@ Resources: DependsOn: - InternetGatewayAttachment - VPC - # DependedOn: NATGateway Properties: Domain: vpc @@ -473,7 +218,6 @@ Resources: Type: AWS::EC2::NatGateway DependsOn: - SubnetPublicA - # DependedOn: Properties: AllocationId: !GetAtt EIPNATGateway.AllocationId SubnetId: !Ref SubnetPublicA @@ -481,7 +225,6 @@ Resources: RouteTablePublic: # Create a route table which will be used within the VPC Type: AWS::EC2::RouteTable DependsOn: VPC - # DependedOn: DefaultRoutePublic, SubnetPublicARouteTableAssociation, SubnetPublicBRouteTableAssociation, SubnetPublicCRouteTableAssociation Properties: VpcId: !Ref VPC Tags: @@ -491,7 +234,6 @@ Resources: RouteTablePrivate: # Create a route table which will be used within the VPC Type: AWS::EC2::RouteTable DependsOn: VPC - # DependedOn: DefaultRoute, SubnetPrivateARouteTableAssociation, SubnetPrivateBRouteTableAssociation, SubnetPrivateCRouteTableAssociation Properties: VpcId: !Ref VPC Tags: @@ -503,7 +245,6 @@ Resources: DependsOn: - RouteTablePublic - InternetGatewayAttachment - # DependedOn: None Properties: DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway @@ -514,7 +255,6 @@ Resources: DependsOn: - RouteTablePrivate - NATGateway - # DependedOn: None Properties: DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: !Ref NATGateway @@ -525,7 +265,6 @@ Resources: DependsOn: - RouteTablePublic - SubnetPublicA - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePublic SubnetId: !Ref SubnetPublicA @@ -535,7 +274,6 @@ Resources: DependsOn: - RouteTablePublic - SubnetPublicB - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePublic SubnetId: !Ref SubnetPublicB @@ -545,7 +283,6 @@ Resources: DependsOn: - RouteTablePublic - SubnetPublicC - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePublic SubnetId: !Ref SubnetPublicC @@ -555,7 +292,6 @@ Resources: DependsOn: - RouteTablePrivate - SubnetPrivateA - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePrivate SubnetId: !Ref SubnetPrivateA @@ -565,7 +301,6 @@ Resources: DependsOn: - RouteTablePrivate - SubnetPrivateB - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePrivate SubnetId: !Ref SubnetPrivateB @@ -575,7 +310,6 @@ Resources: DependsOn: - RouteTablePrivate - SubnetPrivateC - # DependedOn: None Properties: RouteTableId: !Ref RouteTablePrivate SubnetId: !Ref SubnetPrivateC @@ -583,7 +317,6 @@ Resources: SecurityGroupJenkins: # A Security Group that allows ingress access for SSH and the default port that a Jenkins Master will run on Type: AWS::EC2::SecurityGroup DependsOn: VPC - # DependedOn: JenkinsOnDemandEC2Instance Properties: GroupName: Spot CICD Workshop Jenkins Security Group GroupDescription: A Security Group that allows ingress access for SSH and the default port that a Jenkins Master will run on @@ -627,7 +360,6 @@ Resources: SecurityGroupJenkinsALB: # A Security Group that allows ingress access for HTTP on ALBs and used to access the Jenkins Master Type: AWS::EC2::SecurityGroup DependsOn: VPC - # DependedOn: JenkinsMasterALB Properties: GroupName: Spot CICD Workshop Jenkins ALB Security Group GroupDescription: A Security Group that allows ingress access for HTTP on ALBs and used to access the Jenkins Master @@ -643,7 +375,6 @@ Resources: DependsOn: - SecurityGroupJenkins - VPC - # DependedOn: EFSMountTargetJenkinsHomeVolumeA, EFSMountTargetJenkinsHomeVolumeB, EFSMountTargetJenkinsHomeVolumeC Properties: GroupName: Spot CICD Workshop EFS Security Group GroupDescription: A Security Group that allows access to EFS volume targets from the Jenkins Securiy Group @@ -654,217 +385,6 @@ Resources: SourceSecurityGroupId: !Ref SecurityGroupJenkins VpcId: !Ref VPC - AMILookupLambdaFunction: # A Lambda function that will be used to look up the AMI for the latest Amazon Linux AMI. The source code for this Lambda function can be obtained at https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/walkthrough-custom-resources-lambda-lookup-amiids.html. - Type: AWS::Lambda::Function - DependsOn: IAMRoleAMILookupLambdaExecution - #DependedOn: EC2AMILookupCustomResource, ECSAMILookupCustomResource - Properties: - Code: - S3Bucket: !Sub amazon-ec2-spot-cicd-workshop-${AWS::Region} - S3Key: amilookup.zip - Handler: amilookup.handler - Role: !GetAtt IAMRoleAMILookupLambdaExecution.Arn - Runtime: nodejs12.x - Timeout: 30 - - EC2AMILookupCustomResource: # A custom resource that provides the latest Amazon Linux AMI via EC2AMILookupCustomResource.Id - Type: Custom::AMILookup - DependsOn: AMILookupLambdaFunction - # DependedOn: - Properties: - Architecture: HVM64 - Region: !Ref AWS::Region - ServiceToken: !GetAtt AMILookupLambdaFunction.Arn - - ECSAMILookupCustomResource: # A custom resource that provides the latest Amazon ECS-Optimized AMI via ECSAMILookupCustomResource.Id - Type: Custom::AMILookup - DependsOn: AMILookupLambdaFunction - # DependedOn: - Properties: - Architecture: ECSHVM64 - Region: !Ref AWS::Region - ServiceToken: !GetAtt AMILookupLambdaFunction.Arn - - TestEnvironmentLambdaFunction: # A Lambda function that will be used to create and destroy buld testing environments. - DependsOn: - - AMILookupLambdaFunction - - DeploymentArtifactsS3Bucket - - DynamoDBTestEnvironmentTable - - IAMRoleTestEnvironmentLambdaExecution - - SubnetPublicA - - SubnetPublicB - - SubnetPublicC - - VPC - #DependedOn: None - Type: AWS::Lambda::Function - Properties: - FunctionName: SpotCICDWorkshop_ManageTestEnvironment - Description: Handles the deployment and termination of Test Environments witin the Spot CICD Workshop. - Handler: index.handler - MemorySize: 128 - Role: !GetAtt IAMRoleTestEnvironmentLambdaExecution.Arn - Runtime: nodejs12.x - Timeout: 60 - Code: - ZipFile: !Sub | - 'use strict'; - var AWS = require('aws-sdk'); - var actions = { - deploy: function (cfn, ddb, request_payload) { - return new Promise(function (resolve, reject) { - var cfn_params = { - StackName: request_payload.stackName, - Capabilities: [ 'CAPABILITY_IAM' ], - Parameters: [ - { - ParameterKey: 'KeyPair', - ParameterValue: '${KeyPair}' - }, { - ParameterKey: 'CurrentIP', - ParameterValue: '${CurrentIP}' - }, { - ParameterKey: 'AMILookupLambdaFunctionARN', - ParameterValue: '${AMILookupLambdaFunction.Arn}' - }, { - ParameterKey: 'DeploymentArtifactsS3Bucket', - ParameterValue: '${DeploymentArtifactsS3Bucket}' - }, { - ParameterKey: 'VPC', - ParameterValue: '${VPC}' - }, { - ParameterKey: 'SubnetA', - ParameterValue: '${SubnetPublicA}' - }, { - ParameterKey: 'SubnetB', - ParameterValue: '${SubnetPublicB}' - }, { - ParameterKey: 'SubnetC', - ParameterValue: '${SubnetPublicC}' - } - ], - RoleARN: '${IAMRoleTestEnvironmentCloudFormation.Arn}', - TemplateURL: 'https://s3-us-west-2.amazonaws.com/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop_game-of-life.yaml' - }; - cfn.createStack(cfn_params, function(err, cfn_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'StackId: ', cfn_data.StackId); - return new Promise(function (resolve, reject) { - var ddb_params = { - Item: { - 'JobBaseName': { S: request_payload.jobBaseName }, - 'BuildID': { N: request_payload.buildId }, - 'CloudFormationStackID': { S: cfn_data.StackId } - }, - ReturnConsumedCapacity: 'TOTAL', - TableName: '${DynamoDBTestEnvironmentTable}' - }; - ddb.putItem(ddb_params, function(err, ddb_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'Consumed Capacity Units: ', ddb_data.ConsumedCapacity.CapacityUnits); - return resolve(); - }); - }); - }); - }); - }, - terminate: function(cfn, ddb, request_payload) { - return new Promise(function (resolve, reject) { - var ddb_params = { - Key: { - 'JobBaseName': { S: request_payload.jobBaseName }, - 'BuildID': { N: request_payload.buildId } - }, - TableName: '${DynamoDBTestEnvironmentTable}' - }; - ddb.getItem(ddb_params, function(err, ddb_data) { - if (err) { return reject(err); } - console.log('[INFO]', 'CloudFormationStackId: ', ddb_data.Item.CloudFormationStackID.S); - return new Promise(function (resolve, reject) { - var cfn_params = { - StackName: request_payload.stackName, - RoleARN: '${IAMRoleTestEnvironmentCloudFormation.Arn}' - }; - cfn.deleteStack(cfn_params, function(err, cfn_data) { - if (err) { return reject(err); } - return resolve(); - }); - }); - }); - }); - } - }; - exports.handler = function (event, context, callback) { - var p = actions[event.action]; - if (!p) { - return callback('Unknown action'); - } - var msgAction = event.action.toUpperCase() + ' '; - var cfn = new AWS.CloudFormation(); - var ddb = new AWS.DynamoDB(); - console.log('[INFO]', 'Attempting', msgAction); - return p(cfn, ddb, event).then(function (data) { - return callback(null, data); - }).catch(function (err) { - console.log('[ERROR]', JSON.stringify(err)); - return callback(err); - }); - }; - - JenkinsOnDemandEC2Instance: # This workshop starts from a baseline where we have a Jenkins server running on an on-demand EC2 instance. This resource launches and bootstraps this server - Type: AWS::EC2::Instance - DependsOn: - - EC2AMILookupCustomResource - - InstanceProfileJenkins - - SecurityGroupJenkins - - SubnetPublicA - Properties: - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - DeleteOnTermination: "true" - VolumeSize: 50 - VolumeType: gp2 - IamInstanceProfile: !Ref InstanceProfileJenkins - ImageId: !Ref AmazonLinux2LatestAmiId - InstanceType: t3.medium - KeyName: !Ref KeyPair - SecurityGroupIds: - - !Ref SecurityGroupJenkins - SubnetId: !Ref SubnetPublicA - Tags: - - Key: Name - Value: Jenkins Master (On-demand) - UserData: - Fn::Base64: !Sub | - #!/bin/bash - # Install all pending updates to the system - yum -y update - # Install Docker - amazon-linux-extras install docker - chkconfig docker on - service docker start - usermod -a -G docker ec2-user - # Install Docker Compose - curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - # Download files from GitHub - mkdir -p /jenkins_home/jobs/ - mkdir -p /usr/share/jenkins - cd /usr/share/jenkins - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz - tar -xvf seed.tar.gz - cp -r jobs/* /jenkins_home/jobs/ - # Reset the password defined as the JenkinsAdminPassword - sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml - # Configure the Jenkins Location - curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml - # Start Jenkins - docker-compose up --build -d - JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 Type: AWS::ElasticLoadBalancingV2::LoadBalancer DependsOn: @@ -872,9 +392,8 @@ Resources: - SubnetPublicA - SubnetPublicB - SubnetPublicC - # DependedOn: JenkinsMasterALBListener Properties: - Name: JenkinsMasterALB + Name: JenkinsMasterALBECS Scheme: internet-facing SecurityGroups: - !Ref SecurityGroupJenkinsALB @@ -883,35 +402,10 @@ Resources: - !Ref SubnetPublicB - !Ref SubnetPublicC - JenkinsMasterALBTargetGroupEC2: # This is the Target Group used by the JenkinsMasterALB load balancer when Jenkins is running on an EC2 instance - Type: AWS::ElasticLoadBalancingV2::TargetGroup - DependsOn: - - JenkinsOnDemandEC2Instance - - VPC - # DependedOn: JenkinsMasterALBListener, JenkinsMasterALBListenerRule - Properties: - HealthCheckIntervalSeconds: 15 - HealthCheckPath: /login - HealthCheckPort: 8080 - HealthCheckProtocol: HTTP - HealthCheckTimeoutSeconds: 5 - HealthyThresholdCount: 2 - Matcher: - HttpCode: 200 - Name: JenkinsMasterEC2TargetGroup - Port: 8080 - Protocol: HTTP - Targets: - - Id: !Ref JenkinsOnDemandEC2Instance - Port: 8080 - UnhealthyThresholdCount: 4 - VpcId: !Ref VPC - JenkinsMasterALBTargetGroupECS: # This is the Target Group used by the JenkinsMasterALB load balancer when Jenkins is running in a container on an ECS cluster Type: AWS::ElasticLoadBalancingV2::TargetGroup DependsOn: - VPC - # DependedOn: JenkinsMasterALBListener, JenkinsMasterALBListenerRule Properties: HealthCheckIntervalSeconds: 15 HealthCheckPath: /login @@ -925,6 +419,9 @@ Resources: Port: 8080 Protocol: HTTP TargetType: ip + TargetGroupAttributes: + - Key: "slow_start.duration_seconds" + Value: "120" UnhealthyThresholdCount: 4 VpcId: !Ref VPC @@ -932,39 +429,20 @@ Resources: Type: AWS::ElasticLoadBalancingV2::Listener DependsOn: - JenkinsMasterALB - - JenkinsMasterALBTargetGroupEC2 - # DepenededOn: None + - JenkinsMasterALBTargetGroupECS Properties: DefaultActions: - Type: forward - TargetGroupArn: !Ref JenkinsMasterALBTargetGroupEC2 + TargetGroupArn: !Ref JenkinsMasterALBTargetGroupECS LoadBalancerArn: !Ref JenkinsMasterALB Port: 80 Protocol: HTTP - JenkinsMasterALBListenerRuleEC2: # The ALB Listener rule that forwards all traffic destined for the Jenkins Master to the appropriate Target Group - Type: AWS::ElasticLoadBalancingV2::ListenerRule - DependsOn: - - JenkinsMasterALBListener - - JenkinsMasterALBTargetGroupEC2 - # DependedOn: None - Properties: - Actions: - - Type: forward - TargetGroupArn: !Ref JenkinsMasterALBTargetGroupEC2 - Conditions: - - Field: path-pattern - Values: - - "/*" - ListenerArn: !Ref JenkinsMasterALBListener - Priority: 1 - JenkinsMasterALBListenerRuleECS: # The ALB Listener rule that forwards all traffic destined for the Jenkins Master to the appropriate Target Group Type: AWS::ElasticLoadBalancingV2::ListenerRule DependsOn: - JenkinsMasterALBListener - JenkinsMasterALBTargetGroupECS - # DependedOn: None Properties: Actions: - Type: forward @@ -976,134 +454,8 @@ Resources: ListenerArn: !Ref JenkinsMasterALBListener Priority: 2 - JenkinsSpotMasterLaunchTemplate: # This is a launch template that will be used to provision Jenkins Master servers - showing how when used in conjunction with an EFS volume stateful applications can run on self-healing spot architectures. - Type: AWS::EC2::LaunchTemplate - DependsOn: - - EC2AMILookupCustomResource - - EFSJenkinsHomeVolume - - InstanceProfileJenkins - - SecurityGroupJenkins - # DependedOn: None - Properties: - LaunchTemplateName: JenkinsMasterLaunchTemplate - LaunchTemplateData: - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - DeleteOnTermination: "true" - VolumeSize: 8 - VolumeType: gp2 - IamInstanceProfile: - #Arn: !GetAtt InstanceProfileJenkins.Arn - Name: !Ref InstanceProfileJenkins - ImageId: !Ref AmazonLinux2LatestAmiId - InstanceType: t3.medium - KeyName: !Ref KeyPair - SecurityGroupIds: - - !Ref SecurityGroupJenkins - TagSpecifications: - - ResourceType: instance - Tags: - - Key: Name - Value: Jenkins Master (Spot) - UserData: - Fn::Base64: !Sub | - #!/bin/bash - # Install all pending updates to the system - yum -y update - # Install Docker - amazon-linux-extras install docker - chkconfig docker on - service docker start - usermod -a -G docker ec2-user - # Install Docker Compose - curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - # Mount the Jenkins EFS volume - mkdir -p /jenkins_home - mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFSJenkinsHomeVolume}.efs.eu-west-1.amazonaws.com:/ /jenkins_home - # Download files from GitHub - mkdir -p /usr/share/jenkins - cd /usr/share/jenkins - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt - # Reset the password defined as the JenkinsAdminPassword - sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml - # Configure the Jenkins Location - curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml - # Start Jenkins - docker-compose up --build -d - - JenkinsSpotAgentLaunchTemplate: # This is a launch template that will be used to provision Jenkins build agents - showing how spot instances can be used to scale-out build jobs at low cost. - Type: AWS::EC2::LaunchTemplate - DependsOn: - - EC2AMILookupCustomResource - - InstanceProfileJenkins - - SecurityGroupJenkins - # DependedOn: None - Properties: - LaunchTemplateName: JenkinsBuildAgentLaunchTemplate - LaunchTemplateData: - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - DeleteOnTermination: "true" - VolumeSize: 8 - VolumeType: gp2 - IamInstanceProfile: - #Arn: !GetAtt InstanceProfileJenkins.Arn - Name: !Ref InstanceProfileJenkins - ImageId: !Ref AmazonLinux2LatestAmiId - InstanceType: t3.small - KeyName: !Ref KeyPair - SecurityGroupIds: - - !Ref SecurityGroupJenkins - TagSpecifications: - - ResourceType: instance - Tags: - - Key: Name - Value: Jenkins Build Agent - UserData: - Fn::Base64: !Sub | - #!/bin/bash - # Install all pending updates to the system - yum -y update - # Configure YUM to be able to access contributed Maven RPM packages - wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo - # Update the release version in the Maven repository configuration for this mainline release of Amazon Linux - sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo - # Install the Java 8 SDK, Git and Maven - yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel git apache-maven - # Set the default version of java to run out of the Java 8 SDK path (required by Jenkins) - update-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java - update-alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac - - DynamoDBTestEnvironmentTable: - Type: AWS::DynamoDB::Table - # DependsOn: - # DependedOn: IAMRoleTestEnvironmentLambdaExecution, TestEnvironmentLambdaFunction - Properties: - AttributeDefinitions: - - AttributeName: JobBaseName - AttributeType: "S" - - AttributeName: BuildID - AttributeType: "N" - KeySchema: - - AttributeName: JobBaseName - KeyType: HASH - - AttributeName: BuildID - KeyType: RANGE - ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 - TableName: SpotCICDWorkshopTestEnvironmentTracking - EFSJenkinsHomeVolume: Type: AWS::EFS::FileSystem - # DependsOn: None - # DependedOn: AutoScalingECSLaunchConfiguration Properties: PerformanceMode: generalPurpose @@ -1113,7 +465,6 @@ Resources: - EFSJenkinsHomeVolume - SecurityGroupEFS - SubnetPublicA - # DependedOn: Properties: FileSystemId: !Ref EFSJenkinsHomeVolume SecurityGroups: @@ -1126,7 +477,6 @@ Resources: - EFSJenkinsHomeVolume - SecurityGroupEFS - SubnetPublicB - # DependedOn: Properties: FileSystemId: !Ref EFSJenkinsHomeVolume SecurityGroups: @@ -1139,7 +489,6 @@ Resources: - EFSJenkinsHomeVolume - SecurityGroupEFS - SubnetPublicC - # DependedOn: Properties: FileSystemId: !Ref EFSJenkinsHomeVolume SecurityGroups: @@ -1149,11 +498,9 @@ Resources: ECSLaunchTemplate: # This is a launch template that will be used to provision ECS cluster nodes Type: AWS::EC2::LaunchTemplate DependsOn: - - ECSAMILookupCustomResource - EFSJenkinsHomeVolume - InstanceProfileECS - SecurityGroupJenkins - # DependedOn: None Properties: LaunchTemplateName: ECSLaunchTemplate LaunchTemplateData: @@ -1161,7 +508,7 @@ Resources: - DeviceName: "/dev/xvda" Ebs: DeleteOnTermination: "true" - VolumeSize: 8 + VolumeSize: 30 VolumeType: gp2 IamInstanceProfile: Name: !Ref InstanceProfileECS @@ -1174,7 +521,7 @@ Resources: - ResourceType: instance Tags: - Key: Name - Value: ECS Cluster Instance + Value: "Jenkins ECS Cluster Instance" UserData: Fn::Base64: !Sub | #!/bin/bash @@ -1207,22 +554,80 @@ Resources: nohup ./interruption_check.sh &>/dev/null & fi + ASGOnDemand: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + HealthCheckType: EC2 + HealthCheckGracePeriod: 300 + MinSize: 1 + MaxSize: 1 + DesiredCapacity: 1 + VPCZoneIdentifier: + - Ref: SubnetPublicA + - Ref: SubnetPublicB + - Ref: SubnetPublicC + NewInstancesProtectedFromScaleIn: true + MixedInstancesPolicy: + InstancesDistribution: + OnDemandAllocationStrategy: lowest-price + OnDemandBaseCapacity: 0 + OnDemandPercentageAboveBaseCapacity: 100 + LaunchTemplate: + LaunchTemplateSpecification: + LaunchTemplateId: + Ref: ECSLaunchTemplate + Version: !GetAtt ECSLaunchTemplate.LatestVersionNumber + Overrides: + - InstanceRequirements: + VCpuCount: + Min: 2 + Max: 8 + MemoryMiB: + Min: 4096 + CpuManufacturers: + - intel + Tags: + - Key: Name + Value: "EC2OnDemandJenkinsECS" + PropagateAtLaunch: true + ECSCluster: Type: AWS::ECS::Cluster DependsOn: ECSServiceLinkedRole - # DependedOn: ECSServiceJenkinsMaster Properties: ClusterName: SpotCICDWorkshopECSCluster + ECSCapacityProviderOD: + Type: AWS::ECS::CapacityProvider + Properties: + AutoScalingGroupProvider: + AutoScalingGroupArn: !Ref ASGOnDemand + ManagedScaling: + MaximumScalingStepSize: 1 + MinimumScalingStepSize: 1 + Status: ENABLED + TargetCapacity: 100 + ManagedTerminationProtection: ENABLED + + ClusterCPAssociation: + Type: "AWS::ECS::ClusterCapacityProviderAssociations" + Properties: + Cluster: !Ref ECSCluster + CapacityProviders: + - !Ref ECSCapacityProviderOD + DefaultCapacityProviderStrategy: + - Base: 1 + Weight: 100 + CapacityProvider: !Ref ECSCapacityProviderOD + ECSTaskDefinitionJenkinsMaster: Type: AWS::ECS::TaskDefinition DependsOn: ECSServiceLinkedRole - # DependedOn: Properties: ContainerDefinitions: - Cpu: 512 Essential: true - Image: jenkins/jenkins:lts + Image: christianhxc/amazon-ec2-spot-cicd-workshop-jenkins:2.346.3 Memory: 1536 MountPoints: - SourceVolume: JENKINS_HOME @@ -1235,11 +640,15 @@ Resources: HostPort: 8080 - ContainerPort: 50000 HostPort: 50000 - ExecutionRoleArn: !GetAtt IAMRoleJenkins.Arn + Environment: + - Name: "JENKINS_ADMIN_ID" + Value: "admin" + - Name: "JENKINS_ADMIN_PASSWORD" + Value: !Ref JenkinsAdminPassword + Privileged: true + User: "root" + ExecutionRoleArn: !GetAtt IAMRoleECS.Arn NetworkMode: awsvpc - PlacementConstraints: - - Type: memberOf - Expression: attribute:lifecycle != spot Volumes: - Host: SourcePath: /mnt/efs/jenkins_home @@ -1251,16 +660,14 @@ Resources: - ECSCluster - ECSServiceLinkedRole - ECSTaskDefinitionJenkinsMaster - # - IAMRoleECSServiceRole - JenkinsMasterALBListenerRuleECS - ServiceDiscoveryJenkinsMaster - # # DependedOn: None Properties: Cluster: !Ref ECSCluster DeploymentConfiguration: MaximumPercent: 100 MinimumHealthyPercent: 50 - DesiredCount: 0 + DesiredCount: 1 HealthCheckGracePeriodSeconds: 120 LoadBalancers: - ContainerName: SpotCICDWorkshopJenkinsMasterContainer @@ -1274,8 +681,6 @@ Resources: - !Ref SubnetPrivateA - !Ref SubnetPrivateB - !Ref SubnetPrivateC - # Role: !GetAtt IAMRoleECSServiceRole.Arn - # ServiceName: JenkinsMaster ServiceRegistries: - ContainerName: SpotCICDWorkshopJenkinsMasterContainer RegistryArn: !GetAtt ServiceDiscoveryJenkinsMaster.Arn @@ -1284,7 +689,6 @@ Resources: ServiceDiscoveryJenkinsMasterNamespace: Type: AWS::ServiceDiscovery::PrivateDnsNamespace DependsOn: VPC - # DependedOn: None Properties: Vpc: !Ref VPC Name: jenkins.local @@ -1292,7 +696,6 @@ Resources: ServiceDiscoveryJenkinsMaster: Type: AWS::ServiceDiscovery::Service DependsOn: ServiceDiscoveryJenkinsMasterNamespace - # DependedOn: ECSServiceJenkinsMaster Properties: Description: Jenkins Master Service DnsConfig: @@ -1305,10 +708,6 @@ Resources: Name: master Outputs: - DeploymentArtifactsS3Bucket: - Description: The name of the S3 bucket where deployment artifacts are uploaded to. - Value: !Ref DeploymentArtifactsS3Bucket - EFSFileSystemID: Description: The file system ID of the EFS volume that is used to persist JENKINS_HOME across EC2 & ECS instances. Value: !Ref EFSJenkinsHomeVolume @@ -1317,10 +716,6 @@ Outputs: Description: The DNS name of the Application Load Balancer that is used to gain access to your Jenkins server. Value: !GetAtt JenkinsMasterALB.DNSName - JenkinsIAMRoleARN: - Description: The ARN associated with the IAM Role that was created for use by Jenkins. - Value: !GetAtt IAMRoleJenkins.Arn - JenkinsMasterSecurityGroup: Description: Security Group for Jenkins nodes. Use this value to configure Jenkins ECS Plugin Value: !Ref SecurityGroupJenkins @@ -1338,6 +733,6 @@ Outputs: Value: !Join [",", [!Ref SubnetPublicA, !Ref SubnetPublicB, !Ref SubnetPublicC]] - JenkinsMasterEC2TargetGroup: + JenkinsMasterALBTargetGroupECS: Description: Target Group for Jenkins EC2 nodes. - Value: !Ref JenkinsMasterALBTargetGroupEC2 + Value: !Ref JenkinsMasterALBTargetGroupECS diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile index f107833a..bc0dfc86 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.346.2-lts-jdk11 +FROM jenkins/jenkins:2.346.3-lts-jdk11 ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false ENV CASC_JENKINS_CONFIG /usr/share/jenkins/ref/casc.yaml @@ -6,4 +6,10 @@ ENV CASC_JENKINS_CONFIG /usr/share/jenkins/ref/casc.yaml COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt -COPY casc.yaml /usr/share/jenkins/ref/casc.yaml \ No newline at end of file +COPY casc.yaml /usr/share/jenkins/ref/casc.yaml + +WORKDIR /jenkins_seed +RUN curl https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz -o seed.tar.gz \ + && tar -xvf seed.tar.gz \ + && mkdir /usr/share/jenkins/ref/jobs \ + && mv jobs/* /usr/share/jenkins/ref/jobs/ \ No newline at end of file From 2580a1b65d60d115bd8be344bb70ae9ff9641a03 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 5 Sep 2022 15:31:20 +0200 Subject: [PATCH 20/29] Using the Docker Hub container image --- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 2 +- .../amazon-ec2-spot-cicd-workshop/container/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index 58ec8ec6..ad8bb7b7 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -536,7 +536,7 @@ Resources: # Configure the Jenkins Location curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml # Start Jenkins - docker-compose up --build -d + docker-compose up -d JenkinsSpotAgentLaunchTemplate: # This is a launch template that will be used to provision Jenkins build agents - showing how spot instances can be used to scale-out build jobs at low cost. Type: AWS::EC2::LaunchTemplate diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml index 8a5f9157..8d77d8e7 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.2' services: jenkins: - image: flexco-cicd/jenkins + image: christianhxc/amazon-ec2-spot-cicd-workshop-jenkins:2.346.3 privileged: true user: root build: . From 897672d06d1bf3e1bdc19befe91841b34554895f Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 5 Sep 2022 15:33:18 +0200 Subject: [PATCH 21/29] Using the Docker Hub container image --- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index ad8bb7b7..6ee9ed26 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -410,7 +410,7 @@ Resources: # Configure the Jenkins Location curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml # Start Jenkins - docker-compose up --build -d + docker-compose up -d JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 Type: AWS::ElasticLoadBalancingV2::LoadBalancer From f7ae1ed3cb42afdd1fa686e8d3c3dd18412bea30 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 5 Sep 2022 18:09:00 +0200 Subject: [PATCH 22/29] Using the Docker Hub container image --- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index 6ee9ed26..d7497f34 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -402,9 +402,6 @@ Resources: wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/jobs/seed.tar.gz - tar -xvf seed.tar.gz - cp -r jobs/* /jenkins_home/jobs/ # Reset the password defined as the JenkinsAdminPassword sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml # Configure the Jenkins Location From 8feb6de7a308af917574caaa360f9455b3623b9d Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 5 Sep 2022 19:53:49 +0200 Subject: [PATCH 23/29] Using the Docker Hub container image --- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index d7497f34..8126d268 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -407,7 +407,8 @@ Resources: # Configure the Jenkins Location curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml # Start Jenkins - docker-compose up -d + docker-compose pull + docker-compose up -d --no-build JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 Type: AWS::ElasticLoadBalancingV2::LoadBalancer @@ -533,7 +534,8 @@ Resources: # Configure the Jenkins Location curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml # Start Jenkins - docker-compose up -d + docker-compose pull + docker-compose up -d --no-build JenkinsSpotAgentLaunchTemplate: # This is a launch template that will be used to provision Jenkins build agents - showing how spot instances can be used to scale-out build jobs at low cost. Type: AWS::EC2::LaunchTemplate From 99a980979761c1cb7012ad3c87bdbbd599267d65 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 6 Sep 2022 17:30:07 +0200 Subject: [PATCH 24/29] Updating the CI/CD CFN to use ASG --- .../jenkins-asg/prep.md | 12 ++- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 90 ++++++++----------- .../container/plugins.txt | 2 +- .../fisspotinterruption.yaml | 2 +- 4 files changed, 46 insertions(+), 60 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md index 2a4fa3b1..c149f708 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/prep.md @@ -27,17 +27,23 @@ Be sure to give it a stack name of **SpotCICDWorkshop** and ensure that you supp 2. Click on the **Create Stack** button towards the top of the console; 3. At the Select Template screen, select the **Upload a template file** radio button and choose the CloudFormation template you downloaded before, then click on the **Next** button; 4. At the Specify Details screen, enter in **SpotCICDWorkshop** as the Stack name. Under the Parameters section: - 1. Identify what your current public IP address is by going to https://www.google.com.au/search?q=what%27s+my+ip+address. Enter the first three octets of this IP address into the **CurrentIP** parameter field and then add the **.0/24** suffix. For example if your IP address was 54.240.193.193, you would enter 54.240.193.0/24 into the CurrentIP field[^1]; + 1. Identify what your current public IP address is by going to https://www.google.com.au/search?q=what%27s+my+ip+address. Enter the first three octets of this IP address into the **CurrentIP** parameter field and then add the **.0/24** suffix. For example if your IP address was 54.240.193.193, you would enter 54.240.193.0/24 into the CurrentIP field; 2. Enter in a password that you would like to use for the administrator account within the Jenkins server that will be launched by the CloudFormation template; 3. Select the **Spot CICD Workshop Key Pair** option in the Keypair dropdown. 5. Click on the **Next** button; 6. At the Options screen, there is no need to make any changes to the default options – simply click on the **Next** button; 7. Finally at the Review screen, verify your settings, mark the **I acknowledge that AWS CloudFormation might create IAM resources with custom names** checkbox and then click on the **Create** button. Wait for the stack to complete provisioning, which should take a couple of minutes. -[^1]: It's good security practice to ensure that the web and SSH services being used in this workshop are not accessible to everyone on the Internet. In most cases, limiting access to the /24 CIDR block that you IP address is in provides a reasonable level of access control - but this may still be too restrictive in some corporate IT environments. If you have trouble accessing resources, additional instructions within this lab guide will guide you through what settings need to be manually changed. - The stack should take around five minutes to deploy. +{{% notice note %}} +It's good security practice to ensure that the web and SSH services being used in this workshop are not accessible to everyone on the Internet. In most cases, limiting access to the /24 CIDR block that you IP address is in provides a reasonable level of access control - but this may still be too restrictive in some corporate IT environments. If you have trouble accessing resources, additional instructions within this lab guide will guide you through what settings need to be manually changed. +{{% /notice %}} + +{{% notice note %}} +The CloudFormation template creates a new Launch Template to install and bootstrap Jenkins. However, you can continue using any existing Launch Template that you might already have. +{{% /notice %}} + ## Setting Up environment variables You need to set up the following environment variables that you'll use in the workshop, to do so, run the following commands: diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index 8126d268..a2a97b8a 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -358,57 +358,40 @@ Resources: SourceSecurityGroupId: !Ref SecurityGroupJenkins VpcId: !Ref VPC - JenkinsOnDemandEC2Instance: # This workshop starts from a baseline where we have a Jenkins server running on an on-demand EC2 instance. This resource launches and bootstraps this server - Type: AWS::EC2::Instance + JenkinsMasterGroup: + Type: AWS::AutoScaling::AutoScalingGroup DependsOn: - - InstanceProfileJenkins - - SecurityGroupJenkins - - SubnetPublicA - Properties: - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - DeleteOnTermination: "true" - VolumeSize: 50 - VolumeType: gp2 - IamInstanceProfile: !Ref InstanceProfileJenkins - ImageId: !Ref AmazonLinux2LatestAmiId - InstanceType: t3.medium - KeyName: !Ref KeyPair - SecurityGroupIds: - - !Ref SecurityGroupJenkins - SubnetId: !Ref SubnetPublicA + - InstanceProfileJenkins + - SecurityGroupJenkins + - SubnetPublicA + - SubnetPublicB + - SubnetPublicC + Properties: + HealthCheckType: EC2 + HealthCheckGracePeriod: 300 + MinSize: "1" + MaxSize: "1" + DesiredCapacity: "1" + VPCZoneIdentifier: + - Ref: SubnetPublicA + - Ref: SubnetPublicB + - Ref: SubnetPublicC + MixedInstancesPolicy: + InstancesDistribution: + OnDemandAllocationStrategy: lowest-price + OnDemandBaseCapacity: 0 + OnDemandPercentageAboveBaseCapacity: 100 + LaunchTemplate: + LaunchTemplateSpecification: + LaunchTemplateId: + Ref: JenkinsMasterLaunchTemplate + Version: "1" + TargetGroupARNs: + - !Ref JenkinsMasterALBTargetGroupEC2 Tags: - Key: Name - Value: Jenkins Master (On-demand) - UserData: - Fn::Base64: !Sub | - #!/bin/bash - # Install all pending updates to the system - yum -y update - # Install Docker - amazon-linux-extras install docker - chkconfig docker on - service docker start - usermod -a -G docker ec2-user - # Install Docker Compose - curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - # Download files from GitHub - mkdir -p /jenkins_home/jobs/ - mkdir -p /usr/share/jenkins - cd /usr/share/jenkins - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/casc.yaml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile - wget https://raw.githubusercontent.com/christianhxc/ec2-spot-workshops/cicd-upgrade/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt - # Reset the password defined as the JenkinsAdminPassword - sed -i 's/JenkinsAdminPassword/${JenkinsAdminPassword}/' docker-compose.yml - # Configure the Jenkins Location - curl -s http://169.254.169.254/latest/meta-data/public-ipv4 | xargs -I {} sed -i 's/localhost/{}/' casc.yaml - # Start Jenkins - docker-compose pull - docker-compose up -d --no-build + Value: jenkins-master + PropagateAtLaunch: true JenkinsMasterALB: # This is the Application Load Balancer that resides in front of your Jenkins Master instance and is responsible for port-mapping requests from TCP:80 to TCP:8080 Type: AWS::ElasticLoadBalancingV2::LoadBalancer @@ -430,7 +413,6 @@ Resources: JenkinsMasterALBTargetGroupEC2: # This is the Target Group used by the JenkinsMasterALB load balancer when Jenkins is running on an EC2 instance Type: AWS::ElasticLoadBalancingV2::TargetGroup DependsOn: - - JenkinsOnDemandEC2Instance - VPC Properties: HealthCheckIntervalSeconds: 15 @@ -444,9 +426,7 @@ Resources: Name: JenkinsMasterEC2TargetGroup Port: 8080 Protocol: HTTP - Targets: - - Id: !Ref JenkinsOnDemandEC2Instance - Port: 8080 + TargetType: instance UnhealthyThresholdCount: 4 VpcId: !Ref VPC @@ -479,7 +459,7 @@ Resources: ListenerArn: !Ref JenkinsMasterALBListener Priority: 1 - JenkinsSpotMasterLaunchTemplate: # This is a launch template that will be used to provision Jenkins Master servers - showing how when used in conjunction with an EFS volume stateful applications can run on self-healing spot architectures. + JenkinsMasterLaunchTemplate: # This is a launch template that will be used to provision Jenkins Master servers - showing how when used in conjunction with an EFS volume stateful applications can run on self-healing spot architectures. Type: AWS::EC2::LaunchTemplate DependsOn: - EFSJenkinsHomeVolume @@ -505,7 +485,7 @@ Resources: - ResourceType: instance Tags: - Key: Name - Value: Jenkins Master (Spot) + Value: jenkins-master UserData: Fn::Base64: !Sub | #!/bin/bash @@ -562,7 +542,7 @@ Resources: - ResourceType: instance Tags: - Key: Name - Value: Jenkins Build Agent + Value: jenkins-build-agent UserData: Fn::Base64: !Sub | #!/bin/bash diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt index 3e0c706a..11d46baf 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -5,7 +5,7 @@ cloudbees-folder:6.729.v2b_9d1a_74d673 configuration-as-code:1512.vb_79d418d5fc8 credentials-binding:523.vd859a_4b_122e6 email-ext:2.90 -git:4.11.4 +git:4.11.5 github-branch-source:1677.v731f745ea_0cf gradle:1.39.4 ldap:2.10 diff --git a/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml index 97db2e50..3cdfca08 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml @@ -42,7 +42,7 @@ Resources: Targets: SpotIntances: ResourceTags: - Name: "Jenkins Master (Spot)" + Name: "jenkins-build-agent" Filters: - Path: State.Name Values: From 77330d8e44abbf75268f12096b6aac2f7946c350 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 6 Sep 2022 17:35:06 +0200 Subject: [PATCH 25/29] Remove the persistence lab for the CI/CD workshop --- .../jenkins-asg/test-persistence.md | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md deleted file mode 100644 index 285463af..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.md +++ /dev/null @@ -1,95 +0,0 @@ -+++ -title = "Configure Persistence Storage" -weight = 125 -+++ -You're now using Spot instances for your code builds – but your Jenkins server is still using an on-demand instance. Jenkins itself does not natively support running in high-availability configurations because it persists all data on a local file system. If you can store this data durably somewhere else than on the local file system, you can move your Jenkins Master instance to a self-healing Spot instance. To provide persistence for this file system data, you’ll move your Jenkins data to an Elastic File System (EFS) volume and mount this volume on instance spawned by an Auto Scaling group. - -## Get the EFS Filesystem ID -As with the previous steps, the CloudFormation stack deployed during your Workshop Preparation has provisioned some of the resources required for this lab. You will need to determine and make a note of what the **EFS Filesystem ID** is. To get it, run the following command get the ID from the CloudFormation stack output: - -```bash -aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='EFSFileSystemID'].OutputValue" --output text; -``` - -## Copy the contents of Jenkins home to your EFS file system -In order to copy the contents of the `JENKINS_HOME` directory to the EFS file system, you'll need to first mount the file system on the EC2 instance currently running your Jenkins server. To do so, SSH into the Jenkins server: - -1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); -2. Select the instance with the Name tag of **Jenkins Master (On-demand)** and make a note of it's current IPv4 Pubic IP; -3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity; - -Then, run the following command and replace the `EFSFileSystemID` text with the ID you got in the previous section: - -```bash -sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ -$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\ -.EFSFileSystemID.efs.eu-west-1.amazonaws.com:/ /mnt -``` - -Once the file system has been mounted, stop the Jenkins container. - -```bash -cd /usr/share/jenkins -docker-compose stop -``` - -The content of `JENKINS_HOME` is stored under /jenkins_home – copy this content to your EFS file system mount point (/mnt) using the following commands (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): - -```bash -sudo cp -rpv /jenkins_home/* /mnt -``` - -## Provision an Auto Scaling group for the new Jenkins host -The Auto Scaling group that you'll provision for your Jenkins server will be configured in a similar manner to what you did for the build agents: launch only Spot instances with the capacity-optimized strategy, and using ABS to increase diversification. Additionally, you'll configure this Auto Scaling group so that the instances are associated with the Target Group used by the Application Load Balancer that you've been using to access Jenkins. - -Run the following commands to create the Auto Scaling group configuration file: - -```bash -export LAUNCH_TEMPLATE_HOST_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsMasterLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); -cat < ~/asg-jenkins-host-policy.json -{ - "LaunchTemplate":{ - "LaunchTemplateSpecification":{ - "LaunchTemplateId":"${LAUNCH_TEMPLATE_HOST_ID}", - "Version":"\$Latest" - }, - "Overrides":[ - { - "InstanceRequirements": { - "VCpuCount":{"Min": 2, "Max": 8}, - "MemoryMiB":{"Min": 4096} } - } - ] - }, - "InstancesDistribution":{ - "OnDemandBaseCapacity": 0, - "OnDemandPercentageAboveBaseCapacity": 0, - "SpotAllocationStrategy":"capacity-optimized" - } -} -EoF -``` - -Then, run the following command to create the Auto Scaling group. Notice that you might need to re-create the environment variables you created before. - -```bash -aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsHostASG --min-size 1 --max-size 1 --desired-capacity 1 --vpc-zone-identifier "${PUBLIC_SUBNETS}" --mixed-instances-policy file://asg-jenkins-host-policy.json; -``` - -To include the new instances into the Application Load Balancer, attach the Auto Scaling group to the Target Group by running the following command: - -```bash -export EC2_TARGET_GROUP=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsMasterEC2TargetGroup'].OutputValue" --output text); -aws autoscaling attach-load-balancer-target-groups \ - --auto-scaling-group-name EC2SpotJenkinsHostASG \ - --target-group-arns $EC2_TARGET_GROUP; -``` - -## Verify that the new Spot instance is running the Jenkins server -After a few moments, your Spot instance will start up and should attach itself the the Target Group being used by your Application Load Balancer. Determine if this registration has completed successfully and when it has done so, access Jenkins through your Load Balancer and fire off a build of Apache PDFBox to ensure that everything is still working as expected. - -1. Go to the **EC2** console and click on the **Target Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#TargetGroups)); -2. Select the checkbox corresponding to the target group named **JenkinsMasterEC2TargetGroup**, then select the **Targets** tab. You should see your **Jenkins Master (On-demand)** instance with an unhealthy status (as a result of you stopping the Jenkins service on this host). Depending on how quickly you’ve got to this point, you might also see a second instance registered in the target group – the **Jenkins Master (Spot)** instance from the Auto Scaling group that you just created. If you don’t see it, it’s likely that your new spot instance isn't up and running yet – something that usually takes a minute or two to complete. Refresh the list of targets every minute or so until the **Jenkins Master (Spot)** instance appears in the list. While there are no healthy instances in the target group, your web browser should return a HTTP 502 error when you attempt to load Jenkins through your ALB; -3. When the **Jenkins Master (Spot)** instance is shown with a healthy status, you should then be able to reload Jenkins through the ALB's DNS name. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. - -Let's now see what's the impact for Jenkins when a Spot interrumption comes. \ No newline at end of file From 6f085d55fcc92557d000a0afb22b8b1853897995 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 6 Sep 2022 17:38:37 +0200 Subject: [PATCH 26/29] Breaking down the ECS labs from the CI/CD workshop --- .../jenkins-ecs/asg.md | 57 +++++++ .../jenkins-ecs/clean.md | 26 +++ .../jenkins-ecs/ec2-fleet-jenkins.md | 44 +++++ .../jenkins-ecs/lab4.md | 155 ------------------ .../jenkins-ecs/prep.md | 46 ++++++ .../spot_interruption_experiment.md | 105 ++++++++++++ .../jenkins-ecs/spot_interruption_fis.md | 54 ++++++ .../jenkins-ecs/test-build.md | 22 +++ .../tracking_spot_interruptions.md | 36 ++++ .../amazon-ec2-spot-cicd-workshop-ecs.yaml | 5 - 10 files changed, 390 insertions(+), 160 deletions(-) create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/clean.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/ec2-fleet-jenkins.md delete mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/prep.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_experiment.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_fis.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/test-build.md create mode 100644 content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/tracking_spot_interruptions.md diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md new file mode 100644 index 00000000..d4c5676e --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md @@ -0,0 +1,57 @@ ++++ +title = "Launch an Auto Scaling group" +weight = 110 ++++ +An Auto Scaling group contains a collection of Amazon EC2 Instances that are treated as a logical grouping for the purposes of automatic scaling and management. An Auto Scaling group also enables you to use Amazon EC2 Auto Scaling features such as health check replacements and scaling policies. Both maintaining the number of instances in an Auto Scaling group and automatic scaling are the core functionality of the Amazon EC2 Auto Scaling service. + +Amazon EC2 Auto Scaling helps you ensure that you have the correct number of Amazon EC2 Instances available to handle the load for your application. You can specify the minimum and maximum number of instances, and Amazon EC2 Auto Scaling ensures that your group never goes below or above this size. + +When adopting EC2 Spot Instances, our recommendation is use Auto Scaling groups since it offers a very rich API with benefits like scale-in protection, health checks, lifecycle hooks, rebalance recommendation integration, warm pools and predictive scaling, and many more functionalities that we list below. + +## Launching an Auto Scaling group +For this workshop, you need to create an Auto Scaling group that will be used by the Jenkins plugin to perform your application builds. One of the key Spot best practice is to select multiple instance types. At AWS, instance types comprise varying combinations of CPU, memory, storage, and networking capacity to give you the flexibility to choose the appropriate mix of resources for your applications. However, to make this selection simpler, AWS released **[Attribute-Based Instance Type Selection (ABS)](https://aws.amazon.com/blogs/aws/new-attribute-based-instance-type-selection-for-ec2-auto-scaling-and-ec2-fleet/)** to express workload requirements as a set of instance attributes such as: vCPU, memory, type of processor, etc. ABS translates these requirements and selects all matching instance types that meet the criteria. To select which instance to launch, the Auto Scaling group chose instances based on the allocation strategy configured. For Spot Instances we recommend to use **capacity-optimized**, which select the optimal instances that reduce the frequency of interruptions. ABS does also future-proof the Auto Scaling group configuration: *any new instance type we launch that matches the selected attributes, will be included in the list automatically*. No need to update the Auto Scaling group configuration. + +To launch the Auto Scaling group, first create a *json* configuration file. This file describes a mixed-instance-policy section with a set of overrides that drive **diversification of Spot Instance pools** using ABS. The configuration of the Auto Scaling group does refer to the Launch Template that was created in the previous steps with Cloudformation, so make sure you have the corresponding value in the `LAUNCH_TEMPLATE_ID` environment variable + +Here's the command you need to run to create the configuration file: + +```bash +cat < ~/asg-policy.json +{ + "LaunchTemplate":{ + "LaunchTemplateSpecification":{ + "LaunchTemplateId":"${LAUNCH_TEMPLATE_ID}", + "Version":"\$Latest" + }, + "Overrides":[ + { + "InstanceRequirements": { + "VCpuCount":{"Min": 2, "Max": 8}, + "MemoryMiB":{"Min": 4096} } + } + ] + }, + "InstancesDistribution":{ + "OnDemandBaseCapacity": 0, + "OnDemandPercentageAboveBaseCapacity": 0, + "SpotAllocationStrategy":"capacity-optimized" + } +} +EoF +``` + +Notice that in the override section is where you specify the instance attributes with a range of 2 to 8 vCPUs, and a minimum of 4 GBs of memory. ABS will pick a list of instance types that match the criteria like `m5.large`, `c4.large`, or `c5.large`. Then, based on the **capacity-optimized** allocation strategy, the Auto Scaling group will launch instances from pool with more spare capacity available. + +With Auto Scaling groups you can define what is the balance between Spot vs On-Demand Instances that makes sense for your workload. `OnDemandBaseCapacity` allows you to set an initial capacity of On-Demand Instances to use. After that, any new procured capacity will be a mix of Spot and On-Demand Instances as defined by the `OnDemandPercentageAboveBaseCapacity`. This time, *we've configured the Auto Scaling group to launch only Spot instances*. + +Finally, the configuration above, sets the `SpotAllocationStrategy` to `capacity-optimized`. The `capacity-optimized` allocation strategy **allocates instances from the Spot Instance pools with the optimal capacity for the number of instances that are launching**, making use of real-time capacity data and optimizing the selection of used Spot Instances. You can read about the benefits of using `capcity-optimized` in the blog post [Capacity-Optimized Spot Instance allocation in action at Mobileye and Skyscanner](https://aws.amazon.com/blogs/aws/capacity-optimized-spot-instance-allocation-in-action-at-mobileye-and-skyscanner/). + +Let’s now create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (*to avoid running instances when there's no need*), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. + +Run the following command: + +```bash +aws autoscaling create-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG --min-size 0 --max-size 2 --desired-capacity 0 --vpc-zone-identifier "${PRIVATE_SUBNETS}" --mixed-instances-policy file://asg-policy.json +``` + +Nothing else to do here, it's time to configure Jenkins to use this Auto Scaling group. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/clean.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/clean.md new file mode 100644 index 00000000..c8d78338 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/clean.md @@ -0,0 +1,26 @@ ++++ +title = "Cleanup" +weight = 160 ++++ +Congratulations, you have completed the Jenkins with Auto Scaling group lab! Your next challenge is to remove all of the resources that were provisioned in your account so as to ensure that no additional cost can be incurred. Please note that the commands below should be executed in order - some later steps have dependencies on earlier ones! + +## Delete all Auto Scaling groups + +```bash +aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsASG; +aws autoscaling delete-auto-scaling-group --auto-scaling-group-name EC2SpotJenkinsHostASG; +``` + +## Delete the CloudFormation stacks + +```bash +aws cloudformation delete-stack --stack-name SpotCICDWorkshop; +aws cloudformation delete-stack --stack-name track-spot-interruption; +aws cloudformation delete-stack --stack-name fis-spot-interruption; +``` + +## Remove theh EC2 key pair +The final resource that needs to be removed was the first one that you created - the EC2 Key Pair that you created prior to launching the CloudFormation stack. + +1. Go to the **EC2** console and click on the **Key Pairs** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#KeyPairs)); +2. Mark the check box associated with the Key Pair named **Spot CICD Workshop Key Pair** and click on the **Delete** button. At the resulting pop-up, confirm this action by clicking on the **Yes** button. diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/ec2-fleet-jenkins.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/ec2-fleet-jenkins.md new file mode 100644 index 00000000..68a2fd53 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/ec2-fleet-jenkins.md @@ -0,0 +1,44 @@ ++++ +title = "Configure EC2 Fleet plugin" +weight = 115 ++++ +The [EC2 Fleet Plugin](https://plugins.jenkins.io/ec2-fleet/) launches EC2 Spot or On Demand instances as worker nodes for Jenkins CI server, automatically scaling the capacity with the load. The EC2 FLeet plugin will request EC2 instances when excess jobs are detected. You can configure the plugin to use an Auto Scaling Group to launch instances instead of directly launching them by itself. This gives theh plugin all the benefits from Auto Scaling groups like allocation strategies, configure multiple instance types and availability zones, etc. Moreover, the EC2 Fleet plugin can automatically resubmit failed jobs caused by a Spot interruption. + +To start using this plugin, you need to configure it in Jenkins, so let's do it. + +## Sign-in to Jenkins +The CloudFormation template deployed during the Workshop Preparation stage deployed a Jenkins server on to an on-demand instance within your VPC and configured an Application Load Balancer (ALB) to proxy requests from the public Internet to the server. You can obtain the DNS name for the ALB from the Output tab of your CloudFormation template. Point your web browser to this DNS name and sign in using **admin** as the Username and the password that you supplied to the CloudFormation template as the password. + +1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); +2. Click on the checkbox associated with the **SpotCICDWorkshop** stack, then click on the **Outputs** tab toward the bottom of the screen; +3. Make a note of the DNS name for the Application Load Balancer, which is associated with the **JenkinsDNSName** key; +4. Open up a new tab in your browser and enter the DNS name in the address bar. You should be greeted with a Jenkins Sign In screen: + 1. Enter in **admin** as the Username; + 2. Enter in the password that you supplied to the CloudFormation template as the Password. + +## Configure the EC2 Fleet Jenkins plugin +The EC2 Fleet Jenkins Plugin was installed on the Jenkins server during the CloudFormation deployment - but now the plugin needs to be configured. You'll need to get the plugin to **Launch slave agents via SSH** and provide valid SSH credentials (don't forget to consider how Host Key Verification should be set when using Spot instances). + +When configuring the plugin, think about how you could force build processes to run on the spot instances (use the **spot-agents** label), and consider how you can verify that the fleet scales out when there is a backlog of build jobs waiting to be processed. + +1. From the Jenkins home screen, click on the **Manage Jenkins** link on the left side menu, and then the **Manage Nodes and Clouds** link; +2. Click on the **Configure Clouds** link on the left side menu, then click on the **Add a new cloud** dropdown, followed by the **Amazon EC2 Fleet** option; +3. **You don't need to configure any AWS Credentials** as the plugin will use the IAM Role attached to the instance; +4. Select **eu-west-1 EU (Ireland)** from the Region dropdown - the plugin will now attempt to obtain a list of Auto Scaling groups in the selected region; +6. Select the Auto Scaling group that you created earlier (`Auto Scaling Group - EC2SpotJenkinsASG`) from the **EC2 Fleet** dropdown (though it might already be selected) and then select the **Launch slave agents via SSH** option from the Launcher dropdown - this should reveal additional SSH authentication settings; +7. Click the **Add** button next to the Credentials dropdown and select the **Jenkins** option. This will pop up another **Jenkins Credentials Provider: Jenkins** sub-form. Fill out the form as follows: + 1. Change the Kind to **SSH Username with private key**; + 2. Change the Scope to **System (Jenkins and nodes only)** – you also don’t want your builds to have access to these credentials; + 3. At the Username field, enter **ec2-user**; + 4. For the Private Key, select the **Enter directly** radio button. Open the .pem file that you downloaded during the workshop setup in a text editor and copy the contents of the file to the Key field including the *BEGIN RSA PRIVATE KEY* and *END RSA PRIVATE KEY* fields; + 5. Click on the **Add** button. +8. Select the **ec2-user** option from the Credentials dropdown; +9. Given that Spot instances will have a random SSH host fingerprint, select the **Non verifying Verification Strategy** option from the Host Key Verification Strategy dropdown; +10. Mark the **Private IP** checkbox to ensure that your Jenkins Master will always communicate with the Agents via their internal VPC IP addresses (in real-world scenarios, your build agents would likely not be publicly addressable); +11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; +12. Set the **Max Idle Minutes Before Scaledown** to **5**. There's no need to keep a build agent running for too much longer than it's required; +13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); +14. Change the Maximum Cluster Size from **1** to **5** (so that you can test fleet scale-out); +15. Finally, click on the **Save** button. + +For now, no instances are going to be launched as there are no pending jobs to run. So, let's configure an existing Jenkins job to use Spot instances. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md deleted file mode 100644 index f9cd4ed3..00000000 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/lab4.md +++ /dev/null @@ -1,155 +0,0 @@ -+++ -title = "Setup with CloudFormation" -weight = 210 -+++ -You’ve now got a scalable solution using nothing but Spot instances for your CICD systems, your build agents and your test environments – however, you still have some inefficiencies with this setup: - -* Your Jenkins master utilizes a relatively low percentage of the CPU resources on the instance types that Jenkins is running on; and -* You still have at least one Jenkins build agent running at all times; - -These inefficiencies can be addressed by moving your solution to a container environment that continues to utilize Spot instances. This lab will see you configure the ECS cluster resources that were created by the initial CloudFormation template and migrate your Jenkins master and agents to this cluster. - -## OBTAIN THE RELEVANT INFORMATION FOR CLOUDFORMATION FOR THIS LAB -As was the case with Lab 3, you will need a value from the Outputs tab of the CloudFormation stack that you deployed in the Workshop Preparation - this time, make a note of what the **JenkinsIAMRoleARN** is. -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#)); -2. Click on the checkbox associated with the **SpotCICDWorkshop** stack; -3. From the Outputs tab of the SpotCICDWorkshop stack in the CloudFormation console, make note of value associated with the **JenkinsIAMUserARN** key. -{{% /expand%}} - -## INSPECT THE ECSLAUNCHTEMPLATE LAUNCH TEMPLATE -A couple of weeks prior to the 2018 re:Invent conference, AWS launched a new feature for Auto Scaling Groups - the capability to provision Auto Scaling Groups with a mixture of different instance types and different purchasing models. The ECS cluster that you provision in this lab utilises this feature, specifically launching an on-demand instance and a spot instance in the Auto Scaling Group that you create in the next section. - -By deploying the Auto Scaling Group in this manner, you can configure a placement constraint on the Jenkins server container in order to have it preferentially run on the on-demand instance and eliminating the drawbacks of having the container running on a spot instance. Setting up a placement constraint in an ECS Task Definition is relatively straightforward, but having the EC2 instance configure a custom attribute within ECS to indicate whether it is a spot instance or not is a little more complex. By inspecting the **ECSLaunchTemplate** Launch Template, you can see how this is achieved. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and select the **Launch Templates** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#LaunchTemplates:sort=launchTemplateId)); -2. Select the launch template with the Launch Template Name of **ECSLaunchTemplate**. Toward the bottom of the lower pane, expand the **Advanced details** section and click on the **View user data** link; -3. Inspect the User Data defined in the Launch Template: - 1. First, the Instance ID needs is obtained by polling the instance metadata; - 2. Once the Instance ID has been obtained, the user data script issues an EC2 describe-instances call to obtain the Instance Lifecycle. If the instance that this call refers to is a Spot instance, the Instance Lifecycle will be **spot**, otherwise it will be **null**; - 3. Finally, the ECS\_INSTANCE\_ATTRIBUTES configuration directive is added to the /etc/ecs/ecs.config file, defining the **lifecycle** custom attribute. When the user data has been processed, the ECS agent will start and register this custom variable with the ECS service. -{{% /expand%}} - -## PROVISION AN AUTO SCALING GROUP FOR YOUR ECS CLUSTER -While the CloudFormation template that was deployed during the Workshop Preparation has already defined the ECS Cluster resources that you'll utilize in this workshop, you'll need to create a new Auto Scaling Group to provide the compute resources to the ECS Cluster definition. While you could provision the entire Auto Scaling Group to use EC2 Spot Instances, this walkthrough will guide you through how to configure the Auto Scaling Group to use multiple instance types and purchasing options - a feature that was launched just over a week prior to re:Invent 2018. - -1. Go to the **EC2** console and click on the **Auto Scaling groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/autoscaling/home?region=eu-west-1#AutoScalingGroups:)); -2. Click on the **Create Auto Scaling group** button; -3. The recently released capability to deploy EC2 instances using multiple instance types and purchase options requires the use of **Launch Templates**, so select this as the Auto Scaling Group parameters that you want to use. Select the **ECSLaunchTemplate** launch template (which was created by the CloudFormation template deployed during the Workshop Preparation), and then click on the **Next Step** button. -4. At the Configure Auto Scaling group details step of the wizard: - 1. Give the Group a name of **Spot CICD Workshop ECS Auto Scaling Group**; - 2. Change the Fleet Composition to **Combine purchase options and instances** - Selecting this option should reveal additional choices. The Launch Template selected previously does not specify what purchasing model should be used for these instances and the default option will see on-demand instances launched; - 3. At the Instance Types section, add **t3.large**, **t2.medium** and **t2.large** instance types to the Auto Scaling Group configuration; - 4. Remove the tick from the **Instances Distribution** checkbox so that you can explore the additional configuration options; - 5. As you don't have a specific price objective to meet for your ECS cluster, leave the Maximum Spot Price using default bidding. Also leave the Spot Allocation Strategy so that instances are diversified across the 2 lowest priced instances types per Availability Zone (this will typically ensure that the medium-sized instances will almost always be used when instances are launched, but the large instances are able to be used if something unexpected happens to the availability or market price of the medium instances); - 6. While you could run the entire Auto Scaling Group using Spot instances, it might be more desirable to have a portion of the Auto Scaling Group running on-demand instances, to provide a little more assurance that the container running your Jenkins server will always be running. In order to do this, set the Optional On-Demand Base such that you designate the first **1** instance as On-Demand, and then change the On-Demand Percentage Above Base to be **0%** On-Demand and 100% Spot; - 7. Change the Group size to start with **2** instances; - 8. Select the **Amazon EC2 Spot CICD Workshop VPC** from the Network dropdown; - 9. Within the Subnet field, click on and select each of the three subnets that start with **Amazon EC2 Spot CICD Workshop Public Subnet**; - 10. Click on the **Next: Configure scaling policies** button; -5. At the Configure scaling policies step of the wizard, ensure that the **Keep this group at its initial size** option. In a production environment, it's likely that you will want to configure scaling policies to ensure that your compute resources grow in line with the number of containers deployed to the cluster, but a group with no scaling policy should be sufficient for this workshop. Click on the **Next: Configure Notifications** button; -6. You will not be setting up any notifications at the Configure Notifications step of the wizard, so simply click on the **Next: Configure Tags* button; -7. At the Configure Tags step of the wizard, add a tag with a Key of **Name** and a Value of **Spot CICD Workshop ECS Instance**, then click on the **Review** button; -8. At the Review step of the wizard, click on the **Create Auto Scaling Group** button. Once the Auto Scaling Group has been launched, click on the **Close** button. - -## MODIFY PERMISSIONS ON THE EFS FILE SYSTEM -You will be deploying the official Jenkins docker image to your ECS cluster which differs slightly from the RPM bundles typically deployed to Amazon Linux systems. One of the differences is that the docker image will ALWAYS run Jenkins using a Linux uid of 1000, as opposed to the incremental uid used in RPM deployments. Because this uid differs, we'll need to ensure that the EFS file system is configured with the correct permissions to allow our ECS Jenkins Master to read the persisted data. - -To do this, stop the Jenkins service on the Spot instance that you crated in the previous lab and set the file system permission of the root of your EFS file system so that the user with uid of 1000, and the group with a gid of 1000 is the owner of all file system objects. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and select the **Instances** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#Instances:sort=instanceId)); -2. Select the instance with the Name tag of **Jenkins Master (Spot)* and make a note of it's current IPv4 Pubic IP. -3. Establish an SSH session to this IP address (For instructions on how to establish an SSH connection to your EC2 instance, please refer to [this link]( https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstances.html?icmpid=docs_ec2_console)) - you'll need to use the EC2 Key Pair that you generated during the Workshop Preparation to establish connectivity. -4. Stop the Jenkins service by entering the following command: - - ```bash -sudo service jenkins stop -``` -5. The content of JENKINS_HOME is stored under /var/lib/jenkins – execute the following command to set the permissions appropriately (note that this will take a couple of minutes - while it's progressing, commence the next section of this lab): - - ```bash -sudo chown -R 1000:1000 /var/lib/jenkins -``` -{{% /expand%}} - -## MODIFY THE JENKINS MASTER APPLICATION LOAD BALANCER SO THAT REQUESTS ARE FORWARDED TO THE ECS TARGET GROUP -In order to operate properly with ECS Service Discovery (which is used later in this lab), the CloudFormation template that you deployed in the Workshop Preparation deployed two Application Load Balancer Target Groups - one for instance targets (EC2) and the other for IP address targets (ECS). Given that the Jenkins installation on your Spot instance is now stopped, now is an opportune time to change the configuration of the listener for the JenkinsMasterALB Application Load Balancer to use the **JenkinsMasterECSTargetGroup** target group. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and select the **Load Balancers** option from the left pane (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#LoadBalancers:sort=loadBalancerName)); -2. If it is not selected, select the load balancer named **JenkinsMasterALB** and click on the **Listeners** tab in the bottom pane. Then, click on the **View/edit rules** link; -3. Click on the **Reorder Rules** button (signified by an up arrow next to a down arrow), select rule **2** and click on the **Up** button to increase its priority. The rule that forwards content to the **JenkinsMasterECSTargetGroup** should now be at the top of the list - if this is the case, click on the **Save** button. -{{% /expand%}} - -## MODIFY THE JENKINSMASTER ECS SERVICE TO START A JENKINS MASTER TASK -An ECS Service called **SpotCICDWorkshop-ECSServiceJenkinsMaster** was provisioned the CloudFormation stack launched during the Workshop Preparation phase, along with the EFS volume that you're using to persist the contents of JENKINS\_HOME. Up until Lab 3, the contents of this EFS volume was empty, which would have prevented a Jenkins Task from starting successfully until the contents of JENKINS\_HOME resided on the EFS volume, and no other Jenkins installation had a lock file in place on the volume. Therefore during the CloudFormation deployment, the ECS Service definition was configured to have the desired number of tasks running set to **0**. Update the service so that **1** copy of the task is running. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **ECS** console (or [click here](https://eu-west-1.console.aws.amazon.com/ecs/home?region=eu-west-1#/clusters)) and click on the **SpotCICDWorkshopECSCluster** link; -2. Click on the checkbox of the service with a Service Name that begins with **SpotCICDWorkshop-ECSServiceJenkinsMaster**, then click on the **Update** button; -3. Change the Number of tasks from **0** to **1**, then click on the **Next Step** button; -4. At the Configure network screen, click on the **Next Step** button. Repeat the same action at the Set Auto Scaling (optional) screen. Finally at the Review screen, click on the **Update Service** button. Once the service has been updated, click on the **View Service** button. -{{% /expand%}} - -## VERIFY THE STATUS OF YOUR ALB TARGET GROUP, VERIFY ACCESS TO JENKINS AND RUN A TEST BUILD -Within a few moments of updating the service definition, a new task and container will be launched on one of the ECS cluster nodes (specifically, the on-demand instance within the cluster). The ECS service is configured to attach the running container to the JenkinsMasterECSTargetGroup target group, which is now the target group being used by the Application Load Balancer that you've been using to access Jenkins. Verify that the new container has registered with the ALB, then access Jenkins through the ALB and run a test build of the Apache PDFBox project. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Target Groups** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=eu-west-1#TargetGroups)); -2. Select the checkbox corresponding to the target group named **JenkinsMasterECSTargetGroup**, then select the **Targets** tab. You should see one of your Spot CICD Workshop ECS Instance instances with an **initial** status (it generally takes a couple of minutes for Jenkins to completely start up and for the health check to return a **healthy** status); -3. Refresh the list of targets every minute or so and when the registered target is shown with a healthy status, you should then be able to reload Jenkins through the ALB. Once you can do so, initiate a build of the **Apache PDFBox** project to ensure that everything is working as expected. -{{% /expand%}} - -## CANCEL THE EC2 SPOT FLEET REQUEST FOR YOUR SELF-HEALING JENKINS SERVER -As you now have Jenkins running from a container, you no longer need the Spot Fleet that you created in Lab 3 - cancel this Spot Fleet request and terminate the instances belonging to the fleet. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Select the checkbox corresponding to the Spot Fleet request that was created in the previous lab (this should be the Spot Fleet request that has a Capacity of **1 of 1** and a Max price of **$0.0456**); -3. Click the Actions dropdown at the top of the screen and select the **Cancel spot request** option. At the confirmation dialogue, ensure that the **Terminate instances** checkbox is selected and click on the **Confirm** button. -{{% /expand%}} - -## RECONFIGURE JENKINS TO USE BUILD AGENTS RUNNING IN CONTAINERS -When you launched build agents using Spot instances, you took advantage of the fact that you could access those agents via SSH. However with build agent containers, you won't be exposing an SSH interface outside of your container. Instead, your build agent containers will initiate a connection to the Jenkins container via the Java Network Launch Protocol (JNLP), via an interface that is addressed using ECS Service Discovery. - -1. Back at the Jenkins home page, click on the **Manage Jenkins** link on the left side menu again, and then the **Configure System** link; -2. Scroll down to the Cloud section and under the Spot Fleet Configuration section, click on the **Delete Cloud** button (the spot build agents will no longer be required now that you’re starting to use resources on your ECS cluster); -3. Next, click on the **Add a new cloud** button followed by the **Amazon EC2 Container Service Cloud** option; -4. In the Amazon EC2 Container Service Cloud section: - 1. Set **SpotCICDWorkshopECSAgents** as the Name; - 2. In the Amazon ECS Credentials dropdown, select the same IAM Access key that you used when configuring your spot build agents; - 3. Select the **eu-west-1** region from the Amazon ECS Region Name dropdown; - 4. Select the ECS Cluster that has the **SpotCICDWorkshopECSCluster** suffix from the ECS Cluster dropdown; - 5. Click on the **Advanced** button, then set the Alternative Jenkins URL to **http://master.jenkins.local:8080** - this is the address (managed through ECS Service Discovery) that the build agents will use to access Jenkins via the internal VPC network; - 6. Click on the **Add** button that appears just to the right of the ECS slave templates label; - 7. Enter **ecs-agents** into the Label field; - 8. Type **ECSBuildAgent** into the Template Name field; - 9. Enter **cloudbees/jnlp-slave-with-java-build-tools** into the Docker Image field - this is the Docker image that will be used when launching your build agent containers; - 10. Set the Hard Memory Reservation to **1536** and the CPU units to **512*; - 11. Click on the **Advanced** button; - 12. In the Task Role ARN field, enter the value specified by the **JenkinsIAMRoleARN** key obtained at the beginning of this lab (it should be of the format arn:aws:iam::account\_number:role/SpotCICDWorkshop-IAMRoleJenkins-random_chars). -5. Click on the **Save** button at the bottom of the page. - -## CANCEL THE EC2 SPOT FLEET REQUEST FOR YOUR BUILD AGENTS -As you now have build agents configured to run from containers, you no longer need the Spot Fleet that you created in Lab 1 - cancel this Spot Fleet request and terminate the instances belonging to the fleet. - -{{%expand "Click to reveal detailed instructions" %}} -1. Go to the **EC2** console and click on the **Spot Requests** option from the left frame (or [click here](https://eu-west-1.console.aws.amazon.com/ec2sp/v1/spot/home?region=eu-west-1#)); -2. Select the checkbox corresponding to what should be the sole remaining Spot Fleet request; -3. Click the Actions dropdown at the top of the screen and select the **Cancel spot request** option. At the confirmation dialogue, ensure that the **Terminate instances** checkbox is selected and click on the **Confirm** button. -{{% /expand%}} - -## RECONFIGURE THE APACHE PDFBOX BUILD PROJECT TO ONLY USE YOUR ECS BUILD AGENTS, AND RUN A TEST BUILD -As things stand now, your projects in Jenkins won't be able to be built - you've removed all of the EC2 Spot build agents and have statically defined a number of build jobs to use these agents. Recall that when you set up the EC2 Spot build agents, you assigned the **spot-agents** label to your a number of the build projects to force them to be built on your Spot instances? For this lab, reconfigure the **Apache PDFBox** build project to use the **ecs-agents** label and then kick off a test build of this project. - -{{%expand "Click to reveal detailed instructions" %}} -1. Within Jenkins, click on the Jenkins logo in the top left corner of the site – this will take you back to the Jenkins home page. Click on the **Apache PDFBox** build project link, then click on the **Configure** link in the left pane of the page; -2. Under the General section, change the Label Expression so that it says **ecs-agents**; -3. At the bottom of the screen, click on the **Save** button; -4. Finally, click on the **Build Now** link on the left side of the screen to initiate a build of this project. Click on the Jenkins logo at the top left corner of the site to go back to the main screen and see how your build progresses on the ECS build agent. One final thing to note is that when running in a container, the build agent will always show up with a suspended status - due to the ephemeral way in which containers are treated by the plugin, as soon as all of the build executors have been consumed in the container, the plugin will mark the agent as suspended so that new agents are deployed in new containers for subsequent build jobs. -{{% /expand%}} - -## PROCEED TO WORKSHOP CLEANUP -Once your Jenkins infrastructure is completely running in your ECS cluster, you've completed all of the labs in this workshop. Congratulations! You may now proceed with the [Workshop Cleanup](/amazon-ec2-spot-cicd-workshop/clea.html). diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/prep.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/prep.md new file mode 100644 index 00000000..4b8b55c2 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/prep.md @@ -0,0 +1,46 @@ ++++ +title = "Setup with CloudFormation" +weight = 10 ++++ +Before we start presenting content on how to configure Jenkins with EC2 Spot instances in ECS, you'll need to prepare the AWS account that you've come to this workshop with. Specifically, this workshop involves working with a large number of AWS resources as well as a deployment of Jenkins that if manually configured, would leave you little time to discover how to use EC2 Spot instances in the most effective manner. To address that, you will deploy an Amazon CloudFormation template that does a lot of the heavy lifting of provisioning these resources for you. + +## Launchh the CloudFormation template +So that you can concentrate on the aspects of this workshop that directly relate to Amazon EC2 Spot instances, there is a CloudFormation template that will deploy the base AWS infrastructure needed for all of the labs within the workshop - saving you from having to create things like ECS Cluster, VPCs, Security Groups, IAM policies and so forth. + +Download and deploy the CloudFormation template: +[amazon-ec2-spot-cicd-workshop-asg.yaml](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml) + +Be sure to give it a stack name of **SpotCICDWorkshopECS** and ensure that you supply appropriate parameters when prompted. + +1. Go to the **CloudFormation** console (or [click here](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1)); +2. Click on the **Create Stack** button towards the top of the console; +3. At the Select Template screen, select the **Upload a template file** radio button and choose the CloudFormation template you downloaded before, then click on the **Next** button; +4. At the Specify Details screen, enter in **SpotCICDWorkshopECS** as the Stack name. Under the Parameters section: + 1. Identify what your current public IP address is by going to https://www.google.com.au/search?q=what%27s+my+ip+address. Enter the first three octets of this IP address into the **CurrentIP** parameter field and then add the **.0/24** suffix. For example if your IP address was 54.240.193.193, you would enter 54.240.193.0/24 into the CurrentIP field; + 2. Enter in a password that you would like to use for the administrator account within the Jenkins server that will be launched by the CloudFormation template; +5. Click on the **Next** button; +6. At the Options screen, there is no need to make any changes to the default options – simply click on the **Next** button; +7. Finally at the Review screen, verify your settings, mark the **I acknowledge that AWS CloudFormation might create IAM resources with custom names** checkbox and then click on the **Create** button. Wait for the stack to complete provisioning, which should take a couple of minutes. + +The stack should take around five minutes to deploy. + +{{% notice note %}} +It's good security practice to ensure that the web and SSH services being used in this workshop are not accessible to everyone on the Internet. In most cases, limiting access to the /24 CIDR block that you IP address is in provides a reasonable level of access control - but this may still be too restrictive in some corporate IT environments. If you have trouble accessing resources, additional instructions within this lab guide will guide you through what settings need to be manually changed. +{{% /notice %}} + +{{% notice note %}} +The CloudFormation template creates a new Launch Template to install and bootstrap Jenkins. However, you can continue using any existing Launch Template that you might already have. +{{% /notice %}} + +## Setting Up environment variables +You need to set up the following environment variables that you'll use in the workshop, to do so, run the following commands: + +```bash +export PRIVATE_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPrivateSubnets'].OutputValue" --output text); +export PUBLIC_SUBNETS=$(aws cloudformation describe-stacks --stack-name SpotCICDWorkshop --query "Stacks[0].Outputs[?OutputKey=='JenkinsVPCPublicSubnets'].OutputValue" --output text); +export LAUNCH_TEMPLATE_ID=$(aws ec2 describe-launch-templates --filters Name=launch-template-name,Values=JenkinsBuildAgentLaunchTemplate | jq -r '.LaunchTemplates[0].LaunchTemplateId'); +``` + +{{% notice note %}} +If for some reason the environment variables are cleared, you can run the previous commands again without any problem. +{{% /notice %}} diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_experiment.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_experiment.md new file mode 100644 index 00000000..4ec06939 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_experiment.md @@ -0,0 +1,105 @@ +--- +title: "Creating the Spot Interruption Experiment" +weight: 145 +--- + +In this section, you're going to start creating the experiment to [trigger the interruption of Amazon EC2 Spot Instances using AWS Fault Injection Simulator (FIS)](https://aws.amazon.com/blogs/compute/implementing-interruption-tolerance-in-amazon-ec2-spot-with-aws-fault-injection-simulator/). When using Spot instances, you need to be prepared to be interrupted. With FIS, you can test the resiliency of your workload and validate that your application is reacting to the interruption notices that EC2 sends before terminating your instances. You can target individual Spot instances or a subset of instances from an Auto Scaling group. + +#### What do you need to get started? + +Before you start launching Spot interruptions with FIS, you need to create an experiment template. Here is where you define which resources you want to interrupt (targets), and when you want to interrupt the instance. + +You're going to use the following CloudFormation template which creates the IAM role (`FISSpotRole`) with the minimum permissions FIS needs to interrupt an instance, and the experiment template (`FISExperimentTemplate`) you're going to use to trigger a Spot interruption: + +``` +AWSTemplateFormatVersion: 2010-09-09 +Description: FIS for Spot Instances +Parameters: + InstancesToInterrupt: + Description: Number of instances to interrupt + Default: 3 + Type: Number + + DurationBeforeInterruption: + Description: Number of minutes before the interruption + Default: 2 + Type: Number + +Resources: + + FISSpotRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: [fis.amazonaws.com] + Action: ["sts:AssumeRole"] + Path: / + Policies: + - PolicyName: root + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: 'ec2:DescribeInstances' + Resource: '*' + - Effect: Allow + Action: 'ec2:SendSpotInstanceInterruptions' + Resource: 'arn:aws:ec2:*:*:instance/*' + + FISExperimentTemplate: + Type: AWS::FIS::ExperimentTemplate + Properties: + Description: "Interrupt multiple random instances" + Targets: + SpotIntances: + ResourceTags: + Name: "Jenkins Master (Spot)" + Filters: + - Path: State.Name + Values: + - running + ResourceType: aws:ec2:spot-instance + SelectionMode: !Join ["", ["COUNT(", !Ref InstancesToInterrupt, ")"]] + Actions: + interrupt: + ActionId: "aws:ec2:send-spot-instance-interruptions" + Description: "Interrupt multiple Spot instances" + Parameters: + durationBeforeInterruption: !Join ["", ["PT", !Ref DurationBeforeInterruption, "M"]] + Targets: + SpotInstances: SpotIntances + StopConditions: + - Source: none + RoleArn: !GetAtt FISSpotRole.Arn + Tags: + Name: "fis-spot-interruption" + +Outputs: + FISExperimentID: + Value: !GetAtt FISExperimentTemplate.Id +``` + +Here are some important notes about the template: + +* You can configure how many instances you want to interrupt with the `InstancesToInterrupt` parameter. In the template it's defined that it's going to interrupt **three** instances. +* You can also configure how much time you want the experiment to run with the `DurationBeforeInterruption` parameter. By default, it's going to take two minutes. This means that as soon as you launch the experiment, the instance is going to receive the two-minute notification Spot interruption warning. +* The most important section is the `Targets` from the experiment template. This is where you tell FIS which instances to interrupt. In this case, we're simply going to say that we want to interrupt a instance that's `running` and has a `Name` tag with value `Jenkins Master (Spot)`. +* Notice that instances are going to be **chosen randomly** + +#### Create the EC2 Spot Interruption Experiment with FIS + +Let's continue by creating the Spot interruption experiment template using Cloudformation. You can view the CloudFormation template (**fisspotinterruption.yaml**) at GitHub [here](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml). To download it, you can run the following command: + +``` +wget https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml +``` + +Now, simply run the following commands to create the FIS experiment: + +``` +aws cloudformation create-stack --stack-name fis-spot-interruption --template-body file://fisspotinterruption.yaml --capabilities CAPABILITY_NAMED_IAM +aws cloudformation wait stack-create-complete --stack-name fis-spot-interruption +``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_fis.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_fis.md new file mode 100644 index 00000000..d0db6bb2 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/spot_interruption_fis.md @@ -0,0 +1,54 @@ +--- +title: "Interrupting a Spot Instance" +weight: 150 +--- + +In this section, you're going to launch a Spot Interruption using FIS and then verify that the capacity has been replenished and Jenkins is still working. This will help you to confirm the low impact of your workloads when implementing Spot effectively. + +#### Launch the Spot Interruption Experiment +After creating the experiment template in FIS, you can start a new experiment to interrupt one Spot instance. Run the following command: + +``` +FIS_EXP_TEMP_ID=$(aws cloudformation describe-stacks --stack-name fis-spot-interruption --query "Stacks[0].Outputs[?OutputKey=='FISExperimentID'].OutputValue" --output text) +FIS_EXP_ID=$(aws fis start-experiment --experiment-template-id $FIS_EXP_TEMP_ID --no-cli-pager --query "experiment.id" --output text) +``` + +Wait around 30 seconds, and you should see that the experiment completes. Run the following command to confirm: + +``` +aws fis get-experiment --id $FIS_EXP_ID --no-cli-pager +``` + +At this point, FIS has triggered a Spot interruption notice, and in two minutes the instance will be terminated. + +Go to CloudWatch Logs group `/aws/events/spotinterruptions` to confirm that instance received the termination notice. + +You should see a log message like this one: + +![SpotInterruptionLog](/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png) + +#### Verify the actions taken by Jenkins Auto Scaling group + +You are running a Jenkins instance launched by an Auto Scaling group that will launch a new instance if the desired capacity is not compliant. + +You can run the following command to see the list of instances with the date and time when they were launched. + +``` +aws ec2 describe-instances --filters\ + Name='tag:Name',Values='Jenkins Master (Spot)'\ + Name='instance-state-name',Values='running'\ + | jq '.Reservations[].Instances[] | "Instance with ID:\(.InstanceId) launched at \(.LaunchTime)"' +``` + +You should see a list of instances with the date and time when they were launched. You'll also see the new instance launched after the interruption. + +```output +"Instance with ID:i-04c6ced30794e965b launched at 2022-04-06T14:02:49+00:00" +"Instance with ID:i-0136152e14053af81 launched at 2022-04-06T14:11:25+00:00" +``` + +#### Verify that Jenkins is still working + +Login to the Jenkins server again and start running some jobs as [you did previously](/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.html#verify-that-the-new-spot-instance-is-running-the-jenkins-server). + +You should see that the EC2 Fleet plugin is still working and is launching Spot instances as build agents. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/test-build.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/test-build.md new file mode 100644 index 00000000..8df4dae5 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/test-build.md @@ -0,0 +1,22 @@ ++++ +title = "Configure jobs to use Spot" +weight = 120 ++++ +As alluded to in the previous section, you'll need to configure your build jobs so that they are executed on the build agents running in your Spot instances. In addition, you could configure jobs to execute concurrent builds if necessary - this will help you in testing the scale-out of your fleet. + +1. Go back to the Jenkins home screen and **repeat the following for each of the five Apache build projects** that are configured in your Jenkins instance: + 1. Click on the title of the build job and then click on the **Configure** link toward the left side of the screen; + 2. In the General section, click on the **Execute concurrent builds if necessary** checkbox and the **Restrict where this project can be run** checkbox. Next, enter **spot-agents** as the Label Expression (Note: if you select the auto-complete option instead of typing out the full label, Jenkins will add a space to the end of the label - be sure to remove any trailing spaces from the label before proceeding); + 3. Click on the **Save** button towards the bottom of the screen. + +## Test Spot Builds and Scale-out +Now it’s time to test out how Jenkins handles pushing builds to spot instances running build agents at scale. There are two things that you'll want to verify here; that your builds run successfully on the Spot instances, and that your ASG scales out when there are build jobs queued for more than a few minutes. + +1. Go Back to the Jenkins home page, click on the **Schedule a Build** icon (which looks like a play symbol) for each of the five Apache projects, starting from the **Apache PDFBox** project and working upward. This will queue up five build jobs, this will trigger the EC2 Fleet plugin to start launching build agents using Spot instances; +2. When any of the build jobs have been completed, click on the **Schedule a Build** icon corresponding to that job to re-add it back to the build queue - the intent here is to keep the build queue populated with a backlog of build jobs until your Auto Scaling group has scaled out and build jobs are executing on Spot instances; +3. After around four minutes, the **EC2 Fleet Status** reported to the left of the screen will increment the **target** count to 5, indicating that the plugin has requested a scale-out action from the plugin. After a few moments, five build instances will appear in the **Build Executor Status**, though these build agents will initially appear to be offline. Once the instances have had the chance to complete the launch and bootstrapping processes (which takes around two minutes), your Jenkins Master will deploy the build agent to them via SSH, and it will come online and process the next build jobs in the queue. Once you have concurrent builds being executed on the Spot instances, you can stop adding build jobs to the build queue; +4. After a period of around a five minutes, after your builds have completed, the Spot instances should be terminated by the plugin. + +{{% notice note %}} +If the build agents are keep showing as `offline`, make sure that you've configured correctly the SSH key pair in the credentials section when configuring the EC2 Fleet plugin. You can click on the agent to open the logs and see why it's not coming online. +{{% /notice %}} \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/tracking_spot_interruptions.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/tracking_spot_interruptions.md new file mode 100644 index 00000000..ab3ed2a6 --- /dev/null +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/tracking_spot_interruptions.md @@ -0,0 +1,36 @@ +--- +title: "Tracking Spot interruptions" +weight: 140 +--- +At some point in time, you might receive a Spot interruption notice when On-Demand needs the capacity back. When this happens, your Spot instance will be provided with a two-minute notice of termination and after that time lapses, the instance will be terminated. At this point, the Auto Scaling group will observe that there are no running instances in the group and because the desired capacity is one. Then, it will launch a replacement instance from the pool with more capacity available (this is why diversification across many capacity pool is a best practice). The new instance will be launched and bootstrapped in exactly the same manner as your original instance. + +Before we start testing the resiliency of Jenkins with Spot, let's configure CloudWatch Logs to log Spot interruptions. If a Jenkins job fails, we'll be able to check if the failure correlates to a Spot interruption or not. + +{{% notice note %}} +In most cases we don't really need to track the Spot interruptions as Jenkins jobs can be retried if it fails. However, when we're starting with running our Jenkins jobs on Spot instances tracking could be useful. Organizations can use this data to correlate possible job failures or prolonged execution times, in case Spot instances were interrupted during a job execution. +{{% /notice %}} + +#### Creating the CloudFormation Stack to Track EC2 Spot Interruptions + +We've created a CloudFormation template that includes all the resources you need to track EC2 Spot Interruptions. The stack creates the following: + +* An Event Rule for tracking EC2 Spot Interruption Warnings +* A CloudWatch Log group to log interruptions and instance details +* IAM Role to allow the event rule to log into CloudWatch Logs + +You can view the CloudFormation template (**cloudwatchlogs.yaml**) at GitHub [here](https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/track-spot-interruptions/cloudwatchlogs.yaml). To download it, you can run the following command: + +``` +wget https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/workshops/track-spot-interruptions/cloudwatchlogs.yaml +``` + +After downloading the CloudFormation template, run the following command in a terminal: + +``` +aws cloudformation create-stack --stack-name track-spot-interruption --template-body file://cloudwatchlogs.yaml --capabilities CAPABILITY_NAMED_IAM +aws cloudformation wait stack-create-complete --stack-name track-spot-interruption +``` + +You should see an event rule in the Amazon EventBridge console, like this: + +![Spot Interruption Event Rule](/images/tracking-spot/itn-event-rule.png) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml index 28fe1382..0e5be7ab 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-ecs.yaml @@ -3,10 +3,6 @@ AWSTemplateFormatVersion: "2010-09-09" Description: A CloudFormation template that will deploy all AWS resources that are required to run the Amazon EC2 Spot CI/CD Workshop. This template is provided as-is under a modified MIT license - please see https://github.com/aws-samples/amazon-ec2-spot-cicd-workshop/blob/master/LICENSE Parameters: - KeyPair: - Description: The Key Pair created earlier in the Preparation Lab - Type: AWS::EC2::KeyPair::KeyName - CurrentIP: Description: Your current IP address (used to limit access to SSH services on EC2 instances) Type: String @@ -514,7 +510,6 @@ Resources: Name: !Ref InstanceProfileECS ImageId: !Ref ECSAMI InstanceType: t3.medium - KeyName: !Ref KeyPair SecurityGroupIds: - !Ref SecurityGroupJenkins TagSpecifications: From f697b8db5162d679df8fb561cbaf87d1c9ccf466 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Tue, 6 Sep 2022 17:46:40 +0200 Subject: [PATCH 27/29] Added a note of why the ASG starts with zero instances in the CI/CD workshop --- content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md | 4 ++++ content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md index d4c5676e..7080f746 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/asg.md @@ -48,6 +48,10 @@ Finally, the configuration above, sets the `SpotAllocationStrategy` to `capacity Let’s now create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (*to avoid running instances when there's no need*), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. +{{% notice note %}} +We’re initialising the Auto Scaling group with zero instances to reduce costs. However, the impact is that you need to wait around five minutes to launch a new job while the new instance starts. If you want to launch pending jobs faster, you need set up number of minimum number of instances to the ones you'll need as a baseline for capacity of Jenkins agents. +{{% /notice %}} + Run the following command: ```bash diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md index d4c5676e..7080f746 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-ecs/asg.md @@ -48,6 +48,10 @@ Finally, the configuration above, sets the `SpotAllocationStrategy` to `capacity Let’s now create the Auto Scaling group. In this case the Auto Scaling group spans across 3 Availability Zones, and sets the min-size to 0 (*to avoid running instances when there's no need*), max-size to 2 and desired-capacity to 0. You'll override some of this configuration later through Jenkins. +{{% notice note %}} +We’re initialising the Auto Scaling group with zero instances to reduce costs. However, the impact is that you need to wait around five minutes to launch a new job while the new instance starts. If you want to launch pending jobs faster, you need set up number of minimum number of instances to the ones you'll need as a baseline for capacity of Jenkins agents. +{{% /notice %}} + Run the following command: ```bash From 8be11ac7696d3a989cb4f15e0bdfc08e7372e818 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 19 Oct 2022 13:13:23 +0200 Subject: [PATCH 28/29] Updated to the latest Jenkins version --- .../jenkins-asg/ec2-fleet-jenkins.md | 2 +- .../spot_interruption_experiment.md | 47 ++++++++++--------- .../jenkins-asg/spot_interruption_fis.md | 22 ++++----- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 4 +- .../container/Dockerfile | 2 +- .../container/docker-compose.yml | 2 +- .../container/plugins.txt | 3 +- .../fisspotinterruption.yaml | 4 +- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md index 68a2fd53..9e0eb3e1 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/ec2-fleet-jenkins.md @@ -38,7 +38,7 @@ When configuring the plugin, think about how you could force build processes to 11. Change the Label field to be **spot-agents** - you'll shortly configure a build job to run on slave instances featuring this label; 12. Set the **Max Idle Minutes Before Scaledown** to **5**. There's no need to keep a build agent running for too much longer than it's required; 13. Change the Minimum Cluster Size from **1** to **0** (so that it can scale-in to zero instances); -14. Change the Maximum Cluster Size from **1** to **5** (so that you can test fleet scale-out); +14. Change the Maximum Cluster Size from **1** to **3** (so that you can test fleet scale-out); 15. Finally, click on the **Save** button. For now, no instances are going to be launched as there are no pending jobs to run. So, let's configure an existing Jenkins job to use Spot instances. \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md index 4ec06939..48c393cb 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_experiment.md @@ -12,12 +12,13 @@ Before you start launching Spot interruptions with FIS, you need to create an ex You're going to use the following CloudFormation template which creates the IAM role (`FISSpotRole`) with the minimum permissions FIS needs to interrupt an instance, and the experiment template (`FISExperimentTemplate`) you're going to use to trigger a Spot interruption: ``` +--- AWSTemplateFormatVersion: 2010-09-09 Description: FIS for Spot Instances Parameters: InstancesToInterrupt: Description: Number of instances to interrupt - Default: 3 + Default: 1 Type: Number DurationBeforeInterruption: @@ -26,16 +27,15 @@ Parameters: Type: Number Resources: - FISSpotRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - - Effect: Allow - Principal: - Service: [fis.amazonaws.com] - Action: ["sts:AssumeRole"] + - Effect: Allow + Principal: + Service: [fis.amazonaws.com] + Action: ["sts:AssumeRole"] Path: / Policies: - PolicyName: root @@ -43,38 +43,39 @@ Resources: Version: "2012-10-17" Statement: - Effect: Allow - Action: 'ec2:DescribeInstances' - Resource: '*' + Action: "ec2:DescribeInstances" + Resource: "*" - Effect: Allow - Action: 'ec2:SendSpotInstanceInterruptions' - Resource: 'arn:aws:ec2:*:*:instance/*' + Action: "ec2:SendSpotInstanceInterruptions" + Resource: "arn:aws:ec2:*:*:instance/*" FISExperimentTemplate: Type: AWS::FIS::ExperimentTemplate - Properties: + Properties: Description: "Interrupt multiple random instances" - Targets: + Targets: SpotIntances: - ResourceTags: - Name: "Jenkins Master (Spot)" + ResourceTags: + Name: "jenkins-build-agent" Filters: - Path: State.Name - Values: - - running + Values: + - running ResourceType: aws:ec2:spot-instance SelectionMode: !Join ["", ["COUNT(", !Ref InstancesToInterrupt, ")"]] - Actions: + Actions: interrupt: ActionId: "aws:ec2:send-spot-instance-interruptions" Description: "Interrupt multiple Spot instances" - Parameters: - durationBeforeInterruption: !Join ["", ["PT", !Ref DurationBeforeInterruption, "M"]] - Targets: + Parameters: + durationBeforeInterruption: + !Join ["", ["PT", !Ref DurationBeforeInterruption, "M"]] + Targets: SpotInstances: SpotIntances StopConditions: - Source: none RoleArn: !GetAtt FISSpotRole.Arn - Tags: + Tags: Name: "fis-spot-interruption" Outputs: @@ -100,6 +101,6 @@ wget https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/worksho Now, simply run the following commands to create the FIS experiment: ``` -aws cloudformation create-stack --stack-name fis-spot-interruption --template-body file://fisspotinterruption.yaml --capabilities CAPABILITY_NAMED_IAM -aws cloudformation wait stack-create-complete --stack-name fis-spot-interruption +aws cloudformation create-stack --stack-name fis-spot-jenkins-interruption --template-body file://fisspotinterruption.yaml --capabilities CAPABILITY_NAMED_IAM +aws cloudformation wait stack-create-complete --stack-name fis-spot-jenkins-interruption ``` \ No newline at end of file diff --git a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md index d0db6bb2..214ff69a 100644 --- a/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md +++ b/content/amazon-ec2-spot-cicd-workshop/jenkins-asg/spot_interruption_fis.md @@ -5,12 +5,15 @@ weight: 150 In this section, you're going to launch a Spot Interruption using FIS and then verify that the capacity has been replenished and Jenkins is still working. This will help you to confirm the low impact of your workloads when implementing Spot effectively. -#### Launch the Spot Interruption Experiment +#### Configure a retry build in Jenkins jobs +Configure it (thanks to Naginator) and schedule several jobs. + +#### Launch the Spot interruption experiment After creating the experiment template in FIS, you can start a new experiment to interrupt one Spot instance. Run the following command: ``` -FIS_EXP_TEMP_ID=$(aws cloudformation describe-stacks --stack-name fis-spot-interruption --query "Stacks[0].Outputs[?OutputKey=='FISExperimentID'].OutputValue" --output text) -FIS_EXP_ID=$(aws fis start-experiment --experiment-template-id $FIS_EXP_TEMP_ID --no-cli-pager --query "experiment.id" --output text) +FIS_EXP_TEMP_ID=$(aws cloudformation describe-stacks --stack-name fis-spot-jenkins-interruption --query "Stacks[0].Outputs[?OutputKey=='FISExperimentID'].OutputValue" --output text); +FIS_EXP_ID=$(aws fis start-experiment --experiment-template-id $FIS_EXP_TEMP_ID --no-cli-pager --query "experiment.id" --output text); ``` Wait around 30 seconds, and you should see that the experiment completes. Run the following command to confirm: @@ -28,14 +31,13 @@ You should see a log message like this one: ![SpotInterruptionLog](/images/amazon-ec2-spot-cicd-workshop/spotinterruptionlog.png) #### Verify the actions taken by Jenkins Auto Scaling group - You are running a Jenkins instance launched by an Auto Scaling group that will launch a new instance if the desired capacity is not compliant. You can run the following command to see the list of instances with the date and time when they were launched. ``` aws ec2 describe-instances --filters\ - Name='tag:Name',Values='Jenkins Master (Spot)'\ + Name='tag:Name',Values='jenkins-build-agent'\ Name='instance-state-name',Values='running'\ | jq '.Reservations[].Instances[] | "Instance with ID:\(.InstanceId) launched at \(.LaunchTime)"' ``` @@ -43,12 +45,8 @@ aws ec2 describe-instances --filters\ You should see a list of instances with the date and time when they were launched. You'll also see the new instance launched after the interruption. ```output -"Instance with ID:i-04c6ced30794e965b launched at 2022-04-06T14:02:49+00:00" -"Instance with ID:i-0136152e14053af81 launched at 2022-04-06T14:11:25+00:00" +"Instance with ID:i-04c6ced30794e965b launched at 2022-09-06T14:02:49+00:00" +"Instance with ID:i-0136152e14053af81 launched at 2022-09-06T14:11:25+00:00" ``` -#### Verify that Jenkins is still working - -Login to the Jenkins server again and start running some jobs as [you did previously](/amazon-ec2-spot-cicd-workshop/jenkins-asg/test-persistence.html#verify-that-the-new-spot-instance-is-running-the-jenkins-server). - -You should see that the EC2 Fleet plugin is still working and is launching Spot instances as build agents. \ No newline at end of file +#### Verify that Jenkins jobs were retried diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index a2a97b8a..91592f68 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -472,7 +472,7 @@ Resources: - DeviceName: "/dev/xvda" Ebs: DeleteOnTermination: "true" - VolumeSize: 8 + VolumeSize: 50 VolumeType: gp2 IamInstanceProfile: Name: !Ref InstanceProfileJenkins @@ -529,7 +529,7 @@ Resources: - DeviceName: "/dev/xvda" Ebs: DeleteOnTermination: "true" - VolumeSize: 8 + VolumeSize: 50 VolumeType: gp2 IamInstanceProfile: Name: !Ref InstanceProfileJenkins diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile index bc0dfc86..f53d692e 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.346.3-lts-jdk11 +FROM jenkins/jenkins:2.361.2-lts-jdk11 ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false ENV CASC_JENKINS_CONFIG /usr/share/jenkins/ref/casc.yaml diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml index 8d77d8e7..c7dbe5c8 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.2' services: jenkins: - image: christianhxc/amazon-ec2-spot-cicd-workshop-jenkins:2.346.3 + image: christianhxc/amazon-ec2-spot-cicd-workshop-jenkins:2.361.2 privileged: true user: root build: . diff --git a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt index 11d46baf..397f1ea6 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt +++ b/workshops/amazon-ec2-spot-cicd-workshop/container/plugins.txt @@ -19,4 +19,5 @@ timestamper:1.18 workflow-aggregator:590.v6a_d052e5a_a_b_5 ws-cleanup:0.42 authorize-project:1.4.0 -ec2-fleet:2.5.1 \ No newline at end of file +ec2-fleet:2.5.1 +naginator:1.18.1 \ No newline at end of file diff --git a/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml index 3cdfca08..d26aa8bf 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/fisspotinterruption.yaml @@ -4,7 +4,7 @@ Description: FIS for Spot Instances Parameters: InstancesToInterrupt: Description: Number of instances to interrupt - Default: 1 + Default: 3 Type: Number DurationBeforeInterruption: @@ -62,7 +62,7 @@ Resources: - Source: none RoleArn: !GetAtt FISSpotRole.Arn Tags: - Name: "fis-spot-interruption" + Name: "fis-spot-jenkins-interruption" Outputs: FISExperimentID: From a9e96dd09eef8ba8a25fd9fb29a685ad39e6c102 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Thu, 10 Nov 2022 22:46:54 +0100 Subject: [PATCH 29/29] Fixed problems when installing the Jenkins agent --- .../amazon-ec2-spot-cicd-workshop-asg.yaml | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml index 91592f68..089150db 100644 --- a/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml +++ b/workshops/amazon-ec2-spot-cicd-workshop/amazon-ec2-spot-cicd-workshop-asg.yaml @@ -358,14 +358,14 @@ Resources: SourceSecurityGroupId: !Ref SecurityGroupJenkins VpcId: !Ref VPC - JenkinsMasterGroup: + JenkinsMasterGroup: Type: AWS::AutoScaling::AutoScalingGroup DependsOn: - - InstanceProfileJenkins - - SecurityGroupJenkins - - SubnetPublicA - - SubnetPublicB - - SubnetPublicC + - InstanceProfileJenkins + - SecurityGroupJenkins + - SubnetPublicA + - SubnetPublicB + - SubnetPublicC Properties: HealthCheckType: EC2 HealthCheckGracePeriod: 300 @@ -552,11 +552,10 @@ Resources: wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo # Update the release version in the Maven repository configuration for this mainline release of Amazon Linux sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo - # Install the Java 8 SDK, Git and Maven - yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel git apache-maven - # Set the default version of java to run out of the Java 8 SDK path (required by Jenkins) - update-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java - update-alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac + # Install Git + yum install git apache-maven -y + # Install the Java 11 SDK, Git and Maven + amazon-linux-extras install java-openjdk11 EFSJenkinsHomeVolume: Type: AWS::EFS::FileSystem