Add instance update and reactivation

Adds functionality to perform Bottlerocket version upgrades and
reactivates drained instances after successful upgrade.

[Issue: #9]

updater/aws.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/ssm"
 )
@@ -200,15 +201,15 @@ func (u *updater) sendCommand(instanceIDs []string, ssmCommand string) (string,
 	return commandID, nil
 }
 
-func (u *updater) checkSSMCommandOutput(commandID string, instanceIDs []string) ([]string, error) {
+func (u *updater) checkSSMCommandResult(commandID string, instanceIDs []string) ([]string, error) {
 	updateCandidates := make([]string, 0)
 	for _, v := range instanceIDs {
 		resp, err := u.ssm.GetCommandInvocation(&ssm.GetCommandInvocationInput{
 			CommandId:  aws.String(commandID),
 			InstanceId: aws.String(v),
 		})
 		if err != nil {
-			return nil, fmt.Errorf("failed to retreive command invocation output: %#v", err)
+			return nil, fmt.Errorf("failed to retrieve command invocation output: %#v", err)
 		}
 
 		type updateCheckResult struct {
@@ -234,3 +235,42 @@ func (u *updater) checkSSMCommandOutput(commandID string, instanceIDs []string)
 	}
 	return updateCandidates, nil
 }
+
+// getActiveVersion queries a Bottlerocket instance to determine the active version in use by the instance.
+// Takes an SSM Command ID and EC2ID as parameters and returns the active version in use.
+func (u *updater) getActiveVersion(commandID string, instanceID string) (string, error) {
+	resp, err := u.ssm.GetCommandInvocation(&ssm.GetCommandInvocationInput{
+		CommandId:  aws.String(commandID),
+		InstanceId: aws.String(instanceID),
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to retrieve command invocation output: %#v", err)
+	}
+
+	type version struct {
+		Version string `json:"version"`
+	}
+
+	type image struct {
+		Image version `json:"image"`
+	}
+
+	type partition struct {
+		ActivePartition image `json:"active_partition"`
+	}
+
+	var activeVersion partition
+	err = json.Unmarshal([]byte(aws.StringValue(resp.StandardOutputContent)), &activeVersion)
+	if err != nil {
+		log.Printf("failed to unmarshal command invocation output: %#v", err)
+	}
+	versionInUse := activeVersion.ActivePartition.Image.Version
+	return versionInUse, nil
+}
+
+// waitUntilOk takes an EC2 ID as a parameter and waits until the specified EC2 instance is in an Ok status.
+func (u *updater) waitUntilOk(ec2ID string) error {
+	return u.ec2.WaitUntilInstanceStatusOk(&ec2.DescribeInstanceStatusInput{
+		InstanceIds: []*string{aws.String(ec2ID)},
+	})
+}

updater/main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/ssm"
 )
@@ -22,6 +23,7 @@ type updater struct {
 	cluster string
 	ecs     *ecs.ECS
 	ssm     *ssm.SSM
+	ec2     *ec2.EC2
 }
 
 func main() {
@@ -50,6 +52,7 @@ func _main() error {
 		cluster: *flagCluster,
 		ecs:     ecs.New(sess, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)),
 		ssm:     ssm.New(sess, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)),
+		ec2:     ec2.New(sess, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)),
 	}
 
 	listedInstances, err := u.listContainerInstances()
@@ -69,7 +72,7 @@ func _main() error {
 
 	// Make slice of Bottlerocket instances to use with SendCommand and checkCommandOutput
 	instances := make([]string, 0)
-	for instance, _ := range ec2IDtoECSARN {
+	for instance := range ec2IDtoECSARN {
 		instances = append(instances, instance)
 	}
 
@@ -78,7 +81,7 @@ func _main() error {
 		return err
 	}
 
-	candidates, err := u.checkSSMCommandOutput(commandID, instances)
+	candidates, err := u.checkSSMCommandResult(commandID, instances)
 	if err != nil {
 		return err
 	}
@@ -96,6 +99,48 @@ func _main() error {
 			continue
 		}
 		log.Printf("Instance %s drained", ec2ID)
+
+		ec2IDAsSlice := []string{ec2ID}
+		if len(ec2IDAsSlice) != 0 {
+			_, err := u.sendCommand(ec2IDAsSlice, "apiclient update apply -r")
+			if err != nil {
+				log.Printf("%#v", err)
+				continue
+			}
+
+			err = u.waitUntilOk(ec2ID)
+			if err != nil {
+				log.Printf("instance %s failed to reboot: %#v", ec2ID, err)
+				continue
+			}
+
+			err = u.activateInstance(aws.String(containerInstance))
+			if err != nil {
+				log.Printf("%#v", err)
+				continue
+			}
+
+			updateStatus, err := u.sendCommand(ec2IDAsSlice, "apiclient update check")
+			if err != nil {
+				log.Printf("%#v", err)
+			}
+
+			updateResult, _ := u.checkSSMCommandResult(updateStatus, ec2IDAsSlice)
+			if err != nil {
+				log.Printf("%#v", err)
+			}
+			if len(updateResult) == 0 {
+				log.Printf("Update successfully applied")
+			}
+
+			updatedVersion, err := u.getActiveVersion(updateStatus, ec2ID)
+			if err != nil {
+				log.Printf("%#v", err)
+			}
+			if len(updatedVersion) != 0 {
+				log.Printf("Instance %s updated to Bottlerocket: %s", ec2ID, updatedVersion)
+			}
+		}
 	}
 	return nil
 }