Move to more robust backup manager

Author: jonstacks

.gitignore

@@ -1,2 +1,6 @@
 
 vendor/
+
+# Don't checking user's local environment file which will be used for storing
+# configuration unique to their enviornment.
+.env

Gopkg.lock

@@ -59,3 +59,9 @@ Finally, you can create the lambda function like so:
 ```
 make create-lambda-ebs-backup
 ```
+
+### Tagging Volumes for backup
+
+In order to backup a volume, you must opt in by setting a tag key:value pair on
+the volume. By default, this is `lambda-ebs-backup:true`. This tells the lambda
+function that we are to create a snapshot of the volume.

README.md

@@ -30,8 +30,11 @@ Resources:
           Statement:
           - Effect: Allow
             Action:
-            - "ec2:CreateSnapshot"
+            - "ec2:DescribeInstances"
+            - "ec2:DescribeImages"
             - "ec2:DescribeVolumes"
+            - "ec2:CreateImage"
+            - "ec2:CreateSnapshot"
             Resource: "*"
       RoleName: lambda-ebs-backup
 Outputs:

cloudformation/iam.yaml

@@ -2,13 +2,11 @@ package main
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/aws/aws-lambda-go/lambda"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/c2fo/lambda-ebs-backup/pkg/config"
+	"github.com/c2fo/lambda-ebs-backup/pkg/backup"
 )
 
 // HandleRequest handles the lambda request
@@ -22,40 +20,25 @@ func HandleRequest(ctx context.Context) error {
 	)
 
 	ec2Client := ec2.New(sess)
-
-	params := &ec2.DescribeVolumesInput{
-		Filters: []*ec2.Filter{
-			&ec2.Filter{
-				Name:   aws.String(fmt.Sprintf("tag:%s", config.BackupTagKey())),
-				Values: []*string{aws.String(config.BackupTagValue())},
-			},
-		},
-		MaxResults: aws.Int64(500),
+	opts, err := backup.NewManagerOptsFromConfig(ec2Client)
+	if err != nil {
+		return err
 	}
 
-	fmt.Printf("Searching for volumes with tag %s:%s\n", config.BackupTagKey(), config.BackupTagValue())
-	err := ec2Client.DescribeVolumesPages(params,
-		func(page *ec2.DescribeVolumesOutput, lastPage bool) bool {
-			for _, v := range page.Volumes {
-				fmt.Printf("Found %s\n", aws.StringValue(v.VolumeId))
-				snap, snapErr := ec2Client.CreateSnapshot(&ec2.CreateSnapshotInput{
-					VolumeId: v.VolumeId,
-				})
-				if snapErr != nil {
-					fmt.Printf("Err: %s\n", snapErr)
-				}
-				fmt.Printf("Created snapshot '%s' for volume '%s'\n",
-					aws.StringValue(snap.SnapshotId),
-					aws.StringValue(v.VolumeId),
-				)
-			}
-			return !lastPage
-		})
+	backupManager, err := backup.NewManager(opts)
+	if err != nil {
+		return err
+	}
 
+	err = backupManager.Search()
 	if err != nil {
 		return err
 	}
 
+	if err = backupManager.Backup(); err != nil {
+		return err
+	}
+
 	return nil
 }
 

cmd/lambda-ebs-backup/main.go

@@ -0,0 +1,313 @@
+package backup
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+	"text/template"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/c2fo/lambda-ebs-backup/pkg/config"
+	"github.com/c2fo/lambda-ebs-backup/pkg/utils"
+)
+
+// ManagerOpts are options to configure the backup manager
+type ManagerOpts struct {
+	Client *ec2.EC2
+
+	BackupTagKey   string
+	BackupTagValue string
+	ImageTagKey    string
+	ImageTagValue  string
+	ImageNameTag   string
+
+	DefaultImageNameTemplate *template.Template
+	DefaultMaxKeepImages     int
+
+	Verbose bool
+}
+
+// NewManagerOptsFromConfig creates and initializes a new set of options from
+// the config.
+func NewManagerOptsFromConfig(client *ec2.EC2) (*ManagerOpts, error) {
+	var err error
+	opts := &ManagerOpts{
+		Client:         client,
+		BackupTagKey:   config.BackupTagKey(),
+		BackupTagValue: config.BackupTagValue(),
+		ImageTagKey:    config.ImageTagKey(),
+		ImageTagValue:  config.ImageTagValue(),
+		ImageNameTag:   config.ImageNameTag(),
+		Verbose:        true,
+	}
+
+	opts.DefaultImageNameTemplate, err = template.New("default-image-name").Parse(config.DefaultImageNameFormat())
+	if err != nil {
+		return opts, err
+	}
+
+	opts.DefaultMaxKeepImages, err = config.DefaultMaxKeepImages()
+	return opts, err
+}
+
+// Manager manages backups/images of ec2 resources(volumes, instances, etc.)
+type Manager struct {
+	client *ec2.EC2
+
+	volumes   []*ec2.Volume
+	instances []*ec2.Instance
+
+	BackupTagKey     string
+	BackupTagValue   string
+	ImageTagKey      string
+	ImageTagValue    string
+	ImageNameTag     string
+	MaxKeepImagesTag string
+
+	DefaultImageNameTemplate *template.Template
+	DefaultMaxKeepImages     int
+
+	Verbose bool
+}
+
+// NewManager creates a new backup manager from the provided options
+func NewManager(opts *ManagerOpts) (*Manager, error) {
+	m := &Manager{
+		volumes:   make([]*ec2.Volume, 0),
+		instances: make([]*ec2.Instance, 0),
+	}
+	m.client = opts.Client
+	m.BackupTagKey = opts.BackupTagKey
+	m.BackupTagValue = opts.BackupTagValue
+	m.ImageTagKey = opts.ImageTagKey
+	m.ImageTagValue = opts.ImageTagValue
+	m.ImageNameTag = opts.ImageNameTag
+	m.Verbose = opts.Verbose
+	if opts.DefaultImageNameTemplate == nil {
+		return nil, fmt.Errorf("DefaultImageNameTemplate is a required field for ManagerOpts")
+	}
+
+	m.DefaultImageNameTemplate = opts.DefaultImageNameTemplate
+	m.DefaultMaxKeepImages = opts.DefaultMaxKeepImages
+	return m, nil
+}
+
+// Search searches for a volumes and instances to backup
+func (m *Manager) Search() error {
+	return m.all(
+		[]func() error{
+			m.searchVolumes,
+			m.searchInstances,
+		},
+	)
+}
+
+func (m *Manager) searchVolumes() error {
+	params := &ec2.DescribeVolumesInput{
+		Filters: []*ec2.Filter{
+			&ec2.Filter{
+				Name:   aws.String(fmt.Sprintf("tag:%s", m.BackupTagKey)),
+				Values: []*string{aws.String(m.BackupTagValue)},
+			},
+		},
+		MaxResults: aws.Int64(500),
+	}
+
+	m.logf("Searching for volumes with tag %s:%s\n", m.BackupTagKey, m.BackupTagValue)
+
+	return m.client.DescribeVolumesPages(params,
+		func(page *ec2.DescribeVolumesOutput, lastPage bool) bool {
+			for _, v := range page.Volumes {
+				m.volumes = append(m.volumes, v)
+			}
+			return !lastPage
+		})
+}
+
+func (m *Manager) searchInstances() error {
+	params := &ec2.DescribeInstancesInput{
+		Filters: []*ec2.Filter{
+			&ec2.Filter{
+				Name:   aws.String(fmt.Sprintf("tag:%s", m.ImageTagKey)),
+				Values: []*string{aws.String(m.ImageTagValue)},
+			},
+		},
+		MaxResults: aws.Int64(500),
+	}
+
+	m.logf("Searching for instances with tag %s:%s\n", m.ImageTagKey, m.ImageTagValue)
+
+	return m.client.DescribeInstancesPages(params,
+		func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
+			for _, r := range page.Reservations {
+				for _, i := range r.Instances {
+					tags := utils.TagSliceToMap(i.Tags)
+					m.logf(
+						"Found instance %s(%s) with matching tag\n",
+						aws.StringValue(i.InstanceId),
+						tags.GetDefault("Name", "<no value>"),
+					)
+					m.instances = append(m.instances, i)
+				}
+			}
+			return !lastPage
+		})
+}
+
+// Backup performs the necessary backups
+func (m *Manager) Backup() error {
+	return m.all(
+		[]func() error{
+			m.backupVolumes,
+			m.backupInstances,
+		},
+	)
+}
+
+func (m *Manager) backupVolumes() error {
+	var wg sync.WaitGroup
+	errorChan := make(chan error, 1)
+
+	for _, volume := range m.volumes {
+		wg.Add(1)
+		go func(v *ec2.Volume) {
+			defer wg.Done()
+			snap, err := m.client.CreateSnapshot(&ec2.CreateSnapshotInput{
+				VolumeId: v.VolumeId,
+			})
+			if err != nil {
+				m.logf("Error creating snapshot for volume '%s'\n", aws.StringValue(v.VolumeId))
+				errorChan <- err
+			} else {
+				m.logf("Created snapshot '%s' for volume '%s'\n",
+					aws.StringValue(snap.SnapshotId),
+					aws.StringValue(v.VolumeId),
+				)
+			}
+		}(volume)
+	}
+
+	wg.Wait()
+
+	select {
+	case err := <-errorChan:
+		return err
+	default:
+	}
+
+	return nil
+}
+
+func (m *Manager) backupInstances() error {
+	var wg sync.WaitGroup
+	errorChan := make(chan error, 1)
+
+	for _, instance := range m.instances {
+		wg.Add(1)
+		go func(i *ec2.Instance) {
+			defer wg.Done()
+			tags := utils.TagSliceToMap(i.Tags)
+			imageName, err := m.formatImageName(i)
+			if err != nil {
+				errorChan <- err
+				return
+			}
+
+			image, err := m.client.CreateImage(&ec2.CreateImageInput{
+				InstanceId: i.InstanceId,
+				Name:       aws.String(imageName),
+			})
+			if err != nil {
+				m.logf(
+					"Error creating image for instance %s(%s): %s\n",
+					aws.StringValue(i.InstanceId),
+					tags.GetDefault("Name", ""),
+					err,
+				)
+				errorChan <- err
+				return
+			}
+
+			m.logf("Created image '%s'(%s) for instance '%s'(%s)\n",
+				aws.StringValue(image.ImageId),
+				imageName,
+				aws.StringValue(i.InstanceId),
+				tags.GetDefault("Name", ""),
+			)
+		}(instance)
+	}
+
+	wg.Wait()
+
+	select {
+	case err := <-errorChan:
+		return err
+	default:
+	}
+
+	return nil
+}
+
+func (m *Manager) all(funcs []func() error) error {
+	var wg sync.WaitGroup
+	errorChan := make(chan error, 1)
+
+	for _, function := range funcs {
+		wg.Add(1)
+		go func(f func() error) {
+			defer wg.Done()
+			if err := f(); err != nil {
+				errorChan <- err
+			}
+		}(function)
+	}
+
+	wg.Wait()
+
+	select {
+	case err := <-errorChan:
+		return err
+	default:
+	}
+
+	return nil
+}
+
+func (m *Manager) formatImageName(i *ec2.Instance) (string, error) {
+	var nameTemplate *template.Template
+	var err error
+	tags := utils.TagSliceToMap(i.Tags)
+	instanceIDString := aws.StringValue(i.InstanceId)
+
+	// User has supplied a template overide for naming the image. We'll need to
+	// template it out.
+	if templateString, ok := tags.Get(m.ImageNameTag); ok {
+		templateName := fmt.Sprintf("image-name-%s", instanceIDString)
+		m.logf("Using custom image name template for instance '%s'\n", instanceIDString)
+		nameTemplate, err = template.New(templateName).Parse(templateString)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		m.logf("Using DefaultImageNameTemplate for instance '%s'\n", instanceIDString)
+		nameTemplate = m.DefaultImageNameTemplate
+	}
+
+	var buf bytes.Buffer
+	// Execute the template
+	err = nameTemplate.Execute(&buf, newImageNameTemplateContext(i))
+	return buf.String(), err
+}
+
+func (m *Manager) log(v ...interface{}) {
+	if m.Verbose {
+		fmt.Println(v...)
+	}
+}
+
+func (m *Manager) logf(format string, v ...interface{}) {
+	if m.Verbose {
+		fmt.Printf(format, v...)
+	}
+}

pkg/backup/manager.go

@@ -0,0 +1,26 @@
+package backup
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/c2fo/lambda-ebs-backup/pkg/utils"
+)
+
+// ImageNameTemplateContext is what gets passed in as the context to the
+// go template when attempting to create the name of the image for backup.
+type ImageNameTemplateContext struct {
+	Name     string
+	Date     string
+	FullDate string
+}
+
+func newImageNameTemplateContext(i *ec2.Instance) ImageNameTemplateContext {
+	tags := utils.TagSliceToMap(i.Tags)
+
+	return ImageNameTemplateContext{
+		Name:     tags.GetDefault("Name", ""),
+		Date:     time.Now().Format("2006-01-02"),
+		FullDate: time.Now().Format("2006-01-02-15-04-05"),
+	}
+}

pkg/backup/template_context.go

@@ -1,6 +1,9 @@
 package config
 
-import "os"
+import (
+	"os"
+	"strconv"
+)
 
 func envDefault(key, defaultValue string) string {
 	value := os.Getenv(key)
@@ -10,6 +13,10 @@ func envDefault(key, defaultValue string) string {
 	return value
 }
 
+func envDefaultInt(key, defaultValue string) (int, error) {
+	return strconv.Atoi(envDefault(key, defaultValue))
+}
+
 // BackupTagKey is the volume tag key to look for when determing if we should
 // perform a backup or not.
 func BackupTagKey() string {
@@ -21,3 +28,41 @@ func BackupTagKey() string {
 func BackupTagValue() string {
 	return envDefault("BACKUP_TAG_VALUE", "true")
 }
+
+// ImageTagKey is the ec2 instance tag key to look for when deciding if we
+// should create an image for the instance.
+func ImageTagKey() string {
+	return envDefault("IMAGE_TAG_KEY", "lambda-ebs-backup/image")
+}
+
+// ImageTagValue is the ec2 instance tag value to look for when deciding if an
+// image should be created for the instance.
+func ImageTagValue() string {
+	return envDefault("IMAGE_TAG_VALUE", "true")
+}
+
+// ImageNameTag is the tag to look for on instances that decides how an image
+// will be named. This tag supports a GO Template and overrides the default
+// ImageNameFormat on an instance by instance basis.
+func ImageNameTag() string {
+	return envDefault("IMAGE_NAME_TAG", "lambda-ebs-backup/image-name")
+}
+
+// DefaultImageNameFormat is the default template to use for nameing ec2 images
+// if a tag is not found. By default, we will use the name of the Instance
+// postfixed with the date.
+func DefaultImageNameFormat() string {
+	return envDefault("DEFAULT_IMAGE_NAME_FORMAT", "{{.Name}}-{{.Date}}")
+}
+
+// MaxKeepImagesTag is the tag to look at for the maximum number of images to
+// keep for an instance.
+func MaxKeepImagesTag() string {
+	return envDefault("MAX_KEEP_IMAGES_TAG", "lambda-ebs-backup/max-keep-images")
+}
+
+// MaxKeepImagesDefault is the default number of images to keep if not specified
+// on the instance via the MaxKeepImagesTag
+func DefaultMaxKeepImages() (int, error) {
+	return envDefaultInt("DEFAULT_MAX_KEEP_IMAGES", "2")
+}

pkg/config/config.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// TagMap stores the ec2 tags as a map and provides some convenience methods.
+type TagMap struct {
+	m map[string]string
+}
+
+// Get gets the tag with the given key and whether or not it exists
+func (tm *TagMap) Get(key string) (string, bool) {
+	v, ok := tm.m[key]
+	return v, ok
+}
+
+// GetDefault returns the value of the tag matching the given key. If there is
+// no tag matching the key, it returns the default value.
+func (tm *TagMap) GetDefault(key string, defaultValue string) string {
+	v, ok := tm.m[key]
+	if !ok {
+		return defaultValue
+	}
+	return v
+}
+
+// TagSliceToMap takes a slice of *ec2.Tags and returns a mapping of their
+// key -> value.
+func TagSliceToMap(tags []*ec2.Tag) TagMap {
+	m := make(map[string]string)
+	for _, t := range tags {
+		m[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	return TagMap{m: m}
+}

pkg/utils/ec2.go

@@ -0,0 +1,35 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTagMap(t *testing.T) {
+	tags := []*ec2.Tag{
+		&ec2.Tag{
+			Key:   aws.String("Name"),
+			Value: aws.String("TestName"),
+		},
+		&ec2.Tag{
+			Key:   aws.String("TestKey"),
+			Value: aws.String("TestValue"),
+		},
+	}
+
+	tm := TagSliceToMap(tags)
+
+	assert.Equal(t, "TestName", tm.GetDefault("Name", "defaultName"))
+	assert.Equal(t, "defaultValue", tm.GetDefault("MissingKey", "defaultValue"))
+
+	val, ok := tm.Get("TestKey")
+	assert.True(t, ok)
+	assert.Equal(t, "TestValue", val)
+
+	val, ok = tm.Get("MissingKey")
+	assert.False(t, ok)
+	assert.Equal(t, "", val)
+}