diff --git a/README.md b/README.md index 675f81b..b85248f 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,89 @@ and will download automatically - Distributed sequential updates - Multiple Artifact Repository Support +## Installation + +To install on ubuntu do the following: + +``` +echo "deb http://dl.bintray.com/chrismckenzie/deb trusty main" >> /etc/apt/sources.list +sudo apt-get update +sudo apt-get install dropship +``` + +## Configuration + +To setup dropship you will need to add/update the following files. + +First you will need to tell dropship how to connect to your artifact repository +so you will need to uncomment out the desired repo and fill in its options. + +_/etc/dropship.d/dropship.hcl_ +```hcl +# vim: set ft=hcl : +# Location that service config will be read from +service_path = "/etc/dropship.d/services" + +# Rackspace Repo Config +# ===================== +rackspace { + user = "" + key = "" + region = "" +} +``` + +You will then have to create a file in the services directory of dropship. this +will tell dropship how to check and install you artifact. You can have multiple +`service` definitions in one file or multiple files. + +_/etc/dropship.d/services/my-service.hcl_ +```hcl +# vim: set ft=hcl : +service "my-service" { + # Use a semaphore to update one machine at a time + sequentialUpdates = true + + # Check for updates every 10s + checkInterval = "10s" + + # Run this command before update starts + before "script" { + command = "initctl my-service stop" + } + + # Artifact defines what repository to use (rackspace) and where + # your artifact live on that repository + artifact "rackspace" { + bucket = "my-container" + path = "my-service.tar.gz" + destination = "./test/dest" + } + + # After successful update send an event to graphite + # this allows you to show deploy annotations in tools like grafana + # + # The graphite hook will automatically add this services name into the + # graphite tags. You also have access to all of the services meta data + # like Name, "current hash", hostname. + after "graphite-event" { + host = "http://" + tags = "deployment" + what = "deployed to {{.Name}} on {{.Hostname}}" + data = "{{.Hash}}" + } + + # Run this command after the update finishes + after "script" { + command = "initctl my-service start" + } +} +``` + ## Roadmap +- [X] Hooks - [ ] Support for Amazon S3, and FTP - [ ] Support for different file types deb, rpm, file _(currently only tar.gz)_ - [ ] Reporting system -- [ ] Hooks - [ ] Redis, etcd for semaphore diff --git a/commands/agent.go b/commands/agent.go index e9d19a9..09a5028 100644 --- a/commands/agent.go +++ b/commands/agent.go @@ -1,14 +1,12 @@ package commands import ( - "io/ioutil" "log" "os" "os/signal" - "path/filepath" "github.com/ChrisMcKenzie/dropship/service" - "github.com/hashicorp/hcl" + "github.com/ChrisMcKenzie/dropship/work" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -21,18 +19,17 @@ var agentCmd = &cobra.Command{ func agent(c *cobra.Command, args []string) { InitializeConfig() - root := viper.GetString("servicePath") - services, err := loadServices(root) + services, err := service.LoadServices(root) if err != nil { log.Fatalln(err) } - t := service.NewRunner(len(services)) + t := work.NewRunner(len(services)) shutdownCh := make(chan struct{}) for _, s := range services { - _, err := service.NewDispatcher(s, t, shutdownCh) + _, err := work.NewDispatcher(s, t, shutdownCh) if err != nil { log.Fatal(err) } @@ -45,21 +42,3 @@ func agent(c *cobra.Command, args []string) { t.Shutdown() } - -func loadServices(root string) (d []service.Config, err error) { - files, _ := filepath.Glob(root + "/*.hcl") - for _, file := range files { - var data []byte - data, err = ioutil.ReadFile(file) - if err != nil { - return - } - - var deploy struct { - Services []service.Config `hcl:"service,expand"` - } - hcl.Decode(&deploy, string(data)) - d = append(d, deploy.Services...) - } - return -} diff --git a/example.hcl b/example.hcl index e783b80..934af84 100644 --- a/example.hcl +++ b/example.hcl @@ -1,14 +1,36 @@ # vim: set ft=hcl : service "my-service" { + # Use a semaphore to update one machine at a time sequentialUpdates = true - checkInterval = "1s" - - preCommand = "echo hello world" - postCommand = "echo hello world" + # Check for updates every 10s + checkInterval = "10s" + + # Run this command before update starts + before "script" { + command = "initctl my-service stop" + } + + # Artifact defines what repository to use (rackspace) and where + # your artifact live on that repository artifact "rackspace" { - bucket = "my-service" - path = "final/blue/my-service.tar.gz" - destination = "./usr/bin" + bucket = "my-container" + path = "my-service.tar.gz" + destination = "./test/dest" + } + + # After successful update send an event to graphite + # this allows you to show deploy annotations in tools like grafana + after "graphite-event" { + host = "http://my-graphite-server" + tags = "my-service deployment" + what = "deployed to {{.Hostname}}" + data = "{{.Hash}}" + } + + # Run this command after the update finishes + after "script" { + command = "initctl my-service start" } } + diff --git a/hook/consul-event.go b/hook/consul-event.go new file mode 100644 index 0000000..36c3be9 --- /dev/null +++ b/hook/consul-event.go @@ -0,0 +1,39 @@ +package hook + +import ( + "encoding/json" + "fmt" + + "github.com/ChrisMcKenzie/dropship/service" + "github.com/hashicorp/consul/api" +) + +type ConsulEventHook struct{} + +func (h ConsulEventHook) Execute(config map[string]interface{}, service service.Config) error { + client, err := api.NewClient(api.DefaultConfig()) + if err != nil { + return err + } + + payload := map[string]string{ + "hash": service.Hash, + } + + plBytes, err := json.Marshal(payload) + if err != nil { + return err + } + + id, meta, err := client.Event().Fire(&api.UserEvent{ + Name: config["name"].(string), + Payload: plBytes, + ServiceFilter: config["service"].(string), + TagFilter: config["tag"].(string), + NodeFilter: config["node"].(string), + }, nil) + + fmt.Println(id, meta, err) + + return err +} diff --git a/hook/consul-event_test.go b/hook/consul-event_test.go new file mode 100644 index 0000000..1af1244 --- /dev/null +++ b/hook/consul-event_test.go @@ -0,0 +1,18 @@ +package hook + +import "testing" + +func TestConsulEventHook(t *testing.T) { + hook := ConsulEventHook{} + + err := hook.Execute(map[string]string{ + "name": "graphite", + "tag": "blue", + "service": "data-service-api-v4", + "node": "api2.data-service-v4.iad", + }) + + if err != nil { + t.Error(err) + } +} diff --git a/hook/graphite-event.go b/hook/graphite-event.go new file mode 100644 index 0000000..25db676 --- /dev/null +++ b/hook/graphite-event.go @@ -0,0 +1,73 @@ +package hook + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "os" + "text/template" + "time" + + "github.com/ChrisMcKenzie/dropship/service" +) + +type GraphiteEventHook struct{} + +func (h GraphiteEventHook) Execute(config map[string]interface{}, service service.Config) error { + host := config["host"].(string) + delete(config, "host") + + config["when"] = time.Now().Unix() + + what, err := parseTemplate(config["what"].(string), service) + if err != nil { + return err + } + + config["what"] = what + + data, err := parseTemplate(config["what"].(string), service) + if err != nil { + return err + } + + config["data"] = data + + config["tags"] = config["tags"].(string) + service.Name + + body, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("Graphite Hook: %s", err) + } + + resp, err := http.Post(host+"/events/", "application/json", bytes.NewReader(body)) + + if err != nil { + return fmt.Errorf("Graphite Hook: %s", err) + } + + if resp.StatusCode >= 400 { + return fmt.Errorf("Graphite Hook: unable to post to events. responded with %d", resp.StatusCode) + } + + return nil +} + +func parseTemplate(temp string, service service.Config) (string, error) { + tmpl, err := template.New("data").Parse(temp) + if err != nil { + return "", err + } + + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + data := TemplateData{service, hostname} + + var buf bytes.Buffer + err = tmpl.Execute(&buf, data) + return buf.String(), err +} diff --git a/hook/graphite-event_test.go b/hook/graphite-event_test.go new file mode 100644 index 0000000..79d1151 --- /dev/null +++ b/hook/graphite-event_test.go @@ -0,0 +1,18 @@ +package hook + +import "testing" + +func TestGraphiteEventHook(t *testing.T) { + var hook GraphiteEventHook + + err := hook.Execute(map[string]string{ + "host": "http://graphite2.analytics.iad", + "what": "deployed by dropship", + "tags": "data-service deployment", + "data": "dropship is awesome!", + }) + + if err != nil { + t.Error(err) + } +} diff --git a/hook/hook.go b/hook/hook.go new file mode 100644 index 0000000..38a8a21 --- /dev/null +++ b/hook/hook.go @@ -0,0 +1,25 @@ +package hook + +import "github.com/ChrisMcKenzie/dropship/service" + +type TemplateData struct { + service.Config + Hostname string +} + +type Hook interface { + Execute(config map[string]interface{}, service service.Config) error +} + +func GetHookByName(name string) Hook { + switch name { + case "script": + return ScriptHook{} + case "consul-event": + return ConsulEventHook{} + case "graphite-event": + return GraphiteEventHook{} + } + + return nil +} diff --git a/hook/script.go b/hook/script.go new file mode 100644 index 0000000..66633f0 --- /dev/null +++ b/hook/script.go @@ -0,0 +1,21 @@ +package hook + +import ( + "os/exec" + "strings" + + "github.com/ChrisMcKenzie/dropship/service" +) + +type ScriptHook struct{} + +func (h ScriptHook) Execute(config map[string]interface{}, service service.Config) error { + _, err := executeCommand(config["command"].(string)) + return err +} + +func executeCommand(c string) (string, error) { + cmd := strings.Fields(c) + out, err := exec.Command(cmd[0], cmd[1:]...).Output() + return string(out), err +} diff --git a/service/service.go b/service/service.go index f476da0..ee60484 100644 --- a/service/service.go +++ b/service/service.go @@ -1,5 +1,12 @@ package service +import ( + "io/ioutil" + "path/filepath" + + "github.com/hashicorp/hcl" +) + type Artifact struct { Type string `hcl:",key"` Bucket string `hcl:"bucket"` @@ -7,11 +14,48 @@ type Artifact struct { Destination string `hcl:"destination"` } +type Hook map[string]map[string]interface{} + type Config struct { - Name string `hcl:",key"` - CheckInterval string `hcl:"checkInterval"` - PostCommand string `hcl:postCommand` - PreCommand string `hcl:preCommand` - Sequential bool `hcl:"sequentialUpdates"` - Artifact Artifact `hcl:"artifact,expand"` + Name string `hcl:",key"` + CheckInterval string `hcl:"checkInterval"` + PostCommand string `hcl:"postCommand"` + PreCommand string `hcl:"preCommand"` + BeforeHooks []Hook `hcl:"before"` + AfterHooks []Hook `hcl:"after"` + Sequential bool `hcl:"sequentialUpdates"` + Artifact []Artifact `hcl:"artifact"` + Hash string +} + +type ServiceFile struct { + Services []Config `hcl:"service"` +} + +func LoadServices(root string) (d []Config, err error) { + files, _ := filepath.Glob(root + "/*.hcl") + for _, file := range files { + data, err := readFile(file) + if err != nil { + return nil, err + } + + var deploy ServiceFile + err = hcl.Decode(&deploy, data) + if err != nil { + return nil, err + } + + d = append(d, deploy.Services...) + } + return +} + +func readFile(file string) (string, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return "", err + } + + return string(data), nil } diff --git a/service/dispatcher.go b/work/dispatcher.go similarity index 74% rename from service/dispatcher.go rename to work/dispatcher.go index 1abb4f7..13502de 100644 --- a/service/dispatcher.go +++ b/work/dispatcher.go @@ -1,4 +1,4 @@ -package service +package work import ( "errors" @@ -7,8 +7,10 @@ import ( "strings" "time" + "github.com/ChrisMcKenzie/dropship/hook" "github.com/ChrisMcKenzie/dropship/installer" "github.com/ChrisMcKenzie/dropship/lock" + "github.com/ChrisMcKenzie/dropship/service" "github.com/ChrisMcKenzie/dropship/updater" "github.com/hashicorp/consul/api" "github.com/spf13/viper" @@ -17,14 +19,14 @@ import ( // Dispatcher is responsible for managing a given services state and // sending work to the Runner pool type Dispatcher struct { - config Config + config service.Config ticker *time.Ticker task *Runner hash string shutdownCh <-chan struct{} } -func NewDispatcher(cfg Config, t *Runner, shutdownCh <-chan struct{}) (*Dispatcher, error) { +func NewDispatcher(cfg service.Config, t *Runner, shutdownCh <-chan struct{}) (*Dispatcher, error) { w := Dispatcher{ config: cfg, task: t, @@ -64,9 +66,9 @@ func (w *Dispatcher) Work() { region := viper.GetString("rackspaceRegion") u := updater.NewRackspaceUpdater(user, key, region) - opts := &updater.Options{w.config.Artifact.Bucket, w.config.Artifact.Path} + opts := &updater.Options{w.config.Artifact[0].Bucket, w.config.Artifact[0].Path} - isOutOfDate, err := u.IsOutdated(w.hash, opts) + isOutOfDate, err := u.IsOutdated(w.config.Hash, opts) if err != nil { log.Printf("[ERR]: Unable to check updates for %s %v", w.config.Name, err) return @@ -96,6 +98,7 @@ func (w *Dispatcher) Work() { } if w.config.PreCommand != "" { + log.Printf("[INF]: preCommand has been deprecated.") res, err := executeCommand(w.config.PreCommand) if err != nil { log.Printf("[ERR]: Unable to execute preCommand. %v", err) @@ -103,18 +106,24 @@ func (w *Dispatcher) Work() { log.Printf("[INF]: preCommand executed successfully. %v", res) } + err = runHooks(w.config.BeforeHooks, w.config) + if err != nil { + log.Printf("[ERR]: Unable to execute beforeHooks. %v", err) + } + i, err := getInstaller(meta.ContentType) if err != nil { log.Printf("[ERR]: %s for %s", w.config.Name, err) return } - filesWritten, err := i.Install(w.config.Artifact.Destination, fr) + filesWritten, err := i.Install(w.config.Artifact[0].Destination, fr) if err != nil { log.Printf("[ERR]: Unable to install update for %s %s", w.config.Name, err) } if w.config.PostCommand != "" { + log.Printf("[INF]: postCommand has been deprecated.") defer func() { res, err := executeCommand(w.config.PostCommand) if err != nil { @@ -127,7 +136,12 @@ func (w *Dispatcher) Work() { log.Printf("[INF]: Update for %s installed successfully. [hash: %s] [files written: %d]", w.config.Name, meta.Hash, filesWritten) // TODO(ChrisMcKenzie): hashes should be stored somewhere more // permanent. - w.hash = meta.Hash + w.config.Hash = meta.Hash + + err = runHooks(w.config.AfterHooks, w.config) + if err != nil { + log.Printf("[ERR]: Unable to execute beforeHooks. %v", err) + } } else { log.Printf("[INF]: %s is up to date", w.config.Name) } @@ -148,3 +162,16 @@ func getInstaller(contentType string) (installer.Installer, error) { return nil, errors.New("Unable to determine installation method from file type") } + +func runHooks(hooks []service.Hook, service service.Config) error { + for _, h := range hooks { + for hookName, config := range h { + hook := hook.GetHookByName(hookName) + if hook != nil { + log.Printf("[INF]: Executing \"%s\" hook with %+v", hookName, config) + hook.Execute(config, service) + } + } + } + return nil +} diff --git a/service/runner.go b/work/runner.go similarity index 97% rename from service/runner.go rename to work/runner.go index 9c18673..1e04575 100644 --- a/service/runner.go +++ b/work/runner.go @@ -1,4 +1,4 @@ -package service +package work import "sync"