Add new webhooks plugin that superseed github and rollbar plugins.

closes influxdata#1289

Signed-off-by: François de Metz <[email protected]>
Signed-off-by: Cyril Duez <[email protected]>

Rename internals struct.

Signed-off-by: François de Metz <[email protected]>
Signed-off-by: Cyril Duez <[email protected]>

Update changelog.

Signed-off-by: François de Metz <[email protected]>
Signed-off-by: Cyril Duez <[email protected]>

Update READMEs and CHANGELOG.

Signed-off-by: François de Metz <[email protected]>
Signed-off-by: Cyril Duez <[email protected]>

Update SampleConfig.

Update the config format.

Update telegraf config.

Update the webhooks README.

Update changelog.

Update the changelog with an upgrade path.

Update default ports.

Fix indent.

Check for nil value on AvailableWebhooks.

Check for CanInterface.

Author: francois2metz

CHANGELOG.md

@@ -1,7 +1,36 @@
 ## v1.0
 
+### Release Notes
+
+**Breaking Change**: users of github_webhooks must change to the new
+`[[inputs.webhooks]]` plugin.
+
+This means that the default github_webhooks config:
+
+```
+# A Github Webhook Event collector
+[[inputs.github_webhooks]]
+  ## Address and port to host Webhook listener on
+  service_address = ":1618"
+```
+
+should now look like:
+
+```
+# A Webhooks Event collector
+[[inputs.webhooks]]
+  ## Address and port to host Webhook listener on
+  service_address = ":1618"
+
+  [inputs.webhooks.github]
+    path = "/"
+```
+
 ### Features
 
+- [#1289](https://github.com/influxdata/telegraf/pull/1289): webhooks input plugin. Thanks @francois2metz and @cduez!
+- [#1247](https://github.com/influxdata/telegraf/pull/1247): rollbar webhook plugin.
+
 ### Bugfixes
 
 - [#1384](https://github.com/influxdata/telegraf/pull/1384): Fix datarace in apache input plugin.
@@ -50,11 +79,11 @@ in conjunction with wildcard dimension values as it will control the amount of
 time before a new metric is included by the plugin.
 
 ### Features
+
 - [#1262](https://github.com/influxdata/telegraf/pull/1261): Add graylog input pluging.
 - [#1294](https://github.com/influxdata/telegraf/pull/1294): consul input plugin. Thanks @harnash
 - [#1164](https://github.com/influxdata/telegraf/pull/1164): conntrack input plugin. Thanks @robinpercy!
 - [#1165](https://github.com/influxdata/telegraf/pull/1165): vmstat input plugin. Thanks @jshim-xm!
-- [#1247](https://github.com/influxdata/telegraf/pull/1247): rollbar input plugin. Thanks @francois2metz and @cduez!
 - [#1208](https://github.com/influxdata/telegraf/pull/1208): Standardized AWS credentials evaluation & wildcard CloudWatch dimensions. Thanks @johnrengelman!
 - [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin.
 - [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser.

README.md

@@ -217,8 +217,9 @@ Telegraf can also collect metrics via the following service plugins:
 * [mqtt_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mqtt_consumer)
 * [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer)
 * [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer)
-* [github_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/github_webhooks)
-* [rollbar_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rollbar_webhooks)
+* [webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks)
+  * [github](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/github)
+  * [rollbar](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/rollbar)
 
 We'll be adding support for many more over the coming months. Read on if you
 want to add support for another service or third-party API.

etc/telegraf.conf

@@ -1490,12 +1490,6 @@
 #                            SERVICE INPUT PLUGINS                            #
 ###############################################################################
 
-# # A Github Webhook Event collector
-# [[inputs.github_webhooks]]
-#   ## Address and port to host Webhook listener on
-#   service_address = ":1618"
-
-
 # # Read metrics from Kafka topic(s)
 # [[inputs.kafka_consumer]]
 #   ## topic(s) to consume
@@ -1601,12 +1595,6 @@
 #   data_format = "influx"
 
 
-# # A Rollbar Webhook Event collector
-# [[inputs.rollbar_webhooks]]
-#   ## Address and port to host Webhook listener on
-#   service_address = ":1619"
-
-
 # # Statsd Server
 # [[inputs.statsd]]
 #   ## Address and port to host UDP listener on
@@ -1701,3 +1689,15 @@
 #   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
 #   data_format = "influx"
 
+
+# # A Webhooks Event collector
+# [[inputs.webhooks]]
+#   ## Address and port to host Webhook listener on
+#   service_address = ":1619"
+#
+#   [inputs.webhooks.github]
+#     path = "/github"
+#
+#   [inputs.webhooks.rollbar]
+#     path = "/rollbar"
+

plugins/inputs/all/all.go

@@ -19,7 +19,6 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
 	_ "github.com/influxdata/telegraf/plugins/inputs/exec"
 	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
-	_ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
 	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
@@ -57,7 +56,6 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/riak"
-	_ "github.com/influxdata/telegraf/plugins/inputs/rollbar_webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sensors"
 	_ "github.com/influxdata/telegraf/plugins/inputs/snmp"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
@@ -70,6 +68,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
 	_ "github.com/influxdata/telegraf/plugins/inputs/varnish"
+	_ "github.com/influxdata/telegraf/plugins/inputs/webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zfs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"

plugins/inputs/rollbar_webhooks/rollbar_webhooks.go

@@ -0,0 +1,27 @@
+# Webhooks
+
+This is a Telegraf service plugin that start an http server and register multiple webhook listeners.
+
+```sh
+$ telegraf -sample-config -input-filter webhooks -output-filter influxdb > config.conf.new
+```
+
+Change the config file to point to the InfluxDB server you are using and adjust the settings to match your environment. Once that is complete:
+
+```sh
+$ cp config.conf.new /etc/telegraf/telegraf.conf
+$ sudo service telegraf start
+```
+
+## Available webhooks
+
+- [Github](github/)
+- [Rollbar](rollbar/)
+
+## Adding new webhooks plugin
+
+1. Add your webhook plugin inside the `webhooks` folder
+1. Your plugin must implement the `Webhook` interface
+1. Import your plugin in the `webhooks.go` file and add it to the `Webhooks` struct
+
+Both [Github](github/) and [Rollbar](rollbar/) are good example to follow.

plugins/inputs/webhooks/README.md

@@ -1,15 +1,6 @@
-# github_webhooks
+# github webhooks
 
-This is a Telegraf service plugin that listens for events kicked off by Github's Webhooks service and persists data from them into configured outputs. To set up the listener first generate the proper configuration:
-```sh
-$ telegraf -sample-config -input-filter github_webhooks -output-filter influxdb > config.conf.new
-```
-Change the config file to point to the InfluxDB server you are using and adjust the settings to match your environment. Once that is complete:
-```sh
-$ cp config.conf.new /etc/telegraf/telegraf.conf
-$ sudo service telegraf start
-```
-Once the server is running you should configure your Organization's Webhooks to point at the `github_webhooks` service. To do this go to `github.com/{my_organization}` and click `Settings > Webhooks > Add webhook`. In the resulting menu set `Payload URL` to `http://<my_ip>:1618`, `Content type` to `application/json` and under the section `Which events would you like to trigger this webhook?` select 'Send me <b>everything</b>'. By default all of the events will write to the `github_webhooks` measurement, this is configurable by setting the `measurement_name` in the config file.
+You should configure your Organization's Webhooks to point at the `webhooks` service. To do this go to `github.com/{my_organization}` and click `Settings > Webhooks > Add webhook`. In the resulting menu set `Payload URL` to `http://<my_ip>:1619/github`, `Content type` to `application/json` and under the section `Which events would you like to trigger this webhook?` select 'Send me <b>everything</b>'. By default all of the events will write to the `github_webhooks` measurement, this is configurable by setting the `measurement_name` in the config file.
 
 ## Events
 

plugins/inputs/github_webhooks/README.md plugins/inputs/webhooks/github/README.md

@@ -1,78 +1,27 @@
-package github_webhooks
+package github
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
-	"sync"
 
 	"github.com/gorilla/mux"
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
-func init() {
-	inputs.Add("github_webhooks", func() telegraf.Input { return &GithubWebhooks{} })
+type GithubWebhook struct {
+	Path string
+	acc  telegraf.Accumulator
 }
 
-type GithubWebhooks struct {
-	ServiceAddress string
-	// Lock for the struct
-	sync.Mutex
-	// Events buffer to store events between Gather calls
-	events []Event
+func (gh *GithubWebhook) Register(router *mux.Router, acc telegraf.Accumulator) {
+	router.HandleFunc(gh.Path, gh.eventHandler).Methods("POST")
+	log.Printf("Started the webhooks_github on %s\n", gh.Path)
+	gh.acc = acc
 }
 
-func NewGithubWebhooks() *GithubWebhooks {
-	return &GithubWebhooks{}
-}
-
-func (gh *GithubWebhooks) SampleConfig() string {
-	return `
-  ## Address and port to host Webhook listener on
-  service_address = ":1618"
-`
-}
-
-func (gh *GithubWebhooks) Description() string {
-	return "A Github Webhook Event collector"
-}
-
-// Writes the points from <-gh.in to the Accumulator
-func (gh *GithubWebhooks) Gather(acc telegraf.Accumulator) error {
-	gh.Lock()
-	defer gh.Unlock()
-	for _, event := range gh.events {
-		p := event.NewMetric()
-		acc.AddFields("github_webhooks", p.Fields(), p.Tags(), p.Time())
-	}
-	gh.events = make([]Event, 0)
-	return nil
-}
-
-func (gh *GithubWebhooks) Listen() {
-	r := mux.NewRouter()
-	r.HandleFunc("/", gh.eventHandler).Methods("POST")
-	err := http.ListenAndServe(fmt.Sprintf("%s", gh.ServiceAddress), r)
-	if err != nil {
-		log.Printf("Error starting server: %v", err)
-	}
-}
-
-func (gh *GithubWebhooks) Start(_ telegraf.Accumulator) error {
-	go gh.Listen()
-	log.Printf("Started the github_webhooks service on %s\n", gh.ServiceAddress)
-	return nil
-}
-
-func (gh *GithubWebhooks) Stop() {
-	log.Println("Stopping the ghWebhooks service")
-}
-
-// Handles the / route
-func (gh *GithubWebhooks) eventHandler(w http.ResponseWriter, r *http.Request) {
+func (gh *GithubWebhook) eventHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	eventType := r.Header["X-Github-Event"][0]
 	data, err := ioutil.ReadAll(r.Body)
@@ -85,9 +34,10 @@ func (gh *GithubWebhooks) eventHandler(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	gh.Lock()
-	gh.events = append(gh.events, e)
-	gh.Unlock()
+
+	p := e.NewMetric()
+	gh.acc.AddFields("github_webhooks", p.Fields(), p.Tags(), p.Time())
+
 	w.WriteHeader(http.StatusOK)
 }